From 2be94735a83c0236b55ce2f044ad6f1d0a29686a Mon Sep 17 00:00:00 2001 From: Marek Trtik Date: Thu, 12 Sep 2024 23:44:16 +0200 Subject: [PATCH 001/144] Ignore development benchmark --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 68ddfe43..7c5c0585 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ /CMakeSettings.json /cmake-build-* **/__pycache__ -/benchmarks/pending/__debug__* \ No newline at end of file +/benchmarks/pending/__debug__* +/benchmarks/fast/_.c \ No newline at end of file From 9f0fd38cfa185797bd0b476645ce73959328df85 Mon Sep 17 00:00:00 2001 From: Marek Trtik Date: Sat, 14 Sep 2024 12:50:57 +0200 Subject: [PATCH 002/144] Fixed project's name --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9e075b3b..b5ccd054 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -project(fizz) +project(fizzer) cmake_minimum_required(VERSION 3.20 FATAL_ERROR) From f4fa23662b6c35882b327dba68d1695715baf156 Mon Sep 17 00:00:00 2001 From: Marek Trtik Date: Sat, 14 Sep 2024 12:51:38 +0200 Subject: [PATCH 003/144] SIlencing boost's wait_for 'deprecated' message --- src/connection/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/connection/CMakeLists.txt b/src/connection/CMakeLists.txt index 7ca16635..d683d749 100644 --- a/src/connection/CMakeLists.txt +++ b/src/connection/CMakeLists.txt @@ -30,6 +30,8 @@ add_library(${THIS_TARGET_NAME} ./src/benchmark_executor.cpp ) +target_compile_options(${THIS_TARGET_NAME} PUBLIC "-Wno-deprecated") + set_target_properties(${THIS_TARGET_NAME} PROPERTIES DEBUG_OUTPUT_NAME "${THIS_TARGET_NAME}_${CMAKE_SYSTEM_NAME}_Debug" RELEASE_OUTPUT_NAME "${THIS_TARGET_NAME}_${CMAKE_SYSTEM_NAME}_Release" From c3c9eda3f0ad08cbf4d4e91aac126f441d0932b3 Mon Sep 17 00:00:00 2001 From: Marek Trtik Date: Sat, 14 Sep 2024 12:52:37 +0200 Subject: [PATCH 004/144] Instrumenter: Using new LLVM's PassManager --- src/tools/instrumenter/llvm_instrumenter.cpp | 1 + src/tools/instrumenter/llvm_instrumenter.hpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tools/instrumenter/llvm_instrumenter.cpp b/src/tools/instrumenter/llvm_instrumenter.cpp index dcf8b43c..6b01e422 100644 --- a/src/tools/instrumenter/llvm_instrumenter.cpp +++ b/src/tools/instrumenter/llvm_instrumenter.cpp @@ -5,6 +5,7 @@ # pragma warning(disable:4624) // LLVM: warning C4624: 'llvm::detail::copy_construction_triviality_helper': destructor was implicitly defined # pragma warning(disable:4146) // LLVM: warning C4146: unary minus operator applied to unsigned type, result still unsigned #endif +#include #include #include #include diff --git a/src/tools/instrumenter/llvm_instrumenter.hpp b/src/tools/instrumenter/llvm_instrumenter.hpp index 1a812daa..9778a381 100644 --- a/src/tools/instrumenter/llvm_instrumenter.hpp +++ b/src/tools/instrumenter/llvm_instrumenter.hpp @@ -13,10 +13,10 @@ # endif # include # include +# include # include # include # include -# include # include # include # include From 9db751f99ba8e58855850262406968ee9ea41a58 Mon Sep 17 00:00:00 2001 From: Marek Trtik Date: Sat, 14 Sep 2024 13:21:41 +0200 Subject: [PATCH 005/144] Updated README - added install of 32-bit std lib on Linux. --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index c0c832d4..6a169a36 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,11 @@ start with the **age** project: tasks, e.g., building benchmarks and killing non-terminating clients. You only need to copy the file from the `setup` folder to the folder `.vscode` folder. +- (Optional) If you also want to analyze 32-bit programs, then you must also + build 32-bit version of Fizzer's libraries. That is done automatically via + Fizzer's `build.sh` script. However, 32-bit version of C++ standard library + must be available in the C++ compiler. On Linux (Ubuntu) you can install + this library using: `sudo apt install g++-multilib` - (optional) **SmartGit** Git GUI client: https://www.syntevo.com/smartgit/ ## Downloading **SBT-Fizzer** From 0bf0d047a5364dc78bfe770f92bbb70259ebdaa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sun, 15 Sep 2024 22:00:22 +0200 Subject: [PATCH 006/144] feat: creating equation matrix --- src/fuzzing/include/fuzzing/fuzzer.hpp | 35 ++++++++++++++- src/fuzzing/src/fuzzer.cpp | 61 ++++++++++++++++++++++++-- 2 files changed, 91 insertions(+), 5 deletions(-) diff --git a/src/fuzzing/include/fuzzing/fuzzer.hpp b/src/fuzzing/include/fuzzing/fuzzer.hpp index 3db126f7..946fce5a 100644 --- a/src/fuzzing/include/fuzzing/fuzzer.hpp +++ b/src/fuzzing/include/fuzzing/fuzzer.hpp @@ -17,6 +17,7 @@ # include # include # include +# include namespace fuzzing { @@ -217,6 +218,36 @@ struct fuzzer final mutable random_generator_for_natural_32_bit generator_for_pivot_selection; }; + struct iid_node_dependence + { + + struct node_navigation + { + location_id node_id; + bool direction; + + auto operator<=>(node_navigation const& other) const + { + if(auto const cmp = node_id.id <=> other.node_id.id; cmp != 0) + return cmp; + + return direction <=> other.direction; + } + }; + + struct iid_dependence_props + { + std::vector all_paths; + std::set interestins_nodes; + std::vector> matrix; + + void recompute_matrix(); + }; + + std::unordered_map id_to_equation_map; + std::set non_iid_nodes; + }; + static void update_close_flags_from(branching_node* node); static std::vector const& get_input_width_classes(); @@ -285,6 +316,7 @@ struct fuzzer final ); void generate_next_input(vecb& stdin_bits); + void process_node_dependance(branching_node* node); execution_record::execution_flags process_execution_results(); void do_cleanup(); @@ -310,8 +342,9 @@ struct fuzzer final primary_coverage_target_branchings primary_coverage_targets; std::unordered_map iid_pivots; + iid_node_dependence iid_dependences; - std::unordered_set coverage_failures_with_hope; // EXPLAIN + std::unordered_set coverage_failures_with_hope; STATE state; sensitivity_analysis sensitivity; diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index 02ba7ce1..9425af74 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -813,6 +813,7 @@ fuzzer::fuzzer(termination_info const& info) &statistics } , iid_pivots{} + , iid_dependences{} , coverage_failures_with_hope{} @@ -971,6 +972,51 @@ void fuzzer::generate_next_input(vecb& stdin_bits) UNREACHABLE(); } +void fuzzer::process_node_dependance(branching_node *node) +{ + using deps_props = iid_node_dependence::iid_dependence_props; + using node_direction = iid_node_dependence::node_navigation; + + if (iid_dependences.non_iid_nodes.contains(node->get_location_id())) + return; + deps_props& props = iid_dependences.id_to_equation_map[node->get_location_id()]; + + props.all_paths.push_back(node); + props.recompute_matrix(); +} + +void fuzzing::fuzzer::iid_node_dependence::iid_dependence_props::recompute_matrix() +{ + using node_nav = iid_node_dependence::node_navigation; + if (all_paths.empty()) + return; + + matrix.clear(); // This could be done better, but for now it's fine + + for (const auto& path : all_paths) { + std::map directions_in_path; + + branching_node* node = path; + branching_node* prev_node = node->predecessor; + + while (prev_node != nullptr) { + bool direction = prev_node->successor(true).pointer == node; + node_nav nav = { prev_node->get_location_id(), direction }; + directions_in_path[nav]++; + + node = prev_node; + prev_node = node->predecessor; + } + + std::vector values_in_path; + for (const auto& [direction, count] : directions_in_path) { + values_in_path.push_back(count); + } + + matrix.push_back(values_in_path); + } + +} execution_record::execution_flags fuzzer::process_execution_results() { @@ -1009,6 +1055,8 @@ execution_record::execution_flags fuzzer::process_execution_results() ); construction_props.diverging_node = entry_branching; + process_node_dependance(entry_branching); + ++statistics.nodes_created; } @@ -1074,9 +1122,7 @@ execution_record::execution_flags fuzzer::process_execution_results() node->set_closed(false); branching_coverage_info const& succ_info = trace->at(trace_index + 1); - construction_props.leaf->set_successor(info.direction, { - branching_node::successor_pointer::VISITED, - new branching_node( + auto new_node = new branching_node( succ_info.id, trace_index + 1, succ_info.num_input_bytes, @@ -1088,9 +1134,14 @@ execution_record::execution_flags fuzzer::process_execution_results() succ_info.value * succ_info.value, num_driver_executions, succ_info.xor_like_branching_function - ) + ); + construction_props.leaf->set_successor(info.direction, { + branching_node::successor_pointer::VISITED, + new_node }); + process_node_dependance(new_node); + ++statistics.nodes_created; if (construction_props.diverging_node == nullptr) @@ -1321,6 +1372,8 @@ void fuzzer::collect_iid_pivots_from_sensitivity_results() pivots.push_back({ node, &pivot_it_and_state.first->second }); } + + // iid_dependence_props& deps = iid_dependences[node->get_location_id()]; } if (pivots.empty()) return; From 712a18d73098af05af7823a97e7e8f09cf29a4eb Mon Sep 17 00:00:00 2001 From: Marek Trtik Date: Mon, 16 Sep 2024 12:31:56 +0200 Subject: [PATCH 007/144] Use static linking with boost. --- CMakeLists.txt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b5ccd054..37536a89 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -70,11 +70,11 @@ find_package(Threads REQUIRED) # find and add Boost message("Searching for Boost library ...") -find_package(Boost 1.69 REQUIRED COMPONENTS filesystem) +set(Boost_USE_STATIC_LIBS ON) +find_package(Boost REQUIRED COMPONENTS filesystem) message("Boost STATUS: Includes ${Boost_INCLUDE_DIRS} - Libs ${Boost_LIBRARIES} -") + Libs ${Boost_LIBRARIES}") include_directories(${Boost_INCLUDE_DIRS}) set(LLVM_EXPORT_SYMBOLS_FOR_PLUGINS "yes") @@ -92,8 +92,7 @@ message("LLVM STATUS: Definitions ${LLVM_DEFINITIONS} Includes ${LLVM_INCLUDE_DIRS} Libraries ${LLVM_LIBRARY_DIRS} - Targets ${LLVM_TARGETS_TO_BUILD}" -) + Targets ${LLVM_TARGETS_TO_BUILD}") if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) From 9b56efe45de3f35adbab564f00dfc568b9589714 Mon Sep 17 00:00:00 2001 From: Marek Trtik Date: Mon, 16 Sep 2024 18:13:45 +0200 Subject: [PATCH 008/144] Cleanup in CMakeLists files. --- CMakeLists.txt | 7 +++-- src/CMakeLists.txt | 18 ++++++----- src/tools/client/CMakeLists.txt | 2 +- src/tools/server/CMakeLists.txt | 2 +- src/utility/CMakeLists.txt | 55 +++------------------------------ 5 files changed, 22 insertions(+), 62 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 37536a89..049b514b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -76,6 +76,7 @@ message("Boost STATUS: Includes ${Boost_INCLUDE_DIRS} Libs ${Boost_LIBRARIES}") include_directories(${Boost_INCLUDE_DIRS}) +set(BOOST_LIST_OF_LIBRARIES_TO_LINK_WITH "${Boost_LIBRARIES}") set(LLVM_EXPORT_SYMBOLS_FOR_PLUGINS "yes") find_package(LLVM REQUIRED CONFIG) @@ -112,7 +113,9 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/src") # Add project specific sources add_subdirectory(./src) -install(DIRECTORY ./benchmarks DESTINATION .) -install(FILES ./LICENSE.txt DESTINATION .) +if(FIZZ_BUILD_LIBS_32_BIT STREQUAL "No") + install(DIRECTORY ./benchmarks DESTINATION .) + install(FILES ./LICENSE.txt DESTINATION .) +endif() message("Generating build files ...") diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4423b9b7..922c109e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -20,13 +20,15 @@ add_subdirectory(./iomodels) message(" iomodels") add_subdirectory(./connection) message(" connection") - + if(FIZZ_BUILD_ALSO_TOOLS STREQUAL "Yes") - include_directories( - "${PROJECT_SOURCE_DIR}/src/fuzzing/include" - "${PROJECT_SOURCE_DIR}/src/tools" - ) - add_subdirectory(./fuzzing) - message(" fuzzing") - add_subdirectory(./tools) + if(FIZZ_BUILD_LIBS_32_BIT STREQUAL "No") + include_directories( + "${PROJECT_SOURCE_DIR}/src/fuzzing/include" + "${PROJECT_SOURCE_DIR}/src/tools" + ) + add_subdirectory(./fuzzing) + message(" fuzzing") + add_subdirectory(./tools) + endif() endif() diff --git a/src/tools/client/CMakeLists.txt b/src/tools/client/CMakeLists.txt index f4809630..86630d43 100644 --- a/src/tools/client/CMakeLists.txt +++ b/src/tools/client/CMakeLists.txt @@ -17,7 +17,7 @@ target_link_libraries(${THIS_TARGET_NAME} instrumentation iomodels utility - ${Boost_LIBRARIES} + ${BOOST_LIST_OF_LIBRARIES_TO_LINK_WITH} Threads::Threads ) diff --git a/src/tools/server/CMakeLists.txt b/src/tools/server/CMakeLists.txt index 3253073f..6da40dbe 100644 --- a/src/tools/server/CMakeLists.txt +++ b/src/tools/server/CMakeLists.txt @@ -21,7 +21,7 @@ target_link_libraries(${THIS_TARGET_NAME} fuzzing # Duplicating because of Mingw (otherwise linking errors) utility Threads::Threads - ${Boost_LIBRARIES} + ${BOOST_LIST_OF_LIBRARIES_TO_LINK_WITH} ) if (LIBRT) diff --git a/src/utility/CMakeLists.txt b/src/utility/CMakeLists.txt index 328cebd5..1620e9ab 100644 --- a/src/utility/CMakeLists.txt +++ b/src/utility/CMakeLists.txt @@ -1,56 +1,11 @@ set(THIS_TARGET_NAME utility) -add_library(${THIS_TARGET_NAME} - ./include/utility/assumptions.hpp - - ./include/utility/basic_numeric_types.hpp - - ./src/bits_reference.cpp - ./include/utility/bits_reference.hpp - - ./src/bit_count.cpp - ./include/utility/bit_count.hpp - - ./include/utility/config.hpp - - ./include/utility/development.hpp - - ./include/utility/endian.hpp - - ./src/fail_message.cpp - ./include/utility/fail_message.hpp - - ./include/utility/invariants.hpp - - ./src/log.cpp - ./include/utility/log.hpp +file(GLOB "${THIS_TARGET_NAME}_HPP" "./include/${THIS_TARGET_NAME}/*.hpp") +file(GLOB "${THIS_TARGET_NAME}_CPP" "./src/*.cpp") - ./src/timestamp.cpp - ./include/utility/timestamp.hpp - - ./src/timeprof.cpp - ./include/utility/timeprof.hpp - - ./src/checked_number_operations.cpp - ./include/utility/checked_number_operations.hpp - - ./src/random.cpp - ./include/utility/random.hpp - - ./include/utility/typefn_if_then_else.hpp - - ./include/utility/msgstream.hpp - - ./include/utility/dynamic_linking.hpp - - ./include/utility/hash_combine.hpp - ./include/utility/std_pair_hash.hpp - - ./include/utility/program_options_base.hpp - ./src/program_options_base.cpp - - include/utility/math.hpp - src/math.cpp +add_library(${THIS_TARGET_NAME} + "${${THIS_TARGET_NAME}_HPP}" + "${${THIS_TARGET_NAME}_CPP}" ) set_target_properties(${THIS_TARGET_NAME} PROPERTIES From cfce1f71699f24deb79d56d9d534bcbe4ba2ed88 Mon Sep 17 00:00:00 2001 From: Marek Trtik Date: Mon, 16 Sep 2024 18:14:52 +0200 Subject: [PATCH 009/144] Connection: Introduced 'medium' as parent of 'message' and 'shared_memory' --- src/connection/CMakeLists.txt | 2 ++ src/connection/include/connection/medium.hpp | 23 +++++++++++++++++++ src/connection/include/connection/message.hpp | 19 ++++++++------- .../include/connection/shared_memory.hpp | 22 ++++++++++-------- src/connection/src/target_executor.cpp | 2 +- 5 files changed, 50 insertions(+), 18 deletions(-) create mode 100644 src/connection/include/connection/medium.hpp diff --git a/src/connection/CMakeLists.txt b/src/connection/CMakeLists.txt index d683d749..7b062ad3 100644 --- a/src/connection/CMakeLists.txt +++ b/src/connection/CMakeLists.txt @@ -7,6 +7,8 @@ add_library(${THIS_TARGET_NAME} ./include/connection/client.hpp ./src/client.cpp + ./include/connection/medium.hpp + ./include/connection/shared_memory.hpp ./src/shared_memory.cpp diff --git a/src/connection/include/connection/medium.hpp b/src/connection/include/connection/medium.hpp new file mode 100644 index 00000000..4195d284 --- /dev/null +++ b/src/connection/include/connection/medium.hpp @@ -0,0 +1,23 @@ +#ifndef CONNECTION_MEDIUM_HPP_INCLUDED +# define CONNECTION_MEDIUM_HPP_INCLUDED + +# include + +namespace connection { + + +struct medium +{ + virtual ~medium() {} + virtual void clear() {} + virtual bool can_accept_bytes(std::size_t n) const { return true; } + virtual bool can_deliver_bytes(std::size_t n) const { return true; } + virtual void accept_bytes(const void* src, std::size_t n) {} + virtual void deliver_bytes(void* dest, std::size_t n) {} + virtual void set_termination(instrumentation::target_termination termination) {} +}; + + +} + +#endif \ No newline at end of file diff --git a/src/connection/include/connection/message.hpp b/src/connection/include/connection/message.hpp index 14033aff..51b1e2a5 100644 --- a/src/connection/include/connection/message.hpp +++ b/src/connection/include/connection/message.hpp @@ -1,6 +1,7 @@ #ifndef CONNECTION_MESSAGE_HPP_INCLUDED # define CONNECTION_MESSAGE_HPP_INCLUDED +# include # include # include @@ -17,16 +18,18 @@ friend struct message; }; -struct message +struct message : public medium { + message() : medium() {} + natural_32_bit size(); - void clear(); + void clear() override; bool empty(); - bool can_accept_bytes(size_t n) const; - bool can_deliver_bytes(size_t n) const; - void accept_bytes(const void* src, size_t n); - void deliver_bytes(void* dest, size_t n); + bool can_accept_bytes(size_t n) const override; + bool can_deliver_bytes(size_t n) const override; + void accept_bytes(const void* src, size_t n) override; + void deliver_bytes(void* dest, size_t n) override; bool exhausted() const; @@ -49,9 +52,9 @@ struct message message& operator>>(std::string& dest); - message_header header; + message_header header{}; private: - vecu8 bytes; + vecu8 bytes{}; natural_32_bit cursor = 0; friend struct connection; diff --git a/src/connection/include/connection/shared_memory.hpp b/src/connection/include/connection/shared_memory.hpp index 7cab5e20..c59320d5 100644 --- a/src/connection/include/connection/shared_memory.hpp +++ b/src/connection/include/connection/shared_memory.hpp @@ -1,6 +1,7 @@ #ifndef CONNECTION_SHARED_MEMORY_HPP_INCLUDED # define CONNECTION_SHARED_MEMORY_HPP_INCLUDED +# include # include # include # include @@ -13,29 +14,32 @@ namespace connection { -class shared_memory { +class shared_memory : public medium { inline static const char* segment_name = "SBT-Fizzer_Shared_Memory"; - boost::interprocess::shared_memory_object shm; - boost::interprocess::mapped_region region; + boost::interprocess::shared_memory_object shm{}; + boost::interprocess::mapped_region region{}; natural_32_bit cursor = 0; natural_8_bit* memory = nullptr; natural_32_bit* saved = nullptr; public: + + shared_memory() : medium() {} + natural_32_bit get_size() const; void set_size(natural_32_bit bytes); - void clear(); + void clear() override; void open_or_create(); void map_region(); static void remove(); - bool can_accept_bytes(size_t n) const; - bool can_deliver_bytes(size_t n) const; + bool can_accept_bytes(size_t n) const override; + bool can_deliver_bytes(size_t n) const override; - void accept_bytes(const void* src, size_t n); - void deliver_bytes(void* dest, size_t n); + void accept_bytes(const void* src, size_t n) override; + void deliver_bytes(void* dest, size_t n) override; void accept_bytes(message& src); void deliver_bytes(message& dest); @@ -45,7 +49,7 @@ class shared_memory { /*Interprets the first two bytes as termination type*/ std::optional get_termination() const; /*Overwrites the first two bytes to set termination type*/ - void set_termination(instrumentation::target_termination termination); + void set_termination(instrumentation::target_termination termination) override; template::value, int>::type = 0> shared_memory& operator<<(const T& src) diff --git a/src/connection/src/target_executor.cpp b/src/connection/src/target_executor.cpp index 20d3b42a..727cf01b 100644 --- a/src/connection/src/target_executor.cpp +++ b/src/connection/src/target_executor.cpp @@ -36,7 +36,7 @@ void target_executor::init_shared_memory(std::size_t size) { void target_executor::execute_target() { using namespace std::chrono_literals; - bp::child target = bp::child(target_invocation, bp::std_out > bp::null); + bp::child target = bp::child(target_invocation, bp::std_out > bp::null, bp::std_err > bp::null); if (!wait_for_wrapper(target, std::chrono::milliseconds(timeout_ms))) { target.terminate(); get_shared_memory().set_termination(target_termination::timeout); From f867b6805a67902f4236afe1af942498950cdeee Mon Sep 17 00:00:00 2001 From: Marek Trtik Date: Mon, 16 Sep 2024 18:15:51 +0200 Subject: [PATCH 010/144] Cleanup in iomodels --- src/iomodels/include/iomodels/stdin_base.hpp | 3 ++- .../stdin_replay_bytes_then_repeat_byte.hpp | 2 +- .../stdin_replay_bytes_then_repeat_byte.cpp | 19 ++++++++++++------- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/iomodels/include/iomodels/stdin_base.hpp b/src/iomodels/include/iomodels/stdin_base.hpp index 3a526fc7..9371a1fe 100644 --- a/src/iomodels/include/iomodels/stdin_base.hpp +++ b/src/iomodels/include/iomodels/stdin_base.hpp @@ -26,7 +26,8 @@ struct stdin_base virtual bool load_record(connection::message& src) = 0; virtual bool load_record(connection::shared_memory& src) = 0; virtual size_t min_flattened_size() const = 0; - virtual void read(natural_8_bit* ptr, type_of_input_bits type, connection::shared_memory& dest) = 0; + void read(natural_8_bit* ptr, type_of_input_bits type, connection::medium& dest) { if (!read_bytes(ptr, type, dest)) exit(0); } + virtual bool read_bytes(natural_8_bit* ptr, type_of_input_bits type, connection::medium& dest) = 0; virtual vecu8 const& get_bytes() const = 0; virtual input_types_vector const& get_types() const = 0; diff --git a/src/iomodels/include/iomodels/stdin_replay_bytes_then_repeat_byte.hpp b/src/iomodels/include/iomodels/stdin_replay_bytes_then_repeat_byte.hpp index b48546c9..7e224d10 100644 --- a/src/iomodels/include/iomodels/stdin_replay_bytes_then_repeat_byte.hpp +++ b/src/iomodels/include/iomodels/stdin_replay_bytes_then_repeat_byte.hpp @@ -19,7 +19,7 @@ struct stdin_replay_bytes_then_repeat_byte : public stdin_base bool load_record(connection::message& src) override; bool load_record(connection::shared_memory& src) override; std::size_t min_flattened_size() const override; - void read(natural_8_bit* ptr, type_of_input_bits type, connection::shared_memory& dest) override; + bool read_bytes(natural_8_bit* ptr, type_of_input_bits type, connection::medium& dest) override; vecu8 const& get_bytes() const override { return bytes; } input_types_vector const& get_types() const override { return types; } diff --git a/src/iomodels/src/stdin_replay_bytes_then_repeat_byte.cpp b/src/iomodels/src/stdin_replay_bytes_then_repeat_byte.cpp index ef486ecd..191a8ec2 100644 --- a/src/iomodels/src/stdin_replay_bytes_then_repeat_byte.cpp +++ b/src/iomodels/src/stdin_replay_bytes_then_repeat_byte.cpp @@ -119,28 +119,33 @@ std::size_t stdin_replay_bytes_then_repeat_byte::min_flattened_size() const { } - -void stdin_replay_bytes_then_repeat_byte::read(natural_8_bit* ptr, - type_of_input_bits const type, - shared_memory& dest) +bool stdin_replay_bytes_then_repeat_byte::read_bytes( + natural_8_bit* ptr, type_of_input_bits const type, medium& dest + ) { natural_8_bit const count = num_bytes(type); if (cursor + count > max_bytes()) { dest.set_termination(target_termination::boundary_condition_violation); - exit(0); + return false; } if (!dest.can_accept_bytes(count + 2)) { dest.set_termination(target_termination::medium_overflow); - exit(0); + return false; } if (cursor + count > bytes.size()) { bytes.resize(cursor + count, repeat_byte); } memcpy(ptr, bytes.data() + cursor, count); - dest << data_record_id::stdin_bytes << to_id(type); + { + auto const rec_type{ data_record_id::stdin_bytes }; + dest.accept_bytes(&rec_type, sizeof(rec_type)); + auto const type_id{ to_id(type) }; + dest.accept_bytes(&type_id, sizeof(type_id)); + } dest.accept_bytes(bytes.data() + cursor, count); cursor += count; types.push_back(type); + return true; } From 723cbd37e35ff15893942b80210f6fad09293c08 Mon Sep 17 00:00:00 2001 From: Marek Trtik Date: Mon, 16 Sep 2024 18:17:03 +0200 Subject: [PATCH 011/144] Cleanup in fuzz_target --- .../include/instrumentation/fuzz_target.hpp | 7 ++++++- src/instrumentation/src/fuzz_target.cpp | 12 +++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/instrumentation/include/instrumentation/fuzz_target.hpp b/src/instrumentation/include/instrumentation/fuzz_target.hpp index b95c703a..d519aa00 100644 --- a/src/instrumentation/include/instrumentation/fuzz_target.hpp +++ b/src/instrumentation/include/instrumentation/fuzz_target.hpp @@ -30,7 +30,12 @@ class fuzz_target { fuzz_target(); - void process_condition(location_id id, bool direction, branching_function_value_type value, bool xor_like_branching_function); + void process_condition( + location_id::id_type id_type, + bool direction, + branching_function_value_type value, + bool xor_like_branching_function + ); void process_br_instr(location_id id, bool covered_branch); void process_call_begin(natural_32_bit const id); diff --git a/src/instrumentation/src/fuzz_target.cpp b/src/instrumentation/src/fuzz_target.cpp index b70f18b5..d703141a 100644 --- a/src/instrumentation/src/fuzz_target.cpp +++ b/src/instrumentation/src/fuzz_target.cpp @@ -22,7 +22,13 @@ fuzz_target::fuzz_target(): } -void fuzz_target::process_condition(location_id id, bool direction, branching_function_value_type value, bool xor_like_branching_function) { +void fuzz_target::process_condition( + location_id::id_type const id_type, + bool const direction, + branching_function_value_type value, + bool const xor_like_branching_function + ) +{ if (stdin_model->num_bytes_read() == 0) return; @@ -39,8 +45,8 @@ void fuzz_target::process_condition(location_id id, bool direction, branching_fu if (std::isnan(value)) { value = std::numeric_limits::max(); } - - id.context_hash = context_hashes.back(); + + location_id const id{ id_type, context_hashes.back() }; natural_32_bit idx_to_br_instr = br_instr_trace_length; shared_memory << data_record_id::condition << id << direction << value << idx_to_br_instr << xor_like_branching_function; ++trace_length; From 0925a722a4885aaa40f1886a9ec05f68e8bf1eed Mon Sep 17 00:00:00 2001 From: Marek Trtik Date: Mon, 16 Sep 2024 18:17:52 +0200 Subject: [PATCH 012/144] Added missing default program option (data) --- src/utility/src/program_options_base.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/utility/src/program_options_base.cpp b/src/utility/src/program_options_base.cpp index 77c1fa9a..f4c316c4 100644 --- a/src/utility/src/program_options_base.cpp +++ b/src/utility/src/program_options_base.cpp @@ -227,6 +227,4 @@ program_options_default::program_options_default(int argc, char* argv[]) { add_option("help", "Produces this help message.", "0"); add_option("version", "Prints the version string.", "0"); - add_option("data", "A root directory under which program's data are stored.", "1"); - add_value("data", "../data"); } From bff2095f6144410b2224fb453ad2a95769b43f39 Mon Sep 17 00:00:00 2001 From: Marek Trtik Date: Mon, 16 Sep 2024 18:18:56 +0200 Subject: [PATCH 013/144] Sensitivity: fixed bug - do not make sensitive bits which were not read yet. --- src/fuzzing/src/sensitivity_analysis.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fuzzing/src/sensitivity_analysis.cpp b/src/fuzzing/src/sensitivity_analysis.cpp index b1ae24df..f6fc8a20 100644 --- a/src/fuzzing/src/sensitivity_analysis.cpp +++ b/src/fuzzing/src/sensitivity_analysis.cpp @@ -313,9 +313,9 @@ void sensitivity_analysis::process_execution_results(execution_trace_pointer co INVARIANT(info_orig.id == info_curr.id && info_orig.id == n->id); if (info_orig.value != info_curr.value) - for (stdin_bit_index i = probed_bit_start_index; i != probed_bit_end_index; ++i) + for (stdin_bit_index j = probed_bit_start_index, j_end = std::min(probed_bit_end_index, n->get_num_stdin_bits()); j < j_end; ++j) { - auto const it_and_state = n->sensitive_stdin_bits.insert(i); + auto const it_and_state = n->sensitive_stdin_bits.insert(j); if (it_and_state.second) changed_nodes.insert(n); } From 1b61266851e33e0f72ff9a7965c84b16bb4b8613 Mon Sep 17 00:00:00 2001 From: Marek Trtik Date: Mon, 16 Sep 2024 18:20:53 +0200 Subject: [PATCH 014/144] Improved output from fizzer --- benchmarks/benman.py | 165 ++++++++---------- benchmarks/fast/array-1.json | 18 +- benchmarks/fast/array1_pattern.json | 23 ++- benchmarks/fast/big_issue.json | 18 +- benchmarks/fast/bool_flag_one_and_two.json | 23 ++- benchmarks/fast/c_string_count_chars.json | 30 +++- benchmarks/fast/c_string_parse_two_ints.json | 30 +++- benchmarks/fast/call_bool_arg.json | 20 ++- benchmarks/fast/ex2-alloca.json | 23 ++- benchmarks/fast/float_if_deg_to_rad.json | 20 ++- .../fast/float_if_deg_to_rad_inline.json | 20 ++- benchmarks/fast/float_if_parabola.json | 18 +- benchmarks/fast/float_if_parabola2.json | 18 +- benchmarks/fast/float_if_x_eq_c.json | 18 +- benchmarks/fast/float_if_x_eq_cos_x.json | 18 +- benchmarks/fast/float_if_x_lt_c.json | 20 ++- benchmarks/fast/float_if_xy_level_ring.json | 23 ++- benchmarks/fast/infinite01.json | 28 ++- benchmarks/fast/int16_equal.json | 23 ++- benchmarks/fast/int16_if_parabola.json | 18 +- benchmarks/fast/int16_if_x_equal_c.json | 18 +- benchmarks/fast/int16_if_x_equal_y_c.json | 18 +- benchmarks/fast/int16_if_x_ge_c.json | 20 ++- benchmarks/fast/int16_if_x_lt_c.json | 20 ++- benchmarks/fast/int16_if_x_xor_a_eq_b.json | 18 +- benchmarks/fast/int16_less.json | 23 ++- benchmarks/fast/int32_ackermann.json | 23 ++- benchmarks/fast/int32_if_parabola.json | 18 +- benchmarks/fast/int32_if_x_equal_c.json | 18 +- benchmarks/fast/int32_if_x_equal_y_c.json | 18 +- benchmarks/fast/int32_if_x_ge_c.json | 20 ++- benchmarks/fast/int32_if_x_lt_c.json | 20 ++- benchmarks/fast/int32_if_x_xor_a_eq_b.json | 18 +- .../fast/int32_logical_or_two_vars.json | 23 ++- benchmarks/fast/int8_if_hash_x_y_z_eq_c.json | 13 +- benchmarks/fast/int8_if_x_equal_c.json | 20 ++- benchmarks/fast/int8_if_x_equal_y_c.json | 18 +- benchmarks/fast/int8_if_x_ge_c.json | 20 ++- benchmarks/fast/int8_if_x_lt_c.json | 20 ++- benchmarks/fast/int8_if_x_xor_a_eq_b.json | 20 ++- benchmarks/fast/log_and.json | 25 ++- benchmarks/fast/log_cond.json | 23 ++- benchmarks/fast/machine32bit.json | 15 +- benchmarks/fast/matrix-2.json | 18 +- benchmarks/fast/mul_two_int16.json | 18 +- benchmarks/fast/nested_ifs.json | 18 +- benchmarks/fast/std_redef_malloc_free.json | 23 ++- benchmarks/fast/switch.json | 23 ++- benchmarks/fast/uint16_if_parabola.json | 20 ++- benchmarks/fast/uint32_if_parabola.json | 22 ++- .../include/fuzzing/analysis_outcomes.hpp | 16 +- .../include/fuzzing/execution_record.hpp | 3 +- src/fuzzing/include/fuzzing/fuzzer.hpp | 12 +- src/fuzzing/include/fuzzing/optimizer.hpp | 6 +- src/fuzzing/src/dump.cpp | 57 +++--- src/fuzzing/src/dump_native.cpp | 3 + src/fuzzing/src/fuzzer.cpp | 85 +++++---- src/fuzzing/src/fuzzing_loop.cpp | 17 +- src/fuzzing/src/optimizer.cpp | 4 +- src/tools/runner/sbt-fizzer.py | 118 +++++++------ src/tools/server/run.cpp | 56 ++++-- 61 files changed, 1078 insertions(+), 464 deletions(-) mode change 100644 => 100755 src/tools/runner/sbt-fizzer.py diff --git a/benchmarks/benman.py b/benchmarks/benman.py index 3da6a36b..93cb87a9 100755 --- a/benchmarks/benman.py +++ b/benchmarks/benman.py @@ -106,99 +106,69 @@ def _execute_and_check_output(self, cmdline : str, desired_output : str, work_di self._execute(cmdline, os.path.dirname(desired_output) if work_dir is None else work_dir) ASSUMPTION(os.path.isfile(desired_output), "_execute_and_check_output(): the output is missing: " + desired_output) - def _check_outcomes(self, config : dict, outcomes : dict): - checked_properties_and_comparators = { - "termination_type": "EQ", - "termination_reason": "EQ", - "num_executions": "LE", - "num_covered_branchings": "GE", - "covered_branchings": None, - "num_generated_tests": "GE", - "num_crashes": "GE", - "num_boundary_violations": "LE" - } - - def is_valid(obtained, expected, op : str) -> bool: - if op == "EQ": return obtained == expected - if op == "NE": return obtained != expected - if op == "LT": return obtained < expected - if op == "LE": return obtained <= expected - if op == "GT": return obtained > expected - if op == "GE": return obtained >= expected - raise Exception("Invalid comparison operator '" + op + "'.") - - for property, expected_value in config["results"].items(): - ASSUMPTION( - property in checked_properties_and_comparators, - "Unsupported key '" + property + "' in the 'results' section of benchmark's config JSON file." - ) - ASSUMPTION( - property in outcomes, - "The valid key '" + property + "' was not found in the 'outcomes' JSON file." - ) - if type(expected_value) in [int, float, str]: - if not is_valid(outcomes[property], expected_value, checked_properties_and_comparators[property]): + @staticmethod + def _add_error_message(text: str, errors: list, properties: list): + errors.append(("In " + "/".join(properties) + ": " if len(properties) > 0 else "") + text) + + @staticmethod + def _epsilon_for_property(properties): + if len(properties) == 0: return None + if properties[-1] == "num_executions": return 5.0 + return None + + @staticmethod + def _check_outcomes(obtained, expected, errors: list, properties = []) -> bool: + if type(expected) is dict: + if type(obtained) is not dict: + Benchmark._add_error_message("Mismatch in JSON structure. Expected dictionary.", errors, properties) + return False + result = True + for key in expected: + if key not in obtained: + Benchmark._add_error_message("Missing property: " + key, errors, properties) + return False + r = Benchmark._check_outcomes(obtained[key], expected[key], errors, properties + [key]) + result = result and r + return result + elif type(expected) is list: + if type(obtained) is not list: + Benchmark._add_error_message("Mismatch in JSON structure. Expected list.", errors, properties) + return False + if len(obtained) != len(expected): + Benchmark._add_error_message("Different list size.", errors, properties) + return False + result = True + for i in range(min(len(obtained), len(expected))): + r = Benchmark._check_outcomes(obtained[i], expected[i], errors, properties) + result = result and r + return result + elif type(expected) in [int, float]: + if type(obtained) not in [int, float]: + Benchmark._add_error_message("Mismatch in JSON structure. Expected int or float.", errors, properties) + return False + epsilon = Benchmark._epsilon_for_property(properties) + if epsilon is None: + if obtained != expected: + Benchmark._add_error_message("Expected " + str(expected) + ", obtained " + str(obtained), errors, properties) return False else: - ASSUMPTION(property == "covered_branchings", "Only 'covered_branchings' can be a 'list' property to check.") - ASSUMPTION(len(expected_value) % 2 == 0, "Expected covered branchings list must have even number of elements.") - ASSUMPTION(len(outcomes[property]) % 2 == 0, "Obtained covered branchings list must have even number of elements.") - def get_branchings(seq : list) -> set: - result = set() - if len(seq) > 0: - for i in range(0, len(seq)-1, 2): - result.add((seq[i], seq[i+1])) - return result - expected_branchings = get_branchings(expected_value) - obtained_branchings = get_branchings(outcomes[property]) - for x in expected_branchings: - if x not in obtained_branchings: - return False - return True - - def _ok_stats_message(self, config : dict, outcomes : dict) -> str: - max_num_execution = config["results"]["num_executions"] - try: - num_execution = outcomes["num_executions"] - except Exception as e: - return "Unknown executions count" - percentage = 100.0 * num_execution / max_num_execution - return "#" + ("%.2f" % (percentage - 100)) + "%" if max_num_execution >= 100 and percentage < 90 else "" - - def _fail_stats_message(self, config : dict, outcomes : dict) -> str: - expected_termination_type = config["results"]["termination_type"] - try: - termination_type = outcomes["termination_type"] - except Exception as e: - return "Unknown termination type" - if termination_type != expected_termination_type: - return termination_type - - result = "" - - expected_termination_reason = config["results"]["termination_reason"] - try: - termination_reason = outcomes["termination_reason"] - except Exception as e: - return "Unknown termination reason" - if termination_reason != expected_termination_reason: - result += termination_reason - - max_num_execution = config["results"]["num_executions"] - try: - num_execution = outcomes["num_executions"] - except Exception as e: - return result + ("" if len(result) == 0 else ", ") + "unknown executions count" - if len(result) == 0 and num_execution > max_num_execution: - percentage = 100.0 * num_execution / max_num_execution - result += ("" if len(result) == 0 else ", ") + "#" + ("+%.2f" % (percentage - 100)) + "%" - - return result - - def _embrace_stats_message(self, msg : str) -> str: - if len(msg) == 0: - return msg - return "[" + msg + "]" + percentage = (100.0 * obtained) / expected if expected > 0 else 100.0 * obtained + 100.0 + error = percentage - 100.0 + if abs(error) > epsilon: + Benchmark._add_error_message("Expected " + str(expected) + ", obtained " + str(obtained) + " [error: " + ("%.2f" % error) + "%]", errors, properties) + return False + return True + elif type(expected) is str: + if type(obtained) is not str: + Benchmark._add_error_message("Mismatch in JSON structure. Expected string.", errors, properties) + return False + if obtained != expected: + Benchmark._add_error_message("Expected " + expected + ", obtained " + obtained, errors, properties) + return False + return True + else: + Benchmark._add_error_message("Unexpected JSON content [type: " + str(type(expected)) + "].", errors, properties) + return False def build(self, benchmarks_root_dir : str, output_root_dir : str) -> None: self.log("===") @@ -216,7 +186,7 @@ def build(self, benchmarks_root_dir : str, output_root_dir : str) -> None: "--skip_fuzzing", "--input_file", self.src_file, "--output_dir", self.work_dir, - "--silent_build", + "--silent_mode", "--save_mapping" ] + (["--m32"] if "m32" in self.config["args"] and self.config["args"]["m32"] is True else []), self.fuzz_target_file, @@ -263,19 +233,20 @@ def fuzz(self, benchmarks_root_dir : str, output_root_dir : str) -> bool: output_dir ) + errors = [] try: outcomes_pathname = os.path.join(output_dir, self.name + "_outcomes.json") with open(outcomes_pathname, "rb") as fp: outcomes = json.load(fp) - if self._check_outcomes(self.config, outcomes) is True: - stats_msg = self._embrace_stats_message(self._ok_stats_message(self.config, outcomes)) - self.log("The outcomes are as expected => the test has PASSED. [Details: " + stats_msg + "]", "ok " + stats_msg + "\n") + if self._check_outcomes(outcomes, self.config["results"], errors) is True: + ASSUMPTION(len(errors) == 0) + self.log("The outcomes are as expected => the test has PASSED.", "ok\n") return True except Exception as e: self.log("FAILURE due to an EXCEPTION: " + str(e), "EXCEPTION[" + str(e) + "]\n") return False - stats_msg = self._embrace_stats_message(self._fail_stats_message(self.config, outcomes)) - self.log("The outcomes are NOT as expected => the test has FAILED. Details: " + stats_msg, "FAILED " + stats_msg + "\n") + error_messages = "\n " + "\n ".join(errors) + self.log("The outcomes are NOT as expected => the test has FAILED. Details:" + error_messages, "FAILED " + error_messages + "\n") return False def clear(self, benchmarks_root_dir : str, output_root_dir : str) -> None: diff --git a/benchmarks/fast/array-1.json b/benchmarks/fast/array-1.json index 1a633e84..bf6cd552 100644 --- a/benchmarks/fast/array-1.json +++ b/benchmarks/fast/array-1.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,17 @@ "covered_branchings": [ 2,0, 3,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } diff --git a/benchmarks/fast/array1_pattern.json b/benchmarks/fast/array1_pattern.json index a0c5b2f3..2282793c 100644 --- a/benchmarks/fast/array1_pattern.json +++ b/benchmarks/fast/array1_pattern.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -22,8 +23,22 @@ 1,2654435899, 1,2654435962, 3,0, 4,0, 5,0, 6,0, 7,0, 8,0 ], - "num_generated_tests": 7, - "num_crashes": 0, - "num_boundary_violations": 2 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 2, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 1 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 1 + } + } } } diff --git a/benchmarks/fast/big_issue.json b/benchmarks/fast/big_issue.json index 4d72ff91..d934fdd7 100644 --- a/benchmarks/fast/big_issue.json +++ b/benchmarks/fast/big_issue.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,17 @@ "covered_branchings": [ 2,0, 4,0 ], - "num_generated_tests": 3, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } diff --git a/benchmarks/fast/bool_flag_one_and_two.json b/benchmarks/fast/bool_flag_one_and_two.json index d05b81ff..4c60ca7b 100644 --- a/benchmarks/fast/bool_flag_one_and_two.json +++ b/benchmarks/fast/bool_flag_one_and_two.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -22,8 +23,22 @@ 1,0, 2,0, 3,0, 4,0, 5,0, 6,0 ], - "num_generated_tests": 4, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/c_string_count_chars.json b/benchmarks/fast/c_string_count_chars.json index 8497ded3..a0e6cfa2 100644 --- a/benchmarks/fast/c_string_count_chars.json +++ b/benchmarks/fast/c_string_count_chars.json @@ -11,19 +11,39 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 2230, + "num_executions": 2425, "num_covered_branchings": 7, "covered_branchings": [ 1,0, 2,0, 3,0, 4,0, 5,0, 6,0, 7,0 ], - "num_generated_tests": 6, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/c_string_parse_two_ints.json b/benchmarks/fast/c_string_parse_two_ints.json index f7a5e48d..2e1686cc 100644 --- a/benchmarks/fast/c_string_parse_two_ints.json +++ b/benchmarks/fast/c_string_parse_two_ints.json @@ -11,12 +11,13 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 3790, + "num_executions": 4043, "num_covered_branchings": 35, "covered_branchings": [ 1,0, 2,0, 3,0, 4,0, @@ -29,8 +30,27 @@ 18,1013904498, 19,1013904435, 19,1013904498, 20,1013904435, 20,1013904498, 21,1013904435, 21,1013904498 ], - "num_generated_tests": 30, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 10, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 15, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/call_bool_arg.json b/benchmarks/fast/call_bool_arg.json index 383178d7..713fa3ac 100644 --- a/benchmarks/fast/call_bool_arg.json +++ b/benchmarks/fast/call_bool_arg.json @@ -11,18 +11,28 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 35, + "num_executions": 26, "num_covered_branchings": 2, "covered_branchings": [ 1,2654435832, 2,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 1, + "num_boundary_violations": 0 + } + } } } diff --git a/benchmarks/fast/ex2-alloca.json b/benchmarks/fast/ex2-alloca.json index 195511d6..3a2ff83e 100644 --- a/benchmarks/fast/ex2-alloca.json +++ b/benchmarks/fast/ex2-alloca.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -22,8 +23,22 @@ 1,2654435832, 2,2654435832, 3,2654435832, 4,2654435832, 5,2654435832, 6,2654435832 ], - "num_generated_tests": 4, - "num_crashes": 0, - "num_boundary_violations": 1 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 1 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } diff --git a/benchmarks/fast/float_if_deg_to_rad.json b/benchmarks/fast/float_if_deg_to_rad.json index 23bca22a..7808fd2a 100644 --- a/benchmarks/fast/float_if_deg_to_rad.json +++ b/benchmarks/fast/float_if_deg_to_rad.json @@ -11,18 +11,28 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 55, + "num_executions": 52, "num_covered_branchings": 2, "covered_branchings": [ 1,0, 2,0 ], - "num_generated_tests": 3, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/float_if_deg_to_rad_inline.json b/benchmarks/fast/float_if_deg_to_rad_inline.json index 6e602d9b..a2bb4043 100644 --- a/benchmarks/fast/float_if_deg_to_rad_inline.json +++ b/benchmarks/fast/float_if_deg_to_rad_inline.json @@ -11,18 +11,28 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 30, + "num_executions": 26, "num_covered_branchings": 2, "covered_branchings": [ 1,0, 2,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } } + } } \ No newline at end of file diff --git a/benchmarks/fast/float_if_parabola.json b/benchmarks/fast/float_if_parabola.json index bfdf91ae..ababed4f 100644 --- a/benchmarks/fast/float_if_parabola.json +++ b/benchmarks/fast/float_if_parabola.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,17 @@ "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/float_if_parabola2.json b/benchmarks/fast/float_if_parabola2.json index e6dca3c1..e4b1d699 100644 --- a/benchmarks/fast/float_if_parabola2.json +++ b/benchmarks/fast/float_if_parabola2.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,17 @@ "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/float_if_x_eq_c.json b/benchmarks/fast/float_if_x_eq_c.json index 50fec118..53f4d331 100644 --- a/benchmarks/fast/float_if_x_eq_c.json +++ b/benchmarks/fast/float_if_x_eq_c.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,17 @@ "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } } + } } \ No newline at end of file diff --git a/benchmarks/fast/float_if_x_eq_cos_x.json b/benchmarks/fast/float_if_x_eq_cos_x.json index 36676479..48dcd21d 100644 --- a/benchmarks/fast/float_if_x_eq_cos_x.json +++ b/benchmarks/fast/float_if_x_eq_cos_x.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,17 @@ "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/float_if_x_lt_c.json b/benchmarks/fast/float_if_x_lt_c.json index 5f88c5e7..aacd5222 100644 --- a/benchmarks/fast/float_if_x_lt_c.json +++ b/benchmarks/fast/float_if_x_lt_c.json @@ -11,18 +11,28 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 30, + "num_executions": 26, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/float_if_xy_level_ring.json b/benchmarks/fast/float_if_xy_level_ring.json index 4581623f..f936f140 100644 --- a/benchmarks/fast/float_if_xy_level_ring.json +++ b/benchmarks/fast/float_if_xy_level_ring.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,22 @@ "covered_branchings": [ 1,0, 2,0 ], - "num_generated_tests": 3, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/infinite01.json b/benchmarks/fast/infinite01.json index cff0a4e1..92bf7db6 100644 --- a/benchmarks/fast/infinite01.json +++ b/benchmarks/fast/infinite01.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,27 @@ "covered_branchings": [ 1,0, 2,0, 4,0, 7,0 ], - "num_generated_tests": 3, - "num_crashes": 0, - "num_boundary_violations": 5 + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 0, + "num_crashes": 0, + "num_boundary_violations": 1 + }, + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 1 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 1 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 1 + } + } } } diff --git a/benchmarks/fast/int16_equal.json b/benchmarks/fast/int16_equal.json index 5bb41420..dd899997 100644 --- a/benchmarks/fast/int16_equal.json +++ b/benchmarks/fast/int16_equal.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,22 @@ "covered_branchings": [ 1,0, 2,0, 3,0, 4,0 ], - "num_generated_tests": 5, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int16_if_parabola.json b/benchmarks/fast/int16_if_parabola.json index 368fc6fe..f123596f 100644 --- a/benchmarks/fast/int16_if_parabola.json +++ b/benchmarks/fast/int16_if_parabola.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,17 @@ "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int16_if_x_equal_c.json b/benchmarks/fast/int16_if_x_equal_c.json index ce2633b4..32674703 100644 --- a/benchmarks/fast/int16_if_x_equal_c.json +++ b/benchmarks/fast/int16_if_x_equal_c.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,17 @@ "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } } + } } \ No newline at end of file diff --git a/benchmarks/fast/int16_if_x_equal_y_c.json b/benchmarks/fast/int16_if_x_equal_y_c.json index 10e59ccc..d644c882 100644 --- a/benchmarks/fast/int16_if_x_equal_y_c.json +++ b/benchmarks/fast/int16_if_x_equal_y_c.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,17 @@ "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int16_if_x_ge_c.json b/benchmarks/fast/int16_if_x_ge_c.json index ffb36aee..2a8c34f8 100644 --- a/benchmarks/fast/int16_if_x_ge_c.json +++ b/benchmarks/fast/int16_if_x_ge_c.json @@ -11,18 +11,28 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 20, + "num_executions": 10, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int16_if_x_lt_c.json b/benchmarks/fast/int16_if_x_lt_c.json index a97fe99c..638f877b 100644 --- a/benchmarks/fast/int16_if_x_lt_c.json +++ b/benchmarks/fast/int16_if_x_lt_c.json @@ -11,18 +11,28 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 20, + "num_executions": 10, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int16_if_x_xor_a_eq_b.json b/benchmarks/fast/int16_if_x_xor_a_eq_b.json index 08e034e4..15c9453a 100644 --- a/benchmarks/fast/int16_if_x_xor_a_eq_b.json +++ b/benchmarks/fast/int16_if_x_xor_a_eq_b.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,17 @@ "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int16_less.json b/benchmarks/fast/int16_less.json index 770a8e9b..737a609e 100644 --- a/benchmarks/fast/int16_less.json +++ b/benchmarks/fast/int16_less.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,22 @@ "covered_branchings": [ 1,0, 2,0, 3,0, 4,0 ], - "num_generated_tests": 5, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int32_ackermann.json b/benchmarks/fast/int32_ackermann.json index ae2f9a25..99cfd9e6 100644 --- a/benchmarks/fast/int32_ackermann.json +++ b/benchmarks/fast/int32_ackermann.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -27,8 +28,22 @@ 8,2027809127, 8,2654435832, 8,3668340393, 8,3668340399, 8,3668340520, 8,3668340524 ], - "num_generated_tests": 12, - "num_crashes": 0, - "num_boundary_violations": 2 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 7, + "num_crashes": 0, + "num_boundary_violations": 1 + }, + "typed_minimization_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 1 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int32_if_parabola.json b/benchmarks/fast/int32_if_parabola.json index 98c4a671..dcbc0a78 100644 --- a/benchmarks/fast/int32_if_parabola.json +++ b/benchmarks/fast/int32_if_parabola.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,17 @@ "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int32_if_x_equal_c.json b/benchmarks/fast/int32_if_x_equal_c.json index de5498da..fbcc9ce6 100644 --- a/benchmarks/fast/int32_if_x_equal_c.json +++ b/benchmarks/fast/int32_if_x_equal_c.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,17 @@ "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int32_if_x_equal_y_c.json b/benchmarks/fast/int32_if_x_equal_y_c.json index 43697fa4..c3bd194f 100644 --- a/benchmarks/fast/int32_if_x_equal_y_c.json +++ b/benchmarks/fast/int32_if_x_equal_y_c.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,17 @@ "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int32_if_x_ge_c.json b/benchmarks/fast/int32_if_x_ge_c.json index 9487834a..d030245b 100644 --- a/benchmarks/fast/int32_if_x_ge_c.json +++ b/benchmarks/fast/int32_if_x_ge_c.json @@ -11,18 +11,28 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 50, + "num_executions": 26, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int32_if_x_lt_c.json b/benchmarks/fast/int32_if_x_lt_c.json index 9487834a..d030245b 100644 --- a/benchmarks/fast/int32_if_x_lt_c.json +++ b/benchmarks/fast/int32_if_x_lt_c.json @@ -11,18 +11,28 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 50, + "num_executions": 26, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int32_if_x_xor_a_eq_b.json b/benchmarks/fast/int32_if_x_xor_a_eq_b.json index f20148fe..d61793dd 100644 --- a/benchmarks/fast/int32_if_x_xor_a_eq_b.json +++ b/benchmarks/fast/int32_if_x_xor_a_eq_b.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,17 @@ "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int32_logical_or_two_vars.json b/benchmarks/fast/int32_logical_or_two_vars.json index d2f1c2f4..2e34ef0f 100644 --- a/benchmarks/fast/int32_logical_or_two_vars.json +++ b/benchmarks/fast/int32_logical_or_two_vars.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,22 @@ "covered_branchings": [ 1,0, 2,0, 3,0, 4,0 ], - "num_generated_tests": 5, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int8_if_hash_x_y_z_eq_c.json b/benchmarks/fast/int8_if_hash_x_y_z_eq_c.json index 8f4a36de..c79f0dcb 100644 --- a/benchmarks/fast/int8_if_hash_x_y_z_eq_c.json +++ b/benchmarks/fast/int8_if_hash_x_y_z_eq_c.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -20,8 +21,12 @@ "num_covered_branchings": 0, "covered_branchings": [ ], - "num_generated_tests": 1, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int8_if_x_equal_c.json b/benchmarks/fast/int8_if_x_equal_c.json index a97fe99c..688e4c67 100644 --- a/benchmarks/fast/int8_if_x_equal_c.json +++ b/benchmarks/fast/int8_if_x_equal_c.json @@ -11,18 +11,28 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 20, + "num_executions": 17, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int8_if_x_equal_y_c.json b/benchmarks/fast/int8_if_x_equal_y_c.json index e8ae8f20..d8c52796 100644 --- a/benchmarks/fast/int8_if_x_equal_y_c.json +++ b/benchmarks/fast/int8_if_x_equal_y_c.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,17 @@ "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int8_if_x_ge_c.json b/benchmarks/fast/int8_if_x_ge_c.json index a97fe99c..c7c75c33 100644 --- a/benchmarks/fast/int8_if_x_ge_c.json +++ b/benchmarks/fast/int8_if_x_ge_c.json @@ -11,18 +11,28 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 20, + "num_executions": 11, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int8_if_x_lt_c.json b/benchmarks/fast/int8_if_x_lt_c.json index a97fe99c..c7c75c33 100644 --- a/benchmarks/fast/int8_if_x_lt_c.json +++ b/benchmarks/fast/int8_if_x_lt_c.json @@ -11,18 +11,28 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 20, + "num_executions": 11, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/int8_if_x_xor_a_eq_b.json b/benchmarks/fast/int8_if_x_xor_a_eq_b.json index 249c0126..c54aad1b 100644 --- a/benchmarks/fast/int8_if_x_xor_a_eq_b.json +++ b/benchmarks/fast/int8_if_x_xor_a_eq_b.json @@ -11,18 +11,28 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 30, + "num_executions": 24, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/log_and.json b/benchmarks/fast/log_and.json index fd8fdb82..7d08374d 100644 --- a/benchmarks/fast/log_and.json +++ b/benchmarks/fast/log_and.json @@ -11,18 +11,33 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 40, + "num_executions": 37, "num_covered_branchings": 3, "covered_branchings": [ 1,0, 2,0, 3,0 ], - "num_generated_tests": 3, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } } + } } diff --git a/benchmarks/fast/log_cond.json b/benchmarks/fast/log_cond.json index 1efa132f..641df079 100644 --- a/benchmarks/fast/log_cond.json +++ b/benchmarks/fast/log_cond.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,22 @@ "covered_branchings": [ 1,0, 2,0, 3,0, 4,0 ], - "num_generated_tests": 4, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } diff --git a/benchmarks/fast/machine32bit.json b/benchmarks/fast/machine32bit.json index d91181ed..e48abc88 100644 --- a/benchmarks/fast/machine32bit.json +++ b/benchmarks/fast/machine32bit.json @@ -22,8 +22,17 @@ "covered_branchings": [ 4,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } diff --git a/benchmarks/fast/matrix-2.json b/benchmarks/fast/matrix-2.json index 462e2e2a..173fd678 100644 --- a/benchmarks/fast/matrix-2.json +++ b/benchmarks/fast/matrix-2.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,17 @@ "covered_branchings": [ 2,0, 3,0, 4,0, 5,0 ], - "num_generated_tests": 5, - "num_crashes": 1, - "num_boundary_violations": 4 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 1, + "num_boundary_violations": 4 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } diff --git a/benchmarks/fast/mul_two_int16.json b/benchmarks/fast/mul_two_int16.json index bf49c77b..8b327eff 100644 --- a/benchmarks/fast/mul_two_int16.json +++ b/benchmarks/fast/mul_two_int16.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,17 @@ "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/nested_ifs.json b/benchmarks/fast/nested_ifs.json index ab082396..7205bac7 100644 --- a/benchmarks/fast/nested_ifs.json +++ b/benchmarks/fast/nested_ifs.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,17 @@ "covered_branchings": [ 1,0, 2,0, 3,0, 4,0 ], - "num_generated_tests": 5, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/std_redef_malloc_free.json b/benchmarks/fast/std_redef_malloc_free.json index c8769626..9eaec281 100644 --- a/benchmarks/fast/std_redef_malloc_free.json +++ b/benchmarks/fast/std_redef_malloc_free.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -21,8 +22,22 @@ "covered_branchings": [ 1,0, 2,0 ], - "num_generated_tests": 3, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/switch.json b/benchmarks/fast/switch.json index 3169bb2b..d6c10bfd 100644 --- a/benchmarks/fast/switch.json +++ b/benchmarks/fast/switch.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", @@ -22,8 +23,22 @@ 1,0, 2,0, 3,0, 4,0, 5,0 ], - "num_generated_tests": 6, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/fast/uint16_if_parabola.json b/benchmarks/fast/uint16_if_parabola.json index dd5df545..a1cecbc0 100644 --- a/benchmarks/fast/uint16_if_parabola.json +++ b/benchmarks/fast/uint16_if_parabola.json @@ -11,18 +11,28 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 30, + "num_executions": 24, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } } + } } \ No newline at end of file diff --git a/benchmarks/fast/uint32_if_parabola.json b/benchmarks/fast/uint32_if_parabola.json index fa0b9389..4244dde5 100644 --- a/benchmarks/fast/uint32_if_parabola.json +++ b/benchmarks/fast/uint32_if_parabola.json @@ -11,18 +11,28 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 45, + "num_executions": 40, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 - } + "output_statistics": { + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } } \ No newline at end of file diff --git a/src/fuzzing/include/fuzzing/analysis_outcomes.hpp b/src/fuzzing/include/fuzzing/analysis_outcomes.hpp index 96e96e2a..cb946b08 100644 --- a/src/fuzzing/include/fuzzing/analysis_outcomes.hpp +++ b/src/fuzzing/include/fuzzing/analysis_outcomes.hpp @@ -21,23 +21,29 @@ struct analysis_outcomes CLIENT_COMMUNICATION_ERROR, UNCLASSIFIED_ERROR }; + + struct output_statistics + { + natural_32_bit num_generated_tests{ 0U }; + natural_32_bit num_crashes{ 0U }; + natural_32_bit num_boundary_violations{ 0U }; + }; + TERMINATION_TYPE termination_type{ TERMINATION_TYPE::NORMAL }; fuzzer::TERMINATION_REASON termination_reason{ // Valid only if 'termination_type == NORMAL'. fuzzer::TERMINATION_REASON::ALL_REACHABLE_BRANCHINGS_COVERED }; std::string error_message{}; // Valid only if 'termination_type != NORMAL'. natural_32_bit num_executions{ 0U }; - long num_elapsed_seconds{ 0 }; + float_64_bit num_elapsed_seconds{ 0.0 }; std::vector covered_branchings{}; std::vector uncovered_branchings{}; sensitivity_analysis::performance_statistics sensitivity_statistics{}; typed_minimization_analysis::performance_statistics typed_minimization_statistics{}; minimization_analysis::performance_statistics minimization_statistics{}; bitshare_analysis::performance_statistics bitshare_statistics{}; - fuzzer::performance_statistics statistics{}; - natural_32_bit num_generated_tests{ 0U }; - natural_32_bit num_crashes{ 0U }; - natural_32_bit num_boundary_violations{ 0U }; + fuzzer::performance_statistics fuzzer_statistics{}; + std::unordered_map output_statistics{}; }; diff --git a/src/fuzzing/include/fuzzing/execution_record.hpp b/src/fuzzing/include/fuzzing/execution_record.hpp index c663f555..aaad9290 100644 --- a/src/fuzzing/include/fuzzing/execution_record.hpp +++ b/src/fuzzing/include/fuzzing/execution_record.hpp @@ -19,7 +19,8 @@ struct execution_record static execution_flags constexpr MEDIUM_OVERFLOW = 1 << 4; static execution_flags constexpr EMPTY_STARTUP_TRACE = 1 << 5; - execution_flags flags { 0 }; + execution_flags flags { 0 }; + std::string analysis_name {}; vecu8 stdin_bytes {}; input_types_vector stdin_types {}; execution_path path {}; diff --git a/src/fuzzing/include/fuzzing/fuzzer.hpp b/src/fuzzing/include/fuzzing/fuzzer.hpp index 9916389b..c4aa9247 100644 --- a/src/fuzzing/include/fuzzing/fuzzer.hpp +++ b/src/fuzzing/include/fuzzing/fuzzer.hpp @@ -62,11 +62,11 @@ struct fuzzer final termination_info const& get_termination_info() const { return termination_props; } - long num_remaining_driver_executions() const { return (long)termination_props.max_executions - (long)num_driver_executions; } - long num_remaining_seconds() const { return (long)termination_props.max_seconds - get_elapsed_seconds(); } + natural_32_bit num_remaining_driver_executions() const { return termination_props.max_executions - get_performed_driver_executions(); } + float_64_bit num_remaining_seconds() const { return (float_64_bit)termination_props.max_seconds - get_elapsed_seconds(); } natural_32_bit get_performed_driver_executions() const { return num_driver_executions; } - long get_elapsed_seconds() const { return (long)std::chrono::duration_cast(time_point_current - time_point_start).count(); } + float_64_bit get_elapsed_seconds() const { return std::chrono::duration(time_point_current - time_point_start).count(); } std::unordered_set const& get_covered_branchings() const { return covered_branchings; } std::unordered_set const& get_uncovered_branchings() const { return uncovered_branchings; } @@ -74,7 +74,7 @@ struct fuzzer final bool can_make_progress() const { return state != FINISHED; } bool round_begin(TERMINATION_REASON& termination_reason); - execution_record::execution_flags round_end(); + std::pair round_end(); sensitivity_analysis::performance_statistics const& get_sensitivity_statistics() const { return sensitivity.get_statistics(); } typed_minimization_analysis::performance_statistics const& get_typed_minimization_statistics() const { return typed_minimization.get_statistics(); } @@ -216,6 +216,8 @@ struct fuzzer final mutable random_generator_for_natural_32_bit generator_for_pivot_selection; }; + static std::string const& get_analysis_name_from_state(STATE state); + static void update_close_flags_from(branching_node* node); static std::vector const& get_input_width_classes(); @@ -283,7 +285,7 @@ struct fuzzer final probability_generator_random_uniform& location_miss_generator ); - void generate_next_input(vecb& stdin_bits); + bool generate_next_input(vecb& stdin_bits, TERMINATION_REASON& termination_reason); execution_record::execution_flags process_execution_results(); void do_cleanup(); diff --git a/src/fuzzing/include/fuzzing/optimizer.hpp b/src/fuzzing/include/fuzzing/optimizer.hpp index 0d8fcc1b..2da95559 100644 --- a/src/fuzzing/include/fuzzing/optimizer.hpp +++ b/src/fuzzing/include/fuzzing/optimizer.hpp @@ -36,14 +36,14 @@ struct optimizer final struct performance_statistics { natural_32_bit num_executions{ 0 }; - natural_32_bit num_seconds{ 0 }; + float_64_bit num_seconds{ 0.0 }; natural_32_bit num_extended_tests{ 0 }; }; optimizer(configuration const& cfg); - natural_32_bit num_remaining_seconds() const { return (natural_32_bit)config.max_seconds - get_elapsed_seconds(); } - natural_32_bit get_elapsed_seconds() const { return (natural_32_bit)std::chrono::duration_cast(time_point_current - time_point_start).count(); } + float_64_bit num_remaining_seconds() const { return (float_64_bit)config.max_seconds - get_elapsed_seconds(); } + float_64_bit get_elapsed_seconds() const { return std::chrono::duration(time_point_current - time_point_start).count(); } optimization_outcomes run( std::vector const& inputs_leading_to_boundary_violation, diff --git a/src/fuzzing/src/dump.cpp b/src/fuzzing/src/dump.cpp index 3694ce67..ad1f6ecb 100644 --- a/src/fuzzing/src/dump.cpp +++ b/src/fuzzing/src/dump.cpp @@ -33,7 +33,7 @@ void print_fuzzing_configuration( << shift << "\"max_exec_megabytes\": " << ioconfig.max_exec_megabytes << ",\n" << shift << "\"stdin_model\": \"" << ioconfig.stdin_model_name << "\",\n" << shift << "\"stdout_model\": \"" << ioconfig.stdout_model_name << "\"\n" - << "}\n" + << "}" ; } @@ -113,9 +113,9 @@ void print_analysis_outcomes(std::ostream& ostr, analysis_outcomes const& res ostr << shift << "\"error_message\": \"" << results.error_message << "\",\n"; std::vector warnings; - if (results.statistics.leaf_nodes_created != results.statistics.leaf_nodes_destroyed) + if (results.fuzzer_statistics.leaf_nodes_created != results.fuzzer_statistics.leaf_nodes_destroyed) warnings.push_back("The number of created and destroyed leaf nodes differ."); - if (results.statistics.nodes_created != results.statistics.nodes_destroyed) + if (results.fuzzer_statistics.nodes_created != results.fuzzer_statistics.nodes_destroyed) warnings.push_back("The number of created and destroyed nodes differ => Memory leak!"); if (results.sensitivity_statistics.start_calls != results.sensitivity_statistics.stop_calls_regular + results.sensitivity_statistics.stop_calls_early) warnings.push_back("The number of starts does not match to the number of stops in the sensitivity analysis."); @@ -177,22 +177,22 @@ void print_analysis_outcomes(std::ostream& ostr, analysis_outcomes const& res << shift << shift << "\"num_deletions\": " << results.bitshare_statistics.num_deletions << "\n" << shift << "},\n" << shift << "\"fuzzer\": {\n" - << shift << shift << "\"leaf_nodes_created\": " << results.statistics.leaf_nodes_created << ",\n" - << shift << shift << "\"leaf_nodes_destroyed\": " << results.statistics.leaf_nodes_destroyed << ",\n" - << shift << shift << "\"nodes_created\": " << results.statistics.nodes_created << ",\n" - << shift << shift << "\"nodes_destroyed\": " << results.statistics.nodes_destroyed << ",\n" - << shift << shift << "\"max_leaf_nodes\": " << results.statistics.max_leaf_nodes << ",\n" - << shift << shift << "\"max_input_width\": " << results.statistics.max_input_width << ",\n" - << shift << shift << "\"longest_branch\": " << results.statistics.longest_branch << ",\n" - << shift << shift << "\"traces_to_crash\": " << results.statistics.traces_to_crash << ",\n" - << shift << shift << "\"traces_to_boundary_violation\": " << results.statistics.traces_to_boundary_violation << ",\n" - << shift << shift << "\"traces_to_medium_overflow\": " << results.statistics.traces_to_medium_overflow << ",\n" - << shift << shift << "\"strategy_primary_loop_head\": " << results.statistics.strategy_primary_loop_head << ",\n" - << shift << shift << "\"strategy_primary_sensitive\": " << results.statistics.strategy_primary_sensitive << ",\n" - << shift << shift << "\"strategy_primary_untouched\": " << results.statistics.strategy_primary_untouched << ",\n" - << shift << shift << "\"strategy_primary_iid_twins\": " << results.statistics.strategy_primary_iid_twins << ",\n" - << shift << shift << "\"strategy_monte_carlo\": " << results.statistics.strategy_monte_carlo << ",\n" - << shift << shift << "\"coverage_failure_resets\": " << results.statistics.coverage_failure_resets << "\n" + << shift << shift << "\"leaf_nodes_created\": " << results.fuzzer_statistics.leaf_nodes_created << ",\n" + << shift << shift << "\"leaf_nodes_destroyed\": " << results.fuzzer_statistics.leaf_nodes_destroyed << ",\n" + << shift << shift << "\"nodes_created\": " << results.fuzzer_statistics.nodes_created << ",\n" + << shift << shift << "\"nodes_destroyed\": " << results.fuzzer_statistics.nodes_destroyed << ",\n" + << shift << shift << "\"max_leaf_nodes\": " << results.fuzzer_statistics.max_leaf_nodes << ",\n" + << shift << shift << "\"max_input_width\": " << results.fuzzer_statistics.max_input_width << ",\n" + << shift << shift << "\"longest_branch\": " << results.fuzzer_statistics.longest_branch << ",\n" + << shift << shift << "\"traces_to_crash\": " << results.fuzzer_statistics.traces_to_crash << ",\n" + << shift << shift << "\"traces_to_boundary_violation\": " << results.fuzzer_statistics.traces_to_boundary_violation << ",\n" + << shift << shift << "\"traces_to_medium_overflow\": " << results.fuzzer_statistics.traces_to_medium_overflow << ",\n" + << shift << shift << "\"strategy_primary_loop_head\": " << results.fuzzer_statistics.strategy_primary_loop_head << ",\n" + << shift << shift << "\"strategy_primary_sensitive\": " << results.fuzzer_statistics.strategy_primary_sensitive << ",\n" + << shift << shift << "\"strategy_primary_untouched\": " << results.fuzzer_statistics.strategy_primary_untouched << ",\n" + << shift << shift << "\"strategy_primary_iid_twins\": " << results.fuzzer_statistics.strategy_primary_iid_twins << ",\n" + << shift << shift << "\"strategy_monte_carlo\": " << results.fuzzer_statistics.strategy_monte_carlo << ",\n" + << shift << shift << "\"coverage_failure_resets\": " << results.fuzzer_statistics.coverage_failure_resets << "\n" << shift << "},\n" ; @@ -221,11 +221,18 @@ void print_analysis_outcomes(std::ostream& ostr, analysis_outcomes const& res } ostr << '\n' << shift << "],\n"; - ostr << shift << "\"num_generated_tests\": " << results.num_generated_tests << ",\n"; - ostr << shift << "\"num_crashes\": " << results.num_crashes << ",\n"; - ostr << shift << "\"num_boundary_violations\": " << results.num_boundary_violations << "\n"; + ostr << shift << "\"output_statistics\": {\n"; + for (auto it = results.output_statistics.begin(); it != results.output_statistics.end(); ++it) + { + ostr << shift << shift << '\"' << it->first << "\": {\n"; + ostr << shift << shift << shift << "\"num_generated_tests\": " << it->second.num_generated_tests << ",\n"; + ostr << shift << shift << shift << "\"num_crashes\": " << it->second.num_crashes << ",\n"; + ostr << shift << shift << shift << "\"num_boundary_violations\": " << it->second.num_boundary_violations << "\n"; + ostr << shift << shift << '}' << (std::next(it) != results.output_statistics.end() ? "," : "") << '\n'; + } + ostr << shift << "}\n"; - ostr << "}\n"; + ostr << "}"; } @@ -256,7 +263,7 @@ void print_optimization_configuration(std::ostream& ostr, optimizer::configura << shift << "\"max_seconds\": " << config.max_seconds << ",\n" << shift << "\"max_trace_length\": " << config.max_trace_length << ",\n" << shift << "\"max_stdin_bytes\": " << config.max_stdin_bytes << "\n" - << "}\n" + << "}" ; } @@ -354,7 +361,7 @@ void print_optimization_outcomes(std::ostream& ostr, optimization_outcomes con } ostr << '\n' << shift << "]\n"; - ostr << "}\n"; + ostr << "}"; } diff --git a/src/fuzzing/src/dump_native.cpp b/src/fuzzing/src/dump_native.cpp index a129a6d7..49b1f362 100644 --- a/src/fuzzing/src/dump_native.cpp +++ b/src/fuzzing/src/dump_native.cpp @@ -29,6 +29,7 @@ void save_native_test(std::ostream& ostr, execution_record const& record) << shift << "\"boundary_violation\": " << ((record.flags & execution_record::BOUNDARY_CONDITION_VIOLATION) != 0) << ",\n" << shift << "\"medium_overflow\": " << ((record.flags & execution_record::MEDIUM_OVERFLOW) != 0) << ",\n" << shift << "\"empty_startup_trace\": " << ((record.flags & execution_record::EMPTY_STARTUP_TRACE) != 0) << ",\n" + << shift << "\"analysis_name\": \"" << record.analysis_name << "\",\n" ; ostr << shift << "\"num_bytes\": " << record.stdin_bytes.size() << ",\n" @@ -48,7 +49,9 @@ void save_native_test(std::ostream& ostr, execution_record const& record) { if (i % 8U == 0U) ostr << '\n' << shift << shift; ostr << std::dec << '"' << to_string(record.stdin_types.at(i)) << "\","; + ostr << '"'; save_value(ostr, record.stdin_types.at(i), &chunk_values.at(i)); + ostr << '"'; if (i + 1 < n) ostr << ',' << shift; } diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index 52364eaf..ccecc6a7 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -318,6 +318,20 @@ float_32_bit fuzzer::probability_generator_all_then_all::next() } +std::string const& fuzzer::get_analysis_name_from_state(STATE state) +{ + static std::unordered_map const map { + { STARTUP, "STARTUP" }, + { SENSITIVITY, "sensitivity_analysis" }, + { TYPED_MINIMIZATION, "typed_minimization_analysis" }, + { MINIMIZATION, "minimization_analysis" }, + { BITSHARE, "bitshare_analysis" }, + { FINISHED, "FINISHED" }, + }; + return map.at(state); +} + + void fuzzer::update_close_flags_from(branching_node* const node) { if (node->is_closed() || node->is_open_branching()) @@ -825,35 +839,12 @@ bool fuzzer::round_begin(TERMINATION_REASON& termination_reason) { TMPROF_BLOCK(); - if (get_performed_driver_executions() > 0U) - { - if (uncovered_branchings.empty()) - { - terminate(); - termination_reason = TERMINATION_REASON::ALL_REACHABLE_BRANCHINGS_COVERED; - return false; - } - } - - if (num_remaining_seconds() <= 0L) - { - terminate(); - termination_reason = TERMINATION_REASON::TIME_BUDGET_DEPLETED; - return false; - } - - if (num_remaining_driver_executions() <= 0L) - { - terminate(); - termination_reason = TERMINATION_REASON::EXECUTIONS_BUDGET_DEPLETED; - return false; - } - iomodels::iomanager::instance().get_stdin()->clear(); iomodels::iomanager::instance().get_stdout()->clear(); vecb stdin_bits; - generate_next_input(stdin_bits); + if (!generate_next_input(stdin_bits, termination_reason)) + return false; if (!can_make_progress()) { terminate(); @@ -870,55 +861,79 @@ bool fuzzer::round_begin(TERMINATION_REASON& termination_reason) } -execution_record::execution_flags fuzzer::round_end() +std::pair fuzzer::round_end() { TMPROF_BLOCK(); execution_record::execution_flags const flags = process_execution_results(); - time_point_current = std::chrono::steady_clock::now(); ++num_driver_executions; - return flags; + return { flags, get_analysis_name_from_state(state) }; } -void fuzzer::generate_next_input(vecb& stdin_bits) +bool fuzzer::generate_next_input(vecb& stdin_bits, TERMINATION_REASON& termination_reason) { TMPROF_BLOCK(); while (true) { + if (get_performed_driver_executions() > 0U) + { + if (uncovered_branchings.empty()) + { + terminate(); + termination_reason = TERMINATION_REASON::ALL_REACHABLE_BRANCHINGS_COVERED; + return false; + } + } + + time_point_current = std::chrono::steady_clock::now(); + if (num_remaining_seconds() <= 0.0) + { + terminate(); + termination_reason = TERMINATION_REASON::TIME_BUDGET_DEPLETED; + return false; + } + + if (num_remaining_driver_executions() <= 0U) + { + terminate(); + termination_reason = TERMINATION_REASON::EXECUTIONS_BUDGET_DEPLETED; + return false; + } + switch (state) { case STARTUP: if (get_performed_driver_executions() == 0U) - return; + return true; break; case SENSITIVITY: if (sensitivity.generate_next_input(stdin_bits)) - return; + return true; break; case TYPED_MINIMIZATION: if (typed_minimization.generate_next_input(stdin_bits)) - return; + return true; break; case MINIMIZATION: if (minimization.generate_next_input(stdin_bits)) - return; + return true; break; case BITSHARE: if (bitshare.generate_next_input(stdin_bits)) - return; + return true; break; case FINISHED: if (!apply_coverage_failures_with_hope()) - return; + return true; break; default: { UNREACHABLE(); break; } diff --git a/src/fuzzing/src/fuzzing_loop.cpp b/src/fuzzing/src/fuzzing_loop.cpp index e0c0b91c..2459c902 100644 --- a/src/fuzzing/src/fuzzing_loop.cpp +++ b/src/fuzzing/src/fuzzing_loop.cpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace fuzzing { @@ -55,7 +56,7 @@ analysis_outcomes run( } execution_record record; - record.flags = f.round_end(); + std::tie(record.flags, record.analysis_name) = f.round_end(); if ((record.flags & (execution_record::BRANCH_DISCOVERED | execution_record::BRANCH_COVERED | @@ -63,18 +64,18 @@ analysis_outcomes run( { local::fill_record(record); save_execution_record(record); - ++results.num_generated_tests; + ++results.output_statistics[record.analysis_name].num_generated_tests; if ((record.flags & execution_record::EXECUTION_CRASHES) != 0) { hashes_of_crashes.insert(compute_hash(record.path)); - ++results.num_crashes; + ++results.output_statistics[record.analysis_name].num_crashes; } else if ((record.flags & execution_record::BOUNDARY_CONDITION_VIOLATION) != 0) { exit_locations_of_boundary_violations.insert(record.path.back().first.id); collector_of_boundary_violations(record); - ++results.num_boundary_violations; + ++results.output_statistics[record.analysis_name].num_boundary_violations; } } else if ((record.flags & execution_record::EXECUTION_CRASHES) != 0) @@ -83,8 +84,8 @@ analysis_outcomes run( if (hashes_of_crashes.insert(compute_hash(record.path)).second) { save_execution_record(record); - ++results.num_generated_tests; - ++results.num_crashes; + ++results.output_statistics[record.analysis_name].num_generated_tests; + ++results.output_statistics[record.analysis_name].num_crashes; } } else if ((record.flags & execution_record::BOUNDARY_CONDITION_VIOLATION) != 0) @@ -93,7 +94,7 @@ analysis_outcomes run( if (exit_locations_of_boundary_violations.insert(record.path.back().first.id).second) { collector_of_boundary_violations(record); - ++results.num_boundary_violations; + ++results.output_statistics[record.analysis_name].num_boundary_violations; } } } @@ -124,7 +125,7 @@ analysis_outcomes run( results.typed_minimization_statistics = f.get_typed_minimization_statistics(); results.minimization_statistics = f.get_minimization_statistics(); results.bitshare_statistics = f.get_bitshare_statistics(); - results.statistics = f.get_fuzzer_statistics(); + results.fuzzer_statistics = f.get_fuzzer_statistics(); return results; } diff --git a/src/fuzzing/src/optimizer.cpp b/src/fuzzing/src/optimizer.cpp index 4c4a5d83..20469f7f 100644 --- a/src/fuzzing/src/optimizer.cpp +++ b/src/fuzzing/src/optimizer.cpp @@ -165,11 +165,11 @@ optimization_outcomes optimizer::run( std::sort(outcomes.uncovered_branchings.begin(),outcomes.uncovered_branchings.end()); } - outcomes.statistics = statistics; - time_point_current = std::chrono::steady_clock::now(); statistics.num_seconds = get_elapsed_seconds(); + outcomes.statistics = statistics; + return outcomes; } diff --git a/src/tools/runner/sbt-fizzer.py b/src/tools/runner/sbt-fizzer.py old mode 100644 new mode 100755 index d8a6f628..27a80b7e --- a/src/tools/runner/sbt-fizzer.py +++ b/src/tools/runner/sbt-fizzer.py @@ -33,22 +33,23 @@ def benchmark_target_name(input_file): return benchmark_name(input_file) + "_sbt-fizzer_target" -def build(self_dir, input_file, output_dir, options, use_m32, silent_build): +def build(self_dir, input_file, output_dir, options, use_m32, silent_mode): ll_file = os.path.join(output_dir, benchmark_ll_name(input_file)) - if silent_build is False: print("Compiling...", end='', flush=True) + if silent_mode is False: print("\"build_times\": {", flush=True) + if silent_mode is False: print(" \"Compiling[C->LLVM]\": ", end='', flush=True) t0 = time.time() if _execute( [ "clang" ] + (["-m32"] if use_m32 is True else []) + [ "-O0", "-g", "-S", "-emit-llvm", "-Wno-everything", "-fbracket-depth=1024", input_file, "-o", ll_file], None).returncode: - raise Exception("Compilation has failed: " + input_file) + raise Exception("Compilation[C->LLVM] has failed: " + input_file) t1 = time.time() - if silent_build is False: print("Done[%ds]" % int(round(t1 - t0)), flush=True) + if silent_mode is False: print("%.2f," % (t1 - t0), flush=True) instrumented_ll_file = os.path.join(output_dir, benchmark_instrumented_ll_name(input_file)) - if silent_build is False: print("Instrumenting...", end='', flush=True) + if silent_mode is False: print(" \"Instrumenting\": ", end='', flush=True) t0 = time.time() if _execute( [ os.path.join(self_dir, "tools", "@FIZZER_INSTRUMENTER_FILE@") ] + @@ -57,7 +58,7 @@ def build(self_dir, input_file, output_dir, options, use_m32, silent_build): None).returncode: raise Exception("Instrumentation has failed: " + ll_file) t1 = time.time() - if silent_build is False: print("Done[%ds]" % int(round(t1 - t0)), flush=True) + if silent_mode is False: print("%.2f," % (t1 - t0), flush=True) fuzz_target_libraries = list(map( # type: ignore lambda lib_name: os.path.join(self_dir, "lib32" if use_m32 is True else "lib", lib_name).replace("\\", "/"), @@ -65,7 +66,7 @@ def build(self_dir, input_file, output_dir, options, use_m32, silent_build): )) target_file = os.path.join(output_dir, benchmark_target_name(input_file)) - if silent_build is False: print("Linking...", end='', flush=True) + if silent_mode is False: print(" \"Linking\": ", end='', flush=True) t0 = time.time() if _execute( [ "clang++" ] + @@ -75,11 +76,51 @@ def build(self_dir, input_file, output_dir, options, use_m32, silent_build): fuzz_target_libraries + [ "-o", target_file ], None).returncode: - raise Exception("Compilation has failed: " + input_file) + raise Exception("Linking has failed: " + input_file) t1 = time.time() - if silent_build is False: print("Done[%ds]" % int(round(t1 - t0)), flush=True) + if silent_mode is False: print("%.2f," % (t1 - t0), flush=True) + if silent_mode is False: print("},", flush=True) +def adjust_timeouts(options, start_time, silent_mode): + time_taken = time.time() - start_time + if time_taken < 0.1: + return + + def find_option_value_and_index(option): + try: idx = options.index(option) + except Exception: return None, None + if idx >= len(options): + return None + idx += 1 + try: return int(options[idx]), idx + except: return None, None + + def reduce_option_value(name, value, idx, total_time, suffix=""): + if total_time > time_taken: + percentage = 1.0 - time_taken / total_time + else: + percentage = 0.0 + new_value = int(value * percentage) + if silent_mode is False: print(" \"" + name + "\": [ " + str(value) + ", " + str(new_value) + " ]" + suffix, flush=True) + options[idx] = str(new_value) + + if silent_mode is False: print("\"adjusting_timeouts\": {", flush=True) + if silent_mode is False: print(" \"time_already_taken\": %.2f," % time_taken, flush=True) + + fuzz_value, fuzz_idx = find_option_value_and_index("--max_seconds") + opt_value, opt_idx = find_option_value_and_index("--optimizer_max_seconds") + + if fuzz_value is not None and opt_value is not None: + reduce_option_value("--max_seconds", fuzz_value, fuzz_idx, fuzz_value + opt_value, ",") + reduce_option_value("--optimizer_max_seconds", opt_value, opt_idx, fuzz_value + opt_value) + elif fuzz_value is not None: + reduce_option_value("--max_seconds", fuzz_value, fuzz_idx, fuzz_value) + elif opt_value is not None: + reduce_option_value("--optimizer_max_seconds", opt_value, opt_idx, opt_value) + + if silent_mode is False: print("},", flush=True) + def generate_testcomp_metadata_xml(input_file, output_dir, use_m32): test_suite_dir = os.path.join(output_dir, "test-suite") os.makedirs(test_suite_dir, exist_ok=True) @@ -107,36 +148,6 @@ def fuzz(self_dir, input_file, output_dir, options, start_time, silent_mode): if not os.path.isfile(target): raise Exception("Cannot find the fuzzing target file: " + target) - time_taken = time.time() - start_time - if time_taken > 0.5: - def find_option_value_and_index(option): - try: idx = options.index(option) - except ...: return None, None - if idx >= len(options): - return None - idx += 1 - try: return int(options[idx]), idx - except: return None, None - - def reduce_option_value(name, value, idx, total_time): - if total_time > time_taken: - percentage = 1.0 - time_taken / total_time - else: - percentage = 0.0 - new_value = int(round(value * percentage)) - if silent_mode is False: print("Adjusting '" + name + "': " + str(value) + " -> " + str(new_value), flush=True) - options[idx] = str(new_value) - - fuzz_value, fuzz_idx = find_option_value_and_index("--max_seconds") - opt_value, opt_idx = find_option_value_and_index("--optimizer_max_seconds") - - if fuzz_value is not None and opt_value is not None: - reduce_option_value("--max_seconds", fuzz_value, fuzz_idx, fuzz_value + opt_value) - reduce_option_value("--optimizer_max_seconds", opt_value, opt_idx, fuzz_value + opt_value) - elif fuzz_value is not None: - reduce_option_value("--max_seconds", fuzz_value, fuzz_idx, fuzz_value) - elif opt_value is not None: - reduce_option_value("--optimizer_max_seconds", opt_value, opt_idx, opt_value) if _execute( [ os.path.join(self_dir, "tools", "@SERVER_FILE@"), @@ -160,8 +171,7 @@ def help(self_dir): print(" instead of shared memory. This option is introduced so that") print(" you do not have to use options 'path_to_target' and") print(" 'path_to_client' listed below.") - print("silent_build When specified, no messages about the building process") - print(" of the passed source C file will be printed.") + print("silent_mode When specified, no messages will be printed.") print("m32 When specified, the source C file will be compiled for") print(" 32-bit machine (cpu). Otherwise, 64-bit machine is assumed.") print("\nNext follows a listing of options of tools called from this script. When they are") @@ -192,7 +202,6 @@ def main(): skip_building = False skip_fuzzing = False silent_mode = False - silent_build = False copy_source_file = False generate_testcomp_metadata = False use_m32 = False @@ -209,7 +218,6 @@ def main(): return if arg == "--silent_mode": - silent_build = True silent_mode = True elif arg == "--progress_recording": copy_source_file = True @@ -232,8 +240,6 @@ def main(): skip_building = True elif arg == "--skip_fuzzing": skip_fuzzing = True - elif arg == "--silent_build": - silent_build = True elif arg in [ "--save_mapping", "--br_too" ]: options_instument.append(arg) elif arg == "--m32": @@ -242,9 +248,6 @@ def main(): options.append(arg) i += 1 - if input_file is None: - raise Exception("Cannot find the input file.") - if clear_output_dir is True and os.path.isdir(output_dir): shutil.rmtree(output_dir) if copy_source_file is True: @@ -254,19 +257,27 @@ def main(): old_cwd = os.getcwd() os.chdir(output_dir) try: + if input_file is None: + raise Exception("Cannot find the input file.") + if silent_mode is False: print("### starting fizzer's pipeline ###\n{", flush=True) if skip_building is False: - build(self_dir, input_file, output_dir, options_instument, use_m32, silent_build) + build(self_dir, input_file, output_dir, options_instument, use_m32, silent_mode) + adjust_timeouts(options, start_time, silent_mode) if skip_fuzzing is False: if generate_testcomp_metadata is True: generate_testcomp_metadata_xml(input_file, output_dir, use_m32) fuzz(self_dir, input_file, output_dir, options, start_time, silent_mode) + if silent_mode is False: print(",", flush=True) + if silent_mode is False: print("\"exit_code\": 0,", flush=True) except Exception as e: os.chdir(old_cwd) - print("Stopped[%ds]" % int(round(time.time() - start_time)), flush=True) + if silent_mode is False: print("\"error_message\": \"" + str(e) + "\"", flush=True) + if silent_mode is False: print("\"exit_code\": 1,", flush=True) raise e - - if silent_mode is False and ((skip_building is False and silent_build is False) or skip_fuzzing is False): - print("Done[%ds]" % int(round(time.time() - start_time)), flush=True) + finally: + if silent_mode is False: + print("\"total_time\": %.2f" % (time.time() - start_time), flush=True) + print("}", flush=True) if __name__ == "__main__": @@ -275,5 +286,4 @@ def main(): main() except Exception as e: exit_code = 1 - print("ERROR: " + str(e), flush=True) exit(exit_code) diff --git a/src/tools/server/run.cpp b/src/tools/server/run.cpp index e5b2ceca..6b66f596 100644 --- a/src/tools/server/run.cpp +++ b/src/tools/server/run.cpp @@ -18,6 +18,8 @@ void run(int argc, char* argv[]) { + std::chrono::system_clock::time_point const start_time_point = std::chrono::system_clock::now(); + if (get_program_options()->has("list_stdin_models")) { for (auto const& name_and_constructor : iomodels::get_stdin_models_map()) @@ -60,8 +62,17 @@ void run(int argc, char* argv[]) if (get_program_options()->has("clear_output_dir")) { for (const auto& entry : std::filesystem::directory_iterator(output_dir)) - if (entry.is_regular_file() && entry.path().extension() == ".json") - std::filesystem::remove(entry); + if (entry.is_regular_file()) + { + auto const name{ entry.path().filename().string() }; + for (auto const& suffix : { + "_config.json", "_outcomes.json", + "_LOG.html", "_TMPROF.html", + "0.json", "1.json", "2.json", "3.json", "4.json", "5.json", "6.json", "7.json", "8.json", "9.json", + }) + if (name.ends_with(suffix)) + std::filesystem::remove(entry); + } if (std::filesystem::is_directory(output_dir / "test-suite")) for (const auto& entry : std::filesystem::directory_iterator(output_dir / "test-suite")) std::filesystem::remove(entry); @@ -105,7 +116,7 @@ void run(int argc, char* argv[]) } } - fuzzing::termination_info const terminator{ + fuzzing::termination_info terminator{ .max_executions = (natural_32_bit)std::max(0, std::stoi(get_program_options()->value("max_executions"))), .max_seconds = (natural_32_bit)std::max(0, std::stoi(get_program_options()->value("max_seconds"))) }; @@ -121,7 +132,7 @@ void run(int argc, char* argv[]) .stdout_model_name = get_program_options()->value("stdout_model") }); - fuzzing::optimizer::configuration const optimizer_config{ + fuzzing::optimizer::configuration optimizer_config{ .max_seconds = (natural_32_bit)std::max(0, std::stoi(get_program_options()->value("optimizer_max_seconds"))), .max_trace_length = (natural_32_bit)std::max(0, std::stoi(get_program_options()->value("optimizer_max_trace_length"))), .max_stdin_bytes = (iomodels::stdin_base::byte_count_type)std::max(0, std::stoi(get_program_options()->value("optimizer_max_stdin_bytes"))) @@ -154,7 +165,7 @@ void run(int argc, char* argv[]) if (get_program_options()->has("path_to_client")) { if (!get_program_options()->has("silent_mode")) - std::cout << "Communication type: network" << std::endl; + std::cout << "\"communication_type\": \"network\"," << std::endl; benchmark_executor = std::make_shared( get_program_options()->value("path_to_client"), @@ -165,13 +176,29 @@ void run(int argc, char* argv[]) else { if (!get_program_options()->has("silent_mode")) - std::cout << "Communication type: shared memory" << std::endl; + std::cout << "\"communication_type\": \"shared_memory\"," << std::endl; benchmark_executor = std::make_shared( get_program_options()->value("path_to_target") ); } + auto const startup_time = std::chrono::duration(std::chrono::system_clock::now() - start_time_point).count(); + + { + float_64_bit const total_time{ std::max((float_64_bit)(terminator.max_seconds + optimizer_config.max_seconds), 1.0) }; + float_64_bit const remaining_time{ std::max(total_time - startup_time, 0.0) }; + terminator.max_seconds = (natural_32_bit)(remaining_time * (terminator.max_seconds / total_time)); + optimizer_config.max_seconds = (natural_32_bit)(remaining_time * (optimizer_config.max_seconds / total_time)); + + if (!get_program_options()->has("silent_mode")) + std::cout << "\"fuzzing_startup\": {" << std::endl + << " \"time\": " << startup_time << ',' << std::endl + << " \"--max_seconds\": " << terminator.max_seconds << ',' << std::endl + << " \"--optimizer_max_seconds\": " << optimizer_config.max_seconds << std::endl + << "}," << std::endl; + } + fuzzing::execution_record_writer execution_record_writer{ output_dir, target_name, @@ -181,13 +208,14 @@ void run(int argc, char* argv[]) if (!get_program_options()->has("silent_mode")) { - std::cout << "Configuration for fuzzing:" << std::endl; + std::cout << "\"fuzzing_configuration\": "; fuzzing::print_fuzzing_configuration( std::cout, target_name, iomodels::iomanager::instance().get_config(), terminator ); + std::cout << ',' << std::endl; } fuzzing::log_fuzzing_configuration( target_name, @@ -201,9 +229,6 @@ void run(int argc, char* argv[]) terminator ); - if (!get_program_options()->has("silent_mode")) - std::cout << "Fuzzing was started..." << std::endl; - std::vector inputs_leading_to_boundary_violation; fuzzing::analysis_outcomes const results = fuzzing::run( *benchmark_executor, @@ -217,7 +242,7 @@ void run(int argc, char* argv[]) if (!get_program_options()->has("silent_mode")) { - std::cout << "Fuzzing was stopped. Details:" << std::endl; + std::cout << "\"fuzzing_results\": "; fuzzing::print_analysis_outcomes(std::cout, results); } fuzzing::log_analysis_outcomes(results); @@ -229,15 +254,14 @@ void run(int argc, char* argv[]) { if (!get_program_options()->has("silent_mode")) { - std::cout << "Configuration for test suite optimization:" << std::endl; + std::cout << ',' << std::endl + << "\"optimization_configuration\": "; fuzzing::print_optimization_configuration(std::cout, optimizer_config); + std::cout << ',' << std::endl; } fuzzing::log_optimization_configuration(optimizer_config); fuzzing::save_optimization_configuration(output_dir, target_name, optimizer_config); - if (!get_program_options()->has("silent_mode")) - std::cout << "Optimization was started..." << std::endl; - fuzzing::optimizer opt{ optimizer_config }; { @@ -258,7 +282,7 @@ void run(int argc, char* argv[]) if (!get_program_options()->has("silent_mode")) { - std::cout << "Optimization was stopped. Details:" << std::endl; + std::cout << "\"optimization_results\": "; fuzzing::print_optimization_outcomes(std::cout, opt_results); } fuzzing::log_optimization_outcomes(opt_results); From aeef0f82dc920f63110278c6f726615ede5614b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Mon, 16 Sep 2024 22:29:54 +0200 Subject: [PATCH 015/144] feat: added interesting nodes set functionality --- src/fuzzing/include/fuzzing/fuzzer.hpp | 3 +- src/fuzzing/src/fuzzer.cpp | 51 ++++++++++++++++++++++++-- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/src/fuzzing/include/fuzzing/fuzzer.hpp b/src/fuzzing/include/fuzzing/fuzzer.hpp index 946fce5a..c6da5bd3 100644 --- a/src/fuzzing/include/fuzzing/fuzzer.hpp +++ b/src/fuzzing/include/fuzzing/fuzzer.hpp @@ -238,9 +238,10 @@ struct fuzzer final struct iid_dependence_props { std::vector all_paths; - std::set interestins_nodes; + std::set interesting_nodes; std::vector> matrix; + void update_interesting_nodes(branching_node* node); void recompute_matrix(); }; diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index 9425af74..0dd36f1c 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -972,19 +972,65 @@ void fuzzer::generate_next_input(vecb& stdin_bits) UNREACHABLE(); } + void fuzzer::process_node_dependance(branching_node *node) { using deps_props = iid_node_dependence::iid_dependence_props; - using node_direction = iid_node_dependence::node_navigation; if (iid_dependences.non_iid_nodes.contains(node->get_location_id())) return; deps_props& props = iid_dependences.id_to_equation_map[node->get_location_id()]; props.all_paths.push_back(node); + props.update_interesting_nodes(node); props.recompute_matrix(); } +void fuzzing::fuzzer::iid_node_dependence::iid_dependence_props::update_interesting_nodes(branching_node *node) +{ + using node_nav = iid_node_dependence::node_navigation; + + auto get_path = [](branching_node* node) { + std::vector path; + branching_node* current = node; + while (current != nullptr) { + path.push_back(current); + current = current->predecessor; + } + return path; + }; + + auto add_to_interesting = [this](branching_node* node, size_t i) { + while (i > 0) { + branching_node* predecessor = node->predecessor; + + this->interesting_nodes.emplace(node->get_location_id(), node == predecessor->successor(true).pointer); + + node = node->predecessor; + --i; + } + }; + + + for (const auto& path : all_paths) { + std::vector path_1 = get_path(node); + std::vector path_2 = get_path(path); + + ASSUMPTION(path_1.back() == path_2.back()); + + std::size_t i_1 = path_1.size() - 1; + std::size_t i_2 = path_2.size() - 1; + + while (i_1 > 0 && i_2 > 0 && path_1[i_1] == path_2[i_2]) { + --i_1; + --i_2; + } + + add_to_interesting(path_1[i_1], i_1); + add_to_interesting(path_2[i_2], i_2); + } +} + void fuzzing::fuzzer::iid_node_dependence::iid_dependence_props::recompute_matrix() { using node_nav = iid_node_dependence::node_navigation; @@ -1000,8 +1046,7 @@ void fuzzing::fuzzer::iid_node_dependence::iid_dependence_props::recompute_matri branching_node* prev_node = node->predecessor; while (prev_node != nullptr) { - bool direction = prev_node->successor(true).pointer == node; - node_nav nav = { prev_node->get_location_id(), direction }; + node_nav nav = { prev_node->get_location_id(), prev_node->successor(true).pointer == node }; directions_in_path[nav]++; node = prev_node; From 943287e3cbf22d53e36fdb4aad955d9982fb34f6 Mon Sep 17 00:00:00 2001 From: Marek Trtik Date: Tue, 17 Sep 2024 01:46:04 +0200 Subject: [PATCH 016/144] Benchmarks config hanged: stdin_replay_bytes_then_repeat_(85->zero) --- benchmarks/fast/array-1.json | 2 +- benchmarks/fast/array1_pattern.json | 15 +++++-------- benchmarks/fast/big_issue.json | 2 +- benchmarks/fast/bool_flag_one_and_two.json | 4 ++-- benchmarks/fast/c_string_count_chars.json | 8 +++---- benchmarks/fast/c_string_parse_two_ints.json | 10 ++++----- benchmarks/fast/call_bool_arg.json | 8 +++---- benchmarks/fast/ex2-alloca.json | 16 +++++--------- benchmarks/fast/float_if_deg_to_rad.json | 9 ++++++-- .../fast/float_if_deg_to_rad_inline.json | 4 ++-- benchmarks/fast/float_if_parabola.json | 6 +++--- benchmarks/fast/float_if_parabola2.json | 4 ++-- benchmarks/fast/float_if_x_eq_c.json | 4 ++-- benchmarks/fast/float_if_x_eq_cos_x.json | 2 +- benchmarks/fast/float_if_x_lt_c.json | 4 ++-- benchmarks/fast/float_if_xy_level_ring.json | 15 +++++-------- benchmarks/fast/infinite01.json | 21 ++++++------------- benchmarks/fast/int16_equal.json | 4 ++-- benchmarks/fast/int16_if_parabola.json | 6 +++--- benchmarks/fast/int16_if_x_equal_c.json | 2 +- benchmarks/fast/int16_if_x_equal_y_c.json | 4 ++-- benchmarks/fast/int16_if_x_ge_c.json | 4 ++-- benchmarks/fast/int16_if_x_lt_c.json | 4 ++-- benchmarks/fast/int16_if_x_xor_a_eq_b.json | 4 ++-- benchmarks/fast/int16_less.json | 8 +++---- benchmarks/fast/int32_ackermann.json | 9 ++------ benchmarks/fast/int32_if_parabola.json | 6 +++--- benchmarks/fast/int32_if_x_equal_c.json | 2 +- benchmarks/fast/int32_if_x_equal_y_c.json | 2 +- benchmarks/fast/int32_if_x_ge_c.json | 4 ++-- benchmarks/fast/int32_if_x_lt_c.json | 4 ++-- benchmarks/fast/int32_if_x_xor_a_eq_b.json | 4 ++-- .../fast/int32_logical_or_two_vars.json | 11 +++------- benchmarks/fast/int8_if_hash_x_y_z_eq_c.json | 4 ++-- benchmarks/fast/int8_if_x_equal_c.json | 2 +- benchmarks/fast/int8_if_x_equal_y_c.json | 2 +- benchmarks/fast/int8_if_x_ge_c.json | 2 +- benchmarks/fast/int8_if_x_lt_c.json | 2 +- benchmarks/fast/int8_if_x_xor_a_eq_b.json | 2 +- benchmarks/fast/log_and.json | 11 +++------- benchmarks/fast/log_cond.json | 11 +++------- benchmarks/fast/machine32bit.json | 2 +- benchmarks/fast/mul_two_int16.json | 14 ++++--------- benchmarks/fast/nested_ifs.json | 4 ++-- benchmarks/fast/std_redef_malloc_free.json | 11 +++------- benchmarks/fast/switch.json | 8 +++---- benchmarks/fast/uint16_if_parabola.json | 6 +++--- benchmarks/fast/uint32_if_parabola.json | 8 +++---- 48 files changed, 125 insertions(+), 176 deletions(-) diff --git a/benchmarks/fast/array-1.json b/benchmarks/fast/array-1.json index bf6cd552..b4a54506 100644 --- a/benchmarks/fast/array-1.json +++ b/benchmarks/fast/array-1.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, diff --git a/benchmarks/fast/array1_pattern.json b/benchmarks/fast/array1_pattern.json index 2282793c..ae2ab2c2 100644 --- a/benchmarks/fast/array1_pattern.json +++ b/benchmarks/fast/array1_pattern.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -26,18 +26,13 @@ "output_statistics": { "sensitivity_analysis": { "num_generated_tests": 4, - "num_crashes": 2, - "num_boundary_violations": 0 - }, - "typed_minimization_analysis": { - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 1 + "num_crashes": 1, + "num_boundary_violations": 2 }, "STARTUP": { "num_generated_tests": 1, - "num_crashes": 0, - "num_boundary_violations": 1 + "num_crashes": 1, + "num_boundary_violations": 0 } } } diff --git a/benchmarks/fast/big_issue.json b/benchmarks/fast/big_issue.json index d934fdd7..05f6fddb 100644 --- a/benchmarks/fast/big_issue.json +++ b/benchmarks/fast/big_issue.json @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "FUZZING_STRATEGY_DEPLETED", - "num_executions": 105, + "num_executions": 103, "num_covered_branchings": 2, "covered_branchings": [ 2,0, 4,0 diff --git a/benchmarks/fast/bool_flag_one_and_two.json b/benchmarks/fast/bool_flag_one_and_two.json index 4c60ca7b..1aabbfd3 100644 --- a/benchmarks/fast/bool_flag_one_and_two.json +++ b/benchmarks/fast/bool_flag_one_and_two.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 165, + "num_executions": 163, "num_covered_branchings": 6, "covered_branchings": [ 1,0, 2,0, 3,0, 4,0, diff --git a/benchmarks/fast/c_string_count_chars.json b/benchmarks/fast/c_string_count_chars.json index a0e6cfa2..9c777b89 100644 --- a/benchmarks/fast/c_string_count_chars.json +++ b/benchmarks/fast/c_string_count_chars.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 2425, + "num_executions": 3045, "num_covered_branchings": 7, "covered_branchings": [ 1,0, 2,0, 3,0, 4,0, @@ -30,12 +30,12 @@ "num_boundary_violations": 0 }, "sensitivity_analysis": { - "num_generated_tests": 1, + "num_generated_tests": 4, "num_crashes": 0, "num_boundary_violations": 0 }, "typed_minimization_analysis": { - "num_generated_tests": 3, + "num_generated_tests": 1, "num_crashes": 0, "num_boundary_violations": 0 }, diff --git a/benchmarks/fast/c_string_parse_two_ints.json b/benchmarks/fast/c_string_parse_two_ints.json index 2e1686cc..e29efd96 100644 --- a/benchmarks/fast/c_string_parse_two_ints.json +++ b/benchmarks/fast/c_string_parse_two_ints.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 4043, + "num_executions": 4785, "num_covered_branchings": 35, "covered_branchings": [ 1,0, 2,0, 3,0, 4,0, @@ -32,17 +32,17 @@ ], "output_statistics": { "bitshare_analysis": { - "num_generated_tests": 10, + "num_generated_tests": 7, "num_crashes": 0, "num_boundary_violations": 0 }, "sensitivity_analysis": { - "num_generated_tests": 4, + "num_generated_tests": 12, "num_crashes": 0, "num_boundary_violations": 0 }, "typed_minimization_analysis": { - "num_generated_tests": 15, + "num_generated_tests": 8, "num_crashes": 0, "num_boundary_violations": 0 }, diff --git a/benchmarks/fast/call_bool_arg.json b/benchmarks/fast/call_bool_arg.json index 713fa3ac..67a6b777 100644 --- a/benchmarks/fast/call_bool_arg.json +++ b/benchmarks/fast/call_bool_arg.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 26, + "num_executions": 2, "num_covered_branchings": 2, "covered_branchings": [ 1,2654435832, 2,0 @@ -25,12 +25,12 @@ "output_statistics": { "sensitivity_analysis": { "num_generated_tests": 1, - "num_crashes": 0, + "num_crashes": 1, "num_boundary_violations": 0 }, "STARTUP": { "num_generated_tests": 1, - "num_crashes": 1, + "num_crashes": 0, "num_boundary_violations": 0 } } diff --git a/benchmarks/fast/ex2-alloca.json b/benchmarks/fast/ex2-alloca.json index 3a2ff83e..3a44931e 100644 --- a/benchmarks/fast/ex2-alloca.json +++ b/benchmarks/fast/ex2-alloca.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -16,20 +16,14 @@ }, "results": { "termination_type": "NORMAL", - "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 420, - "num_covered_branchings": 6, + "termination_reason": "FUZZING_STRATEGY_DEPLETED", + "num_executions": 4940, + "num_covered_branchings": 2, "covered_branchings": [ - 1,2654435832, 2,2654435832, 3,2654435832, 4,2654435832, - 5,2654435832, 6,2654435832 + 1,2654435832, 2,2654435832 ], "output_statistics": { "sensitivity_analysis": { - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 1 - }, - "typed_minimization_analysis": { "num_generated_tests": 1, "num_crashes": 0, "num_boundary_violations": 0 diff --git a/benchmarks/fast/float_if_deg_to_rad.json b/benchmarks/fast/float_if_deg_to_rad.json index 7808fd2a..b1c6a207 100644 --- a/benchmarks/fast/float_if_deg_to_rad.json +++ b/benchmarks/fast/float_if_deg_to_rad.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 52, + "num_executions": 56, "num_covered_branchings": 2, "covered_branchings": [ 1,0, 2,0 @@ -28,6 +28,11 @@ "num_crashes": 0, "num_boundary_violations": 0 }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, "STARTUP": { "num_generated_tests": 1, "num_crashes": 0, diff --git a/benchmarks/fast/float_if_deg_to_rad_inline.json b/benchmarks/fast/float_if_deg_to_rad_inline.json index a2bb4043..7031d001 100644 --- a/benchmarks/fast/float_if_deg_to_rad_inline.json +++ b/benchmarks/fast/float_if_deg_to_rad_inline.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 26, + "num_executions": 40, "num_covered_branchings": 2, "covered_branchings": [ 1,0, 2,0 diff --git a/benchmarks/fast/float_if_parabola.json b/benchmarks/fast/float_if_parabola.json index ababed4f..8c63cb3a 100644 --- a/benchmarks/fast/float_if_parabola.json +++ b/benchmarks/fast/float_if_parabola.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,13 +17,13 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 115, + "num_executions": 27, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], "output_statistics": { - "typed_minimization_analysis": { + "sensitivity_analysis": { "num_generated_tests": 1, "num_crashes": 0, "num_boundary_violations": 0 diff --git a/benchmarks/fast/float_if_parabola2.json b/benchmarks/fast/float_if_parabola2.json index e4b1d699..a69ccf3d 100644 --- a/benchmarks/fast/float_if_parabola2.json +++ b/benchmarks/fast/float_if_parabola2.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 125, + "num_executions": 121, "num_covered_branchings": 1, "covered_branchings": [ 1,0 diff --git a/benchmarks/fast/float_if_x_eq_c.json b/benchmarks/fast/float_if_x_eq_c.json index 53f4d331..a5a6d382 100644 --- a/benchmarks/fast/float_if_x_eq_c.json +++ b/benchmarks/fast/float_if_x_eq_c.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 50, + "num_executions": 49, "num_covered_branchings": 1, "covered_branchings": [ 1,0 diff --git a/benchmarks/fast/float_if_x_eq_cos_x.json b/benchmarks/fast/float_if_x_eq_cos_x.json index 48dcd21d..ef220aea 100644 --- a/benchmarks/fast/float_if_x_eq_cos_x.json +++ b/benchmarks/fast/float_if_x_eq_cos_x.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, diff --git a/benchmarks/fast/float_if_x_lt_c.json b/benchmarks/fast/float_if_x_lt_c.json index aacd5222..fe9e2ad9 100644 --- a/benchmarks/fast/float_if_x_lt_c.json +++ b/benchmarks/fast/float_if_x_lt_c.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 26, + "num_executions": 34, "num_covered_branchings": 1, "covered_branchings": [ 1,0 diff --git a/benchmarks/fast/float_if_xy_level_ring.json b/benchmarks/fast/float_if_xy_level_ring.json index f936f140..b85c86c7 100644 --- a/benchmarks/fast/float_if_xy_level_ring.json +++ b/benchmarks/fast/float_if_xy_level_ring.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -16,11 +16,11 @@ }, "results": { "termination_type": "NORMAL", - "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 150, - "num_covered_branchings": 2, + "termination_reason": "FUZZING_STRATEGY_DEPLETED", + "num_executions": 169, + "num_covered_branchings": 1, "covered_branchings": [ - 1,0, 2,0 + 1,0 ], "output_statistics": { "sensitivity_analysis": { @@ -28,11 +28,6 @@ "num_crashes": 0, "num_boundary_violations": 0 }, - "typed_minimization_analysis": { - "num_generated_tests": 1, - "num_crashes": 0, - "num_boundary_violations": 0 - }, "STARTUP": { "num_generated_tests": 1, "num_crashes": 0, diff --git a/benchmarks/fast/infinite01.json b/benchmarks/fast/infinite01.json index 92bf7db6..feb3387c 100644 --- a/benchmarks/fast/infinite01.json +++ b/benchmarks/fast/infinite01.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -18,25 +18,16 @@ "termination_type": "NORMAL", "termination_reason": "EXECUTIONS_BUDGET_DEPLETED", "num_executions": 500, - "num_covered_branchings": 4, + "num_covered_branchings": 5, "covered_branchings": [ - 1,0, 2,0, 4,0, 7,0 + 1,0, 2,0, 3,0, 4,0, + 7,0 ], "output_statistics": { - "bitshare_analysis": { - "num_generated_tests": 0, - "num_crashes": 0, - "num_boundary_violations": 1 - }, "sensitivity_analysis": { - "num_generated_tests": 1, + "num_generated_tests": 3, "num_crashes": 0, - "num_boundary_violations": 1 - }, - "typed_minimization_analysis": { - "num_generated_tests": 1, - "num_crashes": 0, - "num_boundary_violations": 1 + "num_boundary_violations": 4 }, "STARTUP": { "num_generated_tests": 1, diff --git a/benchmarks/fast/int16_equal.json b/benchmarks/fast/int16_equal.json index dd899997..7fdea7d2 100644 --- a/benchmarks/fast/int16_equal.json +++ b/benchmarks/fast/int16_equal.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 95, + "num_executions": 92, "num_covered_branchings": 4, "covered_branchings": [ 1,0, 2,0, 3,0, 4,0 diff --git a/benchmarks/fast/int16_if_parabola.json b/benchmarks/fast/int16_if_parabola.json index f123596f..34b97396 100644 --- a/benchmarks/fast/int16_if_parabola.json +++ b/benchmarks/fast/int16_if_parabola.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,13 +17,13 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 60, + "num_executions": 7, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], "output_statistics": { - "typed_minimization_analysis": { + "sensitivity_analysis": { "num_generated_tests": 1, "num_crashes": 0, "num_boundary_violations": 0 diff --git a/benchmarks/fast/int16_if_x_equal_c.json b/benchmarks/fast/int16_if_x_equal_c.json index 32674703..8a006888 100644 --- a/benchmarks/fast/int16_if_x_equal_c.json +++ b/benchmarks/fast/int16_if_x_equal_c.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, diff --git a/benchmarks/fast/int16_if_x_equal_y_c.json b/benchmarks/fast/int16_if_x_equal_y_c.json index d644c882..6ecd1e2c 100644 --- a/benchmarks/fast/int16_if_x_equal_y_c.json +++ b/benchmarks/fast/int16_if_x_equal_y_c.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 45, + "num_executions": 44, "num_covered_branchings": 1, "covered_branchings": [ 1,0 diff --git a/benchmarks/fast/int16_if_x_ge_c.json b/benchmarks/fast/int16_if_x_ge_c.json index 2a8c34f8..41d77310 100644 --- a/benchmarks/fast/int16_if_x_ge_c.json +++ b/benchmarks/fast/int16_if_x_ge_c.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 10, + "num_executions": 11, "num_covered_branchings": 1, "covered_branchings": [ 1,0 diff --git a/benchmarks/fast/int16_if_x_lt_c.json b/benchmarks/fast/int16_if_x_lt_c.json index 638f877b..5326c55d 100644 --- a/benchmarks/fast/int16_if_x_lt_c.json +++ b/benchmarks/fast/int16_if_x_lt_c.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 10, + "num_executions": 11, "num_covered_branchings": 1, "covered_branchings": [ 1,0 diff --git a/benchmarks/fast/int16_if_x_xor_a_eq_b.json b/benchmarks/fast/int16_if_x_xor_a_eq_b.json index 15c9453a..dfd1c9a8 100644 --- a/benchmarks/fast/int16_if_x_xor_a_eq_b.json +++ b/benchmarks/fast/int16_if_x_xor_a_eq_b.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 100, + "num_executions": 98, "num_covered_branchings": 1, "covered_branchings": [ 1,0 diff --git a/benchmarks/fast/int16_less.json b/benchmarks/fast/int16_less.json index 737a609e..b32267a4 100644 --- a/benchmarks/fast/int16_less.json +++ b/benchmarks/fast/int16_less.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,19 +17,19 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 135, + "num_executions": 70, "num_covered_branchings": 4, "covered_branchings": [ 1,0, 2,0, 3,0, 4,0 ], "output_statistics": { "sensitivity_analysis": { - "num_generated_tests": 2, + "num_generated_tests": 3, "num_crashes": 0, "num_boundary_violations": 0 }, "typed_minimization_analysis": { - "num_generated_tests": 2, + "num_generated_tests": 1, "num_crashes": 0, "num_boundary_violations": 0 }, diff --git a/benchmarks/fast/int32_ackermann.json b/benchmarks/fast/int32_ackermann.json index 99cfd9e6..0bee70c9 100644 --- a/benchmarks/fast/int32_ackermann.json +++ b/benchmarks/fast/int32_ackermann.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -30,12 +30,7 @@ ], "output_statistics": { "sensitivity_analysis": { - "num_generated_tests": 7, - "num_crashes": 0, - "num_boundary_violations": 1 - }, - "typed_minimization_analysis": { - "num_generated_tests": 4, + "num_generated_tests": 11, "num_crashes": 0, "num_boundary_violations": 1 }, diff --git a/benchmarks/fast/int32_if_parabola.json b/benchmarks/fast/int32_if_parabola.json index dcbc0a78..34b97396 100644 --- a/benchmarks/fast/int32_if_parabola.json +++ b/benchmarks/fast/int32_if_parabola.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,13 +17,13 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 80, + "num_executions": 7, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], "output_statistics": { - "typed_minimization_analysis": { + "sensitivity_analysis": { "num_generated_tests": 1, "num_crashes": 0, "num_boundary_violations": 0 diff --git a/benchmarks/fast/int32_if_x_equal_c.json b/benchmarks/fast/int32_if_x_equal_c.json index fbcc9ce6..3df9ae0e 100644 --- a/benchmarks/fast/int32_if_x_equal_c.json +++ b/benchmarks/fast/int32_if_x_equal_c.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, diff --git a/benchmarks/fast/int32_if_x_equal_y_c.json b/benchmarks/fast/int32_if_x_equal_y_c.json index c3bd194f..ee9470f9 100644 --- a/benchmarks/fast/int32_if_x_equal_y_c.json +++ b/benchmarks/fast/int32_if_x_equal_y_c.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, diff --git a/benchmarks/fast/int32_if_x_ge_c.json b/benchmarks/fast/int32_if_x_ge_c.json index d030245b..97576d1d 100644 --- a/benchmarks/fast/int32_if_x_ge_c.json +++ b/benchmarks/fast/int32_if_x_ge_c.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 26, + "num_executions": 27, "num_covered_branchings": 1, "covered_branchings": [ 1,0 diff --git a/benchmarks/fast/int32_if_x_lt_c.json b/benchmarks/fast/int32_if_x_lt_c.json index d030245b..97576d1d 100644 --- a/benchmarks/fast/int32_if_x_lt_c.json +++ b/benchmarks/fast/int32_if_x_lt_c.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 26, + "num_executions": 27, "num_covered_branchings": 1, "covered_branchings": [ 1,0 diff --git a/benchmarks/fast/int32_if_x_xor_a_eq_b.json b/benchmarks/fast/int32_if_x_xor_a_eq_b.json index d61793dd..ae47b400 100644 --- a/benchmarks/fast/int32_if_x_xor_a_eq_b.json +++ b/benchmarks/fast/int32_if_x_xor_a_eq_b.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 1040, + "num_executions": 1038, "num_covered_branchings": 1, "covered_branchings": [ 1,0 diff --git a/benchmarks/fast/int32_logical_or_two_vars.json b/benchmarks/fast/int32_logical_or_two_vars.json index 2e34ef0f..ba0ed659 100644 --- a/benchmarks/fast/int32_logical_or_two_vars.json +++ b/benchmarks/fast/int32_logical_or_two_vars.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,19 +17,14 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 150, + "num_executions": 58, "num_covered_branchings": 4, "covered_branchings": [ 1,0, 2,0, 3,0, 4,0 ], "output_statistics": { "sensitivity_analysis": { - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 - }, - "typed_minimization_analysis": { - "num_generated_tests": 2, + "num_generated_tests": 4, "num_crashes": 0, "num_boundary_violations": 0 }, diff --git a/benchmarks/fast/int8_if_hash_x_y_z_eq_c.json b/benchmarks/fast/int8_if_hash_x_y_z_eq_c.json index c79f0dcb..ebdf2eac 100644 --- a/benchmarks/fast/int8_if_hash_x_y_z_eq_c.json +++ b/benchmarks/fast/int8_if_hash_x_y_z_eq_c.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "FUZZING_STRATEGY_DEPLETED", - "num_executions": 15000, + "num_executions": 14947, "num_covered_branchings": 0, "covered_branchings": [ ], diff --git a/benchmarks/fast/int8_if_x_equal_c.json b/benchmarks/fast/int8_if_x_equal_c.json index 688e4c67..94840880 100644 --- a/benchmarks/fast/int8_if_x_equal_c.json +++ b/benchmarks/fast/int8_if_x_equal_c.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, diff --git a/benchmarks/fast/int8_if_x_equal_y_c.json b/benchmarks/fast/int8_if_x_equal_y_c.json index d8c52796..715d8092 100644 --- a/benchmarks/fast/int8_if_x_equal_y_c.json +++ b/benchmarks/fast/int8_if_x_equal_y_c.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, diff --git a/benchmarks/fast/int8_if_x_ge_c.json b/benchmarks/fast/int8_if_x_ge_c.json index c7c75c33..5326c55d 100644 --- a/benchmarks/fast/int8_if_x_ge_c.json +++ b/benchmarks/fast/int8_if_x_ge_c.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, diff --git a/benchmarks/fast/int8_if_x_lt_c.json b/benchmarks/fast/int8_if_x_lt_c.json index c7c75c33..5326c55d 100644 --- a/benchmarks/fast/int8_if_x_lt_c.json +++ b/benchmarks/fast/int8_if_x_lt_c.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, diff --git a/benchmarks/fast/int8_if_x_xor_a_eq_b.json b/benchmarks/fast/int8_if_x_xor_a_eq_b.json index c54aad1b..2191a190 100644 --- a/benchmarks/fast/int8_if_x_xor_a_eq_b.json +++ b/benchmarks/fast/int8_if_x_xor_a_eq_b.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, diff --git a/benchmarks/fast/log_and.json b/benchmarks/fast/log_and.json index 7d08374d..fd3c1c77 100644 --- a/benchmarks/fast/log_and.json +++ b/benchmarks/fast/log_and.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,19 +17,14 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 37, + "num_executions": 53, "num_covered_branchings": 3, "covered_branchings": [ 1,0, 2,0, 3,0 ], "output_statistics": { - "sensitivity_analysis": { - "num_generated_tests": 1, - "num_crashes": 0, - "num_boundary_violations": 0 - }, "typed_minimization_analysis": { - "num_generated_tests": 1, + "num_generated_tests": 2, "num_crashes": 0, "num_boundary_violations": 0 }, diff --git a/benchmarks/fast/log_cond.json b/benchmarks/fast/log_cond.json index 641df079..e5a5c2a8 100644 --- a/benchmarks/fast/log_cond.json +++ b/benchmarks/fast/log_cond.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,19 +17,14 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 55, + "num_executions": 59, "num_covered_branchings": 4, "covered_branchings": [ 1,0, 2,0, 3,0, 4,0 ], "output_statistics": { - "sensitivity_analysis": { - "num_generated_tests": 1, - "num_crashes": 0, - "num_boundary_violations": 0 - }, "typed_minimization_analysis": { - "num_generated_tests": 2, + "num_generated_tests": 3, "num_crashes": 0, "num_boundary_violations": 0 }, diff --git a/benchmarks/fast/machine32bit.json b/benchmarks/fast/machine32bit.json index e48abc88..aa0e825e 100644 --- a/benchmarks/fast/machine32bit.json +++ b/benchmarks/fast/machine32bit.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, diff --git a/benchmarks/fast/mul_two_int16.json b/benchmarks/fast/mul_two_int16.json index 8b327eff..95b7b0bf 100644 --- a/benchmarks/fast/mul_two_int16.json +++ b/benchmarks/fast/mul_two_int16.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -16,18 +16,12 @@ }, "results": { "termination_type": "NORMAL", - "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 70, - "num_covered_branchings": 1, + "termination_reason": "FUZZING_STRATEGY_DEPLETED", + "num_executions": 37, + "num_covered_branchings": 0, "covered_branchings": [ - 1,0 ], "output_statistics": { - "typed_minimization_analysis": { - "num_generated_tests": 1, - "num_crashes": 0, - "num_boundary_violations": 0 - }, "STARTUP": { "num_generated_tests": 1, "num_crashes": 0, diff --git a/benchmarks/fast/nested_ifs.json b/benchmarks/fast/nested_ifs.json index 7205bac7..839bcd1b 100644 --- a/benchmarks/fast/nested_ifs.json +++ b/benchmarks/fast/nested_ifs.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 185, + "num_executions": 183, "num_covered_branchings": 4, "covered_branchings": [ 1,0, 2,0, 3,0, 4,0 diff --git a/benchmarks/fast/std_redef_malloc_free.json b/benchmarks/fast/std_redef_malloc_free.json index 9eaec281..45719e0c 100644 --- a/benchmarks/fast/std_redef_malloc_free.json +++ b/benchmarks/fast/std_redef_malloc_free.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,19 +17,14 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 40, + "num_executions": 10, "num_covered_branchings": 2, "covered_branchings": [ 1,0, 2,0 ], "output_statistics": { "sensitivity_analysis": { - "num_generated_tests": 1, - "num_crashes": 0, - "num_boundary_violations": 0 - }, - "typed_minimization_analysis": { - "num_generated_tests": 1, + "num_generated_tests": 2, "num_crashes": 0, "num_boundary_violations": 0 }, diff --git a/benchmarks/fast/switch.json b/benchmarks/fast/switch.json index d6c10bfd..f99800bb 100644 --- a/benchmarks/fast/switch.json +++ b/benchmarks/fast/switch.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 100, + "num_executions": 77, "num_covered_branchings": 5, "covered_branchings": [ 1,0, 2,0, 3,0, 4,0, @@ -25,12 +25,12 @@ ], "output_statistics": { "sensitivity_analysis": { - "num_generated_tests": 1, + "num_generated_tests": 4, "num_crashes": 0, "num_boundary_violations": 0 }, "typed_minimization_analysis": { - "num_generated_tests": 4, + "num_generated_tests": 1, "num_crashes": 0, "num_boundary_violations": 0 }, diff --git a/benchmarks/fast/uint16_if_parabola.json b/benchmarks/fast/uint16_if_parabola.json index a1cecbc0..34b97396 100644 --- a/benchmarks/fast/uint16_if_parabola.json +++ b/benchmarks/fast/uint16_if_parabola.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,13 +17,13 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 24, + "num_executions": 7, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], "output_statistics": { - "typed_minimization_analysis": { + "sensitivity_analysis": { "num_generated_tests": 1, "num_crashes": 0, "num_boundary_violations": 0 diff --git a/benchmarks/fast/uint32_if_parabola.json b/benchmarks/fast/uint32_if_parabola.json index 4244dde5..34b97396 100644 --- a/benchmarks/fast/uint32_if_parabola.json +++ b/benchmarks/fast/uint32_if_parabola.json @@ -7,7 +7,7 @@ "max_stdin_bytes": 6400, "max_exec_milliseconds": 250, "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_85", + "stdin_model": "stdin_replay_bytes_then_repeat_zero", "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, @@ -17,13 +17,13 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 40, + "num_executions": 7, "num_covered_branchings": 1, "covered_branchings": [ 1,0 ], "output_statistics": { - "typed_minimization_analysis": { + "sensitivity_analysis": { "num_generated_tests": 1, "num_crashes": 0, "num_boundary_violations": 0 @@ -33,6 +33,6 @@ "num_crashes": 0, "num_boundary_violations": 0 } - } + } } } \ No newline at end of file From 9e64c8982773ab6dc0492534801f999dddce684e Mon Sep 17 00:00:00 2001 From: Marek Trtik Date: Mon, 16 Sep 2024 12:33:52 +0200 Subject: [PATCH 017/144] Improved instrumentation --- .../include/fuzzing/branching_node.hpp | 6 +- src/fuzzing/src/fuzzer.cpp | 32 ++++- .../include/instrumentation/fuzz_target.hpp | 3 +- .../instrumentation/instrumentation_types.hpp | 15 ++ src/instrumentation/src/fuzz_target.cpp | 9 +- src/instrumentation/src/instrumentation.cpp | 5 +- .../src/instrumentation_types.cpp | 15 ++ src/iomodels/src/iomanager.cpp | 1 + src/tools/instrumenter/llvm_instrumenter.cpp | 128 ++++++++++++------ 9 files changed, 162 insertions(+), 52 deletions(-) diff --git a/src/fuzzing/include/fuzzing/branching_node.hpp b/src/fuzzing/include/fuzzing/branching_node.hpp index e1a2a2f7..2203bedc 100644 --- a/src/fuzzing/include/fuzzing/branching_node.hpp +++ b/src/fuzzing/include/fuzzing/branching_node.hpp @@ -43,7 +43,8 @@ struct branching_node branching_function_value_type best_coverage_value_, branching_function_value_type best_summary_value_, natural_32_bit execution_number, - bool xor_like_branching_function_ + bool xor_like_branching_function_, + BRANCHING_PREDICATE branching_predicate_ ) : id{ id_ } , trace_index{ trace_index_ } @@ -70,6 +71,7 @@ struct branching_node , sensitive_stdin_bits{} , xor_like_branching_function{ xor_like_branching_function_ } + , branching_predicate{ branching_predicate_ } , closed{ false } @@ -136,6 +138,8 @@ struct branching_node bool xor_like_branching_function; + BRANCHING_PREDICATE branching_predicate; + bool closed; trace_index_type max_successors_trace_index; diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index ccecc6a7..faac92dd 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -982,7 +982,8 @@ execution_record::execution_flags fuzzer::process_execution_results() std::numeric_limits::max(), std::numeric_limits::max(), num_driver_executions, - trace->front().xor_like_branching_function + trace->front().xor_like_branching_function, + trace->front().predicate ); construction_props.diverging_node = entry_branching; @@ -1023,6 +1024,32 @@ execution_record::execution_flags fuzzer::process_execution_results() } } + // Here we try to remove bad float (INF, NaN) from 'info.value'. + // It would be better, if fuzzer and analyses could deal with bad floats, but that is complicated. + if (!std::isfinite(info.value) || std::isnan(info.value)) + { + branching_function_value_type& value_ref{ const_cast(info.value) }; + switch (info.predicate) + { + case BP_EQUAL: + value_ref = info.direction ? 0.0 : std::numeric_limits::max(); + break; + case BP_UNEQUAL: + value_ref = info.direction ? std::numeric_limits::max() : 0.0; + break; + case BP_LESS_EQUAL: + case BP_LESS: + value_ref = (info.direction ? -1.0 : 1.0) * std::numeric_limits::max(); + break; + break; + case BP_GREATER: + case BP_GREATER_EQUAL: + value_ref = (info.direction ? 1.0 : -1.0) * std::numeric_limits::max(); + break; + default: UNREACHABLE(); break; + } + } + summary_value += info.value * info.value; bool const value_ok = std::isfinite(summary_value); if (construction_props.leaf->best_stdin == nullptr || (value_ok && construction_props.leaf->best_summary_value > summary_value)) @@ -1064,7 +1091,8 @@ execution_record::execution_flags fuzzer::process_execution_results() succ_info.value, succ_info.value * succ_info.value, num_driver_executions, - succ_info.xor_like_branching_function + succ_info.xor_like_branching_function, + succ_info.predicate ) }); diff --git a/src/instrumentation/include/instrumentation/fuzz_target.hpp b/src/instrumentation/include/instrumentation/fuzz_target.hpp index d519aa00..7e33babd 100644 --- a/src/instrumentation/include/instrumentation/fuzz_target.hpp +++ b/src/instrumentation/include/instrumentation/fuzz_target.hpp @@ -34,7 +34,8 @@ class fuzz_target { location_id::id_type id_type, bool direction, branching_function_value_type value, - bool xor_like_branching_function + bool xor_like_branching_function, + natural_8_bit predicate ); void process_br_instr(location_id id, bool covered_branch); diff --git a/src/instrumentation/include/instrumentation/instrumentation_types.hpp b/src/instrumentation/include/instrumentation/instrumentation_types.hpp index 3c29138e..3b0ff120 100644 --- a/src/instrumentation/include/instrumentation/instrumentation_types.hpp +++ b/src/instrumentation/include/instrumentation/instrumentation_types.hpp @@ -36,6 +36,20 @@ std::ostream& operator<<(std::ostream& ostr, location_id id); using branching_function_value_type = float_64_bit; + +enum BRANCHING_PREDICATE : natural_8_bit +{ + BP_EQUAL = 0, + BP_UNEQUAL = 1, + BP_LESS = 2, + BP_LESS_EQUAL = 3, + BP_GREATER = 4, + BP_GREATER_EQUAL = 5 +}; + +BRANCHING_PREDICATE opposite_predicate(BRANCHING_PREDICATE predicate); + + struct branching_coverage_info { explicit branching_coverage_info(location_id const id_); @@ -46,6 +60,7 @@ struct branching_coverage_info branching_function_value_type value; natural_32_bit idx_to_br_instr; bool xor_like_branching_function; + BRANCHING_PREDICATE predicate; natural_32_bit num_input_bytes; }; diff --git a/src/instrumentation/src/fuzz_target.cpp b/src/instrumentation/src/fuzz_target.cpp index d703141a..c6232c79 100644 --- a/src/instrumentation/src/fuzz_target.cpp +++ b/src/instrumentation/src/fuzz_target.cpp @@ -26,7 +26,8 @@ void fuzz_target::process_condition( location_id::id_type const id_type, bool const direction, branching_function_value_type value, - bool const xor_like_branching_function + bool const xor_like_branching_function, + natural_8_bit const predicate ) { if (stdin_model->num_bytes_read() == 0) @@ -42,13 +43,9 @@ void fuzz_target::process_condition( exit(0); } - if (std::isnan(value)) { - value = std::numeric_limits::max(); - } - location_id const id{ id_type, context_hashes.back() }; natural_32_bit idx_to_br_instr = br_instr_trace_length; - shared_memory << data_record_id::condition << id << direction << value << idx_to_br_instr << xor_like_branching_function; + shared_memory << data_record_id::condition << id << direction << value << idx_to_br_instr << xor_like_branching_function << predicate; ++trace_length; } diff --git a/src/instrumentation/src/instrumentation.cpp b/src/instrumentation/src/instrumentation.cpp index 87dfb06a..67626b91 100644 --- a/src/instrumentation/src/instrumentation.cpp +++ b/src/instrumentation/src/instrumentation.cpp @@ -12,9 +12,10 @@ void __sbt_fizzer_process_condition( location_id::id_type const id, bool const direction, branching_function_value_type const value, - bool const xor_like_branching_function + bool const xor_like_branching_function, + natural_8_bit const predicate ) { - sbt_fizzer_target->process_condition(id, direction, value, xor_like_branching_function); + sbt_fizzer_target->process_condition(id, direction, value, xor_like_branching_function, predicate); } void __sbt_fizzer_process_br_instr(location_id::id_type const id, bool const direction) { diff --git a/src/instrumentation/src/instrumentation_types.cpp b/src/instrumentation/src/instrumentation_types.cpp index 8423d557..9dce76ea 100644 --- a/src/instrumentation/src/instrumentation_types.cpp +++ b/src/instrumentation/src/instrumentation_types.cpp @@ -6,6 +6,21 @@ namespace instrumentation { +BRANCHING_PREDICATE opposite_predicate(BRANCHING_PREDICATE predicate) +{ + switch (predicate) + { + case BP_EQUAL: return BP_UNEQUAL; + case BP_UNEQUAL: return BP_EQUAL; + case BP_LESS: return BP_GREATER_EQUAL; + case BP_LESS_EQUAL: return BP_GREATER; + case BP_GREATER: return BP_LESS_EQUAL; + case BP_GREATER_EQUAL: return BP_LESS; + default: UNREACHABLE(); + } +} + + branching_coverage_info::branching_coverage_info(location_id const id_) : id{id_} , direction{} diff --git a/src/iomodels/src/iomanager.cpp b/src/iomodels/src/iomanager.cpp index 863cad73..2fc3f591 100644 --- a/src/iomodels/src/iomanager.cpp +++ b/src/iomodels/src/iomanager.cpp @@ -62,6 +62,7 @@ bool iomanager::load_trace_record(Medium& src) { src >> info.value; src >> info.idx_to_br_instr; src >> uchr; info.xor_like_branching_function = uchr != 0U; + src >> uchr; info.predicate = (BRANCHING_PREDICATE)uchr; info.num_input_bytes = (natural_32_bit)get_stdin()->get_bytes().size(); trace.push_back(info); return true; diff --git a/src/tools/instrumenter/llvm_instrumenter.cpp b/src/tools/instrumenter/llvm_instrumenter.cpp index 6b01e422..53d624ae 100644 --- a/src/tools/instrumenter/llvm_instrumenter.cpp +++ b/src/tools/instrumenter/llvm_instrumenter.cpp @@ -34,7 +34,7 @@ bool llvm_instrumenter::doInitialization(Module *M) { processCondFunc = module->getOrInsertFunction("__sbt_fizzer_process_condition", VoidTy, - Int32Ty, Int1Ty, DoubleTy, Int1Ty); + Int32Ty, Int1Ty, DoubleTy, Int1Ty, Int8Ty); processCondBrFunc = module->getOrInsertFunction("__sbt_fizzer_process_br_instr", VoidTy, @@ -75,52 +75,45 @@ void llvm_instrumenter::printErrCond(Value *cond) { Value *llvm_instrumenter::instrumentIcmp(Value *lhs, Value *rhs, CmpInst *cmpInst, IRBuilder<> &builder) { - - // pointer comparison -> consider the distance to be 1 - if (lhs->getType()->isPointerTy()) { - return ConstantFP::get(DoubleTy, 1); - } - - bool isUnsigned = cmpInst->isUnsigned(); - - if ((!lhs->getType()->isIntegerTy() || ((llvm::IntegerType const*)lhs->getType())->getBitWidth() < 64) && - (!rhs->getType()->isIntegerTy() || ((llvm::IntegerType const*)rhs->getType())->getBitWidth() < 64) ) + if (lhs->getType()->isPointerTy() && rhs->getType()->isPointerTy()) { - // if the value was extended we can't overflow meaning we don't need to - // cast to a higher type - if (!(dyn_cast(lhs) || dyn_cast(lhs) || - dyn_cast(rhs) || dyn_cast(rhs))) { - - // extend based on the signedness - if (isUnsigned) { - lhs = - builder.CreateZExt(lhs, lhs->getType()->getExtendedType()); - rhs = - builder.CreateZExt(rhs, rhs->getType()->getExtendedType()); - } - else { - lhs = - builder.CreateSExt(lhs, lhs->getType()->getExtendedType()); - rhs = - builder.CreateSExt(rhs, rhs->getType()->getExtendedType()); - } + llvm::ConstantPointerNull* const val = llvm::ConstantPointerNull::get(llvm::cast(lhs->getType())); + if (val == lhs || val == rhs) + { + Value* value = cmpInst->getPredicate() == llvm::CmpInst::Predicate::ICMP_EQ ? builder.CreateXor(cmpInst, 1) : cmpInst; + return builder.CreateUIToFP(builder.CreateZExt(value, Int32Ty), DoubleTy); } } - Value *distance = builder.CreateSub(lhs, rhs); + if (lhs->getType()->isPointerTy()) + lhs = builder.CreatePtrToInt(lhs, Int64Ty); + if (rhs->getType()->isPointerTy()) + rhs = builder.CreatePtrToInt(rhs, Int64Ty); - if (isUnsigned) { - return builder.CreateUIToFP(distance, DoubleTy); + if (cmpInst->isUnsigned()) + { + if (lhs->getType()->isIntegerTy()) + lhs = builder.CreateUIToFP(lhs, DoubleTy); + if (rhs->getType()->isIntegerTy()) + rhs = builder.CreateUIToFP(rhs, DoubleTy); } - return builder.CreateSIToFP(distance, DoubleTy); + else + { + if (lhs->getType()->isIntegerTy()) + lhs = builder.CreateSIToFP(lhs, DoubleTy); + if (rhs->getType()->isIntegerTy()) + rhs = builder.CreateSIToFP(rhs, DoubleTy); + } + + return instrumentFcmp(lhs, rhs, nullptr, builder); } -Value *llvm_instrumenter::instrumentFcmp(Value *lhs, Value *rhs, CmpInst *cmpInst, +Value *llvm_instrumenter::instrumentFcmp(Value *lhs, Value *rhs, CmpInst *, IRBuilder<> &builder) { - if (lhs->getType()->isFloatTy()) { + if (lhs->getType()->isFloatTy()) lhs = builder.CreateFPExt(lhs, DoubleTy); + if (rhs->getType()->isFloatTy()) rhs = builder.CreateFPExt(rhs, DoubleTy); - } Value *distance = builder.CreateFSub(lhs, rhs); @@ -148,15 +141,65 @@ bool llvm_instrumenter::instrumentCond(Instruction *inst, bool const xor_like_br } IRBuilder<> builder(inst->getNextNode()); + // We emulate the 'enum BRANCHING_PREDICATE' (see instrumentation_types.hpp) as follows: + natural_8_bit constexpr BP_EQUAL{ 0 }; + natural_8_bit constexpr BP_UNEQUAL{ 1 }; + natural_8_bit constexpr BP_LESS{ 2 }; + natural_8_bit constexpr BP_LESS_EQUAL{ 3 }; + natural_8_bit constexpr BP_GREATER{ 4 }; + natural_8_bit constexpr BP_GREATER_EQUAL{ 5 }; + natural_8_bit predicate{ BP_UNEQUAL }; + Value *distance; if (auto *cmpInst = dyn_cast(inst)) { distance = instrumentCmp(cmpInst, builder); + switch (cmpInst->getPredicate()) + { + case llvm::CmpInst::Predicate::FCMP_OEQ: + case llvm::CmpInst::Predicate::FCMP_UEQ: + case llvm::CmpInst::Predicate::ICMP_EQ: + predicate = BP_EQUAL; + break; + case llvm::CmpInst::Predicate::FCMP_OGT: + case llvm::CmpInst::Predicate::FCMP_UGT: + case llvm::CmpInst::Predicate::ICMP_UGT: + case llvm::CmpInst::Predicate::ICMP_SGT: + predicate = BP_GREATER; + break; + case llvm::CmpInst::Predicate::FCMP_OGE: + case llvm::CmpInst::Predicate::FCMP_UGE: + case llvm::CmpInst::Predicate::ICMP_UGE: + case llvm::CmpInst::Predicate::ICMP_SGE: + predicate = BP_GREATER_EQUAL; + break; + case llvm::CmpInst::Predicate::FCMP_OLT: + case llvm::CmpInst::Predicate::FCMP_ULT: + case llvm::CmpInst::Predicate::ICMP_ULT: + case llvm::CmpInst::Predicate::ICMP_SLT: + predicate = BP_LESS; + break; + case llvm::CmpInst::Predicate::FCMP_OLE: + case llvm::CmpInst::Predicate::FCMP_ULE: + case llvm::CmpInst::Predicate::ICMP_ULE: + case llvm::CmpInst::Predicate::ICMP_SLE: + predicate = BP_LESS_EQUAL; + break; + case llvm::CmpInst::Predicate::FCMP_ONE: + case llvm::CmpInst::Predicate::FCMP_UNE: + case llvm::CmpInst::Predicate::ICMP_NE: + predicate = BP_UNEQUAL; + break; + + case llvm::CmpInst::Predicate::FCMP_UNO: // TODO: "is_nan" - what to do with that? + default: + break; + } // truncating a number to i1, happens for example with bool in C } else if (dyn_cast(inst)) { - distance = ConstantFP::get(DoubleTy, 1); + distance = builder.CreateUIToFP(builder.CreateZExt(inst, Int32Ty), DoubleTy); // i1 as a return from a call to a function } else if (dyn_cast(inst)) { - distance = ConstantFP::get(DoubleTy, 1); + distance = builder.CreateUIToFP(builder.CreateZExt(inst, Int32Ty), DoubleTy); } else { return false; } @@ -164,8 +207,13 @@ bool llvm_instrumenter::instrumentCond(Instruction *inst, bool const xor_like_br Value *location = ConstantInt::get(Int32Ty, ++condCounter); Value *cond = inst; - builder.CreateCall(processCondFunc, - {location, cond, distance, ConstantInt::get(Int1Ty, xor_like_branching_function ? 1 : 0) }); + builder.CreateCall(processCondFunc, { + location, + cond, + distance, + ConstantInt::get(Int1Ty, xor_like_branching_function ? 1 : 0), + ConstantInt::get(Int8Ty, predicate) + }); return true; } From fccb83183dd2511431d44538a3f67e5399b5a6b1 Mon Sep 17 00:00:00 2001 From: Marek Trtik Date: Tue, 17 Sep 2024 02:43:14 +0200 Subject: [PATCH 018/144] Updated json of one benchmark --- benchmarks/fast/c_string_count_chars.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/fast/c_string_count_chars.json b/benchmarks/fast/c_string_count_chars.json index 9c777b89..465ad190 100644 --- a/benchmarks/fast/c_string_count_chars.json +++ b/benchmarks/fast/c_string_count_chars.json @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 3045, + "num_executions": 2709, "num_covered_branchings": 7, "covered_branchings": [ 1,0, 2,0, 3,0, 4,0, From acfd5a18a8dc7e98ee77a0b53026208176ea12fd Mon Sep 17 00:00:00 2001 From: Marek Trtik Date: Tue, 17 Sep 2024 16:32:13 +0200 Subject: [PATCH 019/144] Reducing randomness in fuzzer. --- src/fuzzing/include/fuzzing/minimization_analysis.hpp | 2 +- src/fuzzing/src/fuzzer.cpp | 6 +++--- src/fuzzing/src/typed_minimization_analysis.cpp | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/fuzzing/include/fuzzing/minimization_analysis.hpp b/src/fuzzing/include/fuzzing/minimization_analysis.hpp index 0f6c619f..f0718adb 100644 --- a/src/fuzzing/include/fuzzing/minimization_analysis.hpp +++ b/src/fuzzing/include/fuzzing/minimization_analysis.hpp @@ -62,7 +62,7 @@ struct minimization_analysis , descent{} , computed_input_stdin{} , hashes_of_generated_bits{} - , random_generator{} + , random_generator{ 1U } , stopped_early{ false } , statistics{} {} diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index faac92dd..e973e436 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -801,9 +801,9 @@ fuzzer::fuzzer(termination_info const& info) , max_input_width{ 0U } - , generator_for_iid_location_selection{} - , generator_for_iid_approach_selection{} - , generator_for_generator_selection{} + , generator_for_iid_location_selection{ 1U } + , generator_for_iid_approach_selection{ 1U } + , generator_for_generator_selection{ 1U } , statistics{} {} diff --git a/src/fuzzing/src/typed_minimization_analysis.cpp b/src/fuzzing/src/typed_minimization_analysis.cpp index 78445797..1b3d5811 100644 --- a/src/fuzzing/src/typed_minimization_analysis.cpp +++ b/src/fuzzing/src/typed_minimization_analysis.cpp @@ -43,8 +43,8 @@ typed_minimization_analysis::typed_minimization_analysis() , hashes_of_generated_bits{} , num_fast_and_genuine_executions{ 0U } , stopped_early{ false } - , random_generator32{} - , random_generator64{} + , random_generator32{ 1U } + , random_generator64{ 1U } , statistics{} {} From ae7af3ad2c93f7d51c63786edcdea8de156f9e97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Tue, 17 Sep 2024 18:43:27 +0200 Subject: [PATCH 020/144] feat: updating matrix with one equation --- src/fuzzing/include/fuzzing/fuzzer.hpp | 3 +- src/fuzzing/src/fuzzer.cpp | 95 ++++++++++++++++---------- 2 files changed, 62 insertions(+), 36 deletions(-) diff --git a/src/fuzzing/include/fuzzing/fuzzer.hpp b/src/fuzzing/include/fuzzing/fuzzer.hpp index c6da5bd3..f00407c7 100644 --- a/src/fuzzing/include/fuzzing/fuzzer.hpp +++ b/src/fuzzing/include/fuzzing/fuzzer.hpp @@ -241,8 +241,9 @@ struct fuzzer final std::set interesting_nodes; std::vector> matrix; - void update_interesting_nodes(branching_node* node); + bool update_interesting_nodes(branching_node* node); void recompute_matrix(); + void add_equation(branching_node* node); }; std::unordered_map id_to_equation_map; diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index 0dd36f1c..1fa792cb 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -972,56 +972,70 @@ void fuzzer::generate_next_input(vecb& stdin_bits) UNREACHABLE(); } - void fuzzer::process_node_dependance(branching_node *node) { using deps_props = iid_node_dependence::iid_dependence_props; if (iid_dependences.non_iid_nodes.contains(node->get_location_id())) return; - deps_props& props = iid_dependences.id_to_equation_map[node->get_location_id()]; + deps_props &props = iid_dependences.id_to_equation_map[node->get_location_id()]; props.all_paths.push_back(node); - props.update_interesting_nodes(node); - props.recompute_matrix(); + + if (props.update_interesting_nodes(node)) + { + props.recompute_matrix(); + } + + props.add_equation(node); } -void fuzzing::fuzzer::iid_node_dependence::iid_dependence_props::update_interesting_nodes(branching_node *node) +bool fuzzing::fuzzer::iid_node_dependence::iid_dependence_props::update_interesting_nodes(branching_node *node) { using node_nav = iid_node_dependence::node_navigation; + bool set_changed = false; - auto get_path = [](branching_node* node) { - std::vector path; - branching_node* current = node; - while (current != nullptr) { + auto get_path = [](branching_node *node) + { + std::vector path; + branching_node *current = node; + while (current != nullptr) + { path.push_back(current); current = current->predecessor; } return path; }; - auto add_to_interesting = [this](branching_node* node, size_t i) { - while (i > 0) { - branching_node* predecessor = node->predecessor; + auto add_to_interesting = [this, &set_changed](branching_node *node, size_t i) + { + while (i > 0) + { + branching_node *predecessor = node->predecessor; - this->interesting_nodes.emplace(node->get_location_id(), node == predecessor->successor(true).pointer); + auto result = this->interesting_nodes.emplace(node->get_location_id(), node == predecessor->successor(true).pointer); + if (result.second) + { + set_changed = true; + } node = node->predecessor; --i; } }; - - for (const auto& path : all_paths) { - std::vector path_1 = get_path(node); - std::vector path_2 = get_path(path); - + for (const auto &path : all_paths) + { + std::vector path_1 = get_path(node); + std::vector path_2 = get_path(path); + ASSUMPTION(path_1.back() == path_2.back()); std::size_t i_1 = path_1.size() - 1; std::size_t i_2 = path_2.size() - 1; - while (i_1 > 0 && i_2 > 0 && path_1[i_1] == path_2[i_2]) { + while (i_1 > 0 && i_2 > 0 && path_1[i_1] == path_2[i_2]) + { --i_1; --i_2; } @@ -1029,6 +1043,8 @@ void fuzzing::fuzzer::iid_node_dependence::iid_dependence_props::update_interest add_to_interesting(path_1[i_1], i_1); add_to_interesting(path_2[i_2], i_2); } + + return set_changed; } void fuzzing::fuzzer::iid_node_dependence::iid_dependence_props::recompute_matrix() @@ -1039,28 +1055,37 @@ void fuzzing::fuzzer::iid_node_dependence::iid_dependence_props::recompute_matri matrix.clear(); // This could be done better, but for now it's fine - for (const auto& path : all_paths) { - std::map directions_in_path; + for (const auto &path : all_paths) + { + add_equation(path); + } +} - branching_node* node = path; - branching_node* prev_node = node->predecessor; +void fuzzing::fuzzer::iid_node_dependence::iid_dependence_props::add_equation(branching_node *node) +{ + using node_nav = iid_node_dependence::node_navigation; + + std::map directions_in_path; - while (prev_node != nullptr) { - node_nav nav = { prev_node->get_location_id(), prev_node->successor(true).pointer == node }; - directions_in_path[nav]++; + branching_node *node = node; + branching_node *prev_node = node->predecessor; - node = prev_node; - prev_node = node->predecessor; - } + while (prev_node != nullptr) + { + node_nav nav = {prev_node->get_location_id(), prev_node->successor(true).pointer == node}; + directions_in_path[nav]++; - std::vector values_in_path; - for (const auto& [direction, count] : directions_in_path) { - values_in_path.push_back(count); - } + node = prev_node; + prev_node = node->predecessor; + } - matrix.push_back(values_in_path); + std::vector values_in_path; + for (const auto &[direction, count] : directions_in_path) + { + values_in_path.push_back(count); } - + + matrix.push_back(values_in_path); } execution_record::execution_flags fuzzer::process_execution_results() From 12a249ba477020ac883aaa636f4c9893ff9125fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Tue, 17 Sep 2024 19:14:11 +0200 Subject: [PATCH 021/144] feat: equation approximation --- src/fuzzing/include/fuzzing/fuzzer.hpp | 12 ++--- src/fuzzing/src/fuzzer.cpp | 69 ++++++++++++++++++++++++-- 2 files changed, 69 insertions(+), 12 deletions(-) diff --git a/src/fuzzing/include/fuzzing/fuzzer.hpp b/src/fuzzing/include/fuzzing/fuzzer.hpp index f00407c7..04b1d6a6 100644 --- a/src/fuzzing/include/fuzzing/fuzzer.hpp +++ b/src/fuzzing/include/fuzzing/fuzzer.hpp @@ -226,13 +226,7 @@ struct fuzzer final location_id node_id; bool direction; - auto operator<=>(node_navigation const& other) const - { - if(auto const cmp = node_id.id <=> other.node_id.id; cmp != 0) - return cmp; - - return direction <=> other.direction; - } + auto operator<=>(node_navigation const& other) const; }; struct iid_dependence_props @@ -240,10 +234,12 @@ struct fuzzer final std::vector all_paths; std::set interesting_nodes; std::vector> matrix; + std::vector best_values; bool update_interesting_nodes(branching_node* node); void recompute_matrix(); - void add_equation(branching_node* node); + void add_equation(branching_node* path); + std::vector approximate_matrix(); }; std::unordered_map id_to_equation_map; diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index 1fa792cb..47594161 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -972,6 +972,14 @@ void fuzzer::generate_next_input(vecb& stdin_bits) UNREACHABLE(); } +auto fuzzing::fuzzer::iid_node_dependence::node_navigation::operator<=>(node_navigation const &other) const +{ + if (auto const cmp = node_id.id <=> other.node_id.id; cmp != 0) + return cmp; + + return direction <=> other.direction; +} + void fuzzer::process_node_dependance(branching_node *node) { using deps_props = iid_node_dependence::iid_dependence_props; @@ -1009,11 +1017,12 @@ bool fuzzing::fuzzer::iid_node_dependence::iid_dependence_props::update_interest auto add_to_interesting = [this, &set_changed](branching_node *node, size_t i) { - while (i > 0) + while (i > 1) { branching_node *predecessor = node->predecessor; - auto result = this->interesting_nodes.emplace(node->get_location_id(), node == predecessor->successor(true).pointer); + bool direction = node == predecessor->successor(true).pointer; + auto result = this->interesting_nodes.emplace(node->get_location_id(), direction); if (result.second) { set_changed = true; @@ -1061,13 +1070,13 @@ void fuzzing::fuzzer::iid_node_dependence::iid_dependence_props::recompute_matri } } -void fuzzing::fuzzer::iid_node_dependence::iid_dependence_props::add_equation(branching_node *node) +void fuzzing::fuzzer::iid_node_dependence::iid_dependence_props::add_equation(branching_node *path) { using node_nav = iid_node_dependence::node_navigation; std::map directions_in_path; - branching_node *node = node; + branching_node *node = path; branching_node *prev_node = node->predecessor; while (prev_node != nullptr) @@ -1086,6 +1095,58 @@ void fuzzing::fuzzer::iid_node_dependence::iid_dependence_props::add_equation(br } matrix.push_back(values_in_path); + best_values.push_back(node->best_coverage_value); +} + +std::vector fuzzing::fuzzer::iid_node_dependence::iid_dependence_props::approximate_matrix() +{ + const float learning_rate = 0.01f; + const int max_iterations = 1000; + const float tolerance = 1e-6f; + + size_t m = matrix.size(); + size_t n = matrix[0].size(); + + std::vector weights(n, 0.0f); // Initialize weights to zero + + for (int iter = 0; iter < max_iterations; ++iter) + { + std::vector gradient(n, 0.0f); + + // Compute gradient + for (size_t i = 0; i < m; ++i) + { + float dot_product = 0.0f; + for (size_t j = 0; j < n; ++j) + { + dot_product += matrix[i][j] * weights[j]; + } + float error = dot_product - best_values[i]; + for (size_t j = 0; j < n; ++j) + { + gradient[j] += matrix[i][j] * error; + } + } + + // Update weights + for (size_t j = 0; j < n; ++j) + { + weights[j] -= learning_rate * gradient[j]; + } + + // Check for convergence + float max_change = 0.0f; + for (size_t j = 0; j < n; ++j) + { + max_change = std::max(max_change, std::abs(learning_rate * gradient[j])); + } + if (max_change < tolerance) + { + break; + } + } + + return weights; } execution_record::execution_flags fuzzer::process_execution_results() From a0dcb86faca3da8ca8d362fb804546b55ea4a5c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Tue, 17 Sep 2024 19:53:32 +0200 Subject: [PATCH 022/144] feat: struct refactoring --- src/fuzzing/include/fuzzing/fuzzer.hpp | 43 +++++++++++++------------- src/fuzzing/src/fuzzer.cpp | 37 ++++++++++------------ 2 files changed, 37 insertions(+), 43 deletions(-) diff --git a/src/fuzzing/include/fuzzing/fuzzer.hpp b/src/fuzzing/include/fuzzing/fuzzer.hpp index 04b1d6a6..061bc67b 100644 --- a/src/fuzzing/include/fuzzing/fuzzer.hpp +++ b/src/fuzzing/include/fuzzing/fuzzer.hpp @@ -218,30 +218,29 @@ struct fuzzer final mutable random_generator_for_natural_32_bit generator_for_pivot_selection; }; - struct iid_node_dependence + struct node_navigation + { + location_id node_id; + bool direction; + + auto operator<=>(node_navigation const& other) const; + }; + + struct iid_dependence_props { - - struct node_navigation - { - location_id node_id; - bool direction; - - auto operator<=>(node_navigation const& other) const; - }; - - struct iid_dependence_props - { - std::vector all_paths; - std::set interesting_nodes; - std::vector> matrix; - std::vector best_values; - - bool update_interesting_nodes(branching_node* node); - void recompute_matrix(); - void add_equation(branching_node* path); - std::vector approximate_matrix(); - }; + std::vector all_paths; + std::set interesting_nodes; + std::vector> matrix; + std::vector best_values; + + bool update_interesting_nodes(branching_node* node); + void recompute_matrix(); + void add_equation(branching_node* path); + std::vector approximate_matrix(); + }; + struct iid_node_dependence + { std::unordered_map id_to_equation_map; std::set non_iid_nodes; }; diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index 47594161..38942f5e 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -972,7 +972,7 @@ void fuzzer::generate_next_input(vecb& stdin_bits) UNREACHABLE(); } -auto fuzzing::fuzzer::iid_node_dependence::node_navigation::operator<=>(node_navigation const &other) const +auto fuzzing::fuzzer::node_navigation::operator<=>(node_navigation const &other) const { if (auto const cmp = node_id.id <=> other.node_id.id; cmp != 0) return cmp; @@ -982,11 +982,9 @@ auto fuzzing::fuzzer::iid_node_dependence::node_navigation::operator<=>(node_nav void fuzzer::process_node_dependance(branching_node *node) { - using deps_props = iid_node_dependence::iid_dependence_props; - if (iid_dependences.non_iid_nodes.contains(node->get_location_id())) return; - deps_props &props = iid_dependences.id_to_equation_map[node->get_location_id()]; + iid_dependence_props &props = iid_dependences.id_to_equation_map[node->get_location_id()]; props.all_paths.push_back(node); @@ -998,9 +996,8 @@ void fuzzer::process_node_dependance(branching_node *node) props.add_equation(node); } -bool fuzzing::fuzzer::iid_node_dependence::iid_dependence_props::update_interesting_nodes(branching_node *node) +bool fuzzing::fuzzer::iid_dependence_props::update_interesting_nodes(branching_node *node) { - using node_nav = iid_node_dependence::node_navigation; bool set_changed = false; auto get_path = [](branching_node *node) @@ -1015,21 +1012,22 @@ bool fuzzing::fuzzer::iid_node_dependence::iid_dependence_props::update_interest return path; }; - auto add_to_interesting = [this, &set_changed](branching_node *node, size_t i) + auto add_to_interesting = [this, &set_changed](std::vector &nodes, int i) { - while (i > 1) + for (; i >= 0; --i) { + branching_node *node = nodes[i]; branching_node *predecessor = node->predecessor; + if (predecessor == nullptr) + continue; + bool direction = node == predecessor->successor(true).pointer; auto result = this->interesting_nodes.emplace(node->get_location_id(), direction); if (result.second) { set_changed = true; } - - node = node->predecessor; - --i; } }; @@ -1049,16 +1047,15 @@ bool fuzzing::fuzzer::iid_node_dependence::iid_dependence_props::update_interest --i_2; } - add_to_interesting(path_1[i_1], i_1); - add_to_interesting(path_2[i_2], i_2); + add_to_interesting(path_1, i_1); + add_to_interesting(path_2, i_2); } return set_changed; } -void fuzzing::fuzzer::iid_node_dependence::iid_dependence_props::recompute_matrix() +void fuzzing::fuzzer::iid_dependence_props::recompute_matrix() { - using node_nav = iid_node_dependence::node_navigation; if (all_paths.empty()) return; @@ -1070,18 +1067,16 @@ void fuzzing::fuzzer::iid_node_dependence::iid_dependence_props::recompute_matri } } -void fuzzing::fuzzer::iid_node_dependence::iid_dependence_props::add_equation(branching_node *path) +void fuzzing::fuzzer::iid_dependence_props::add_equation(branching_node *path) { - using node_nav = iid_node_dependence::node_navigation; - - std::map directions_in_path; + std::map directions_in_path; branching_node *node = path; branching_node *prev_node = node->predecessor; while (prev_node != nullptr) { - node_nav nav = {prev_node->get_location_id(), prev_node->successor(true).pointer == node}; + node_navigation nav = {prev_node->get_location_id(), prev_node->successor(true).pointer == node}; directions_in_path[nav]++; node = prev_node; @@ -1098,7 +1093,7 @@ void fuzzing::fuzzer::iid_node_dependence::iid_dependence_props::add_equation(br best_values.push_back(node->best_coverage_value); } -std::vector fuzzing::fuzzer::iid_node_dependence::iid_dependence_props::approximate_matrix() +std::vector fuzzing::fuzzer::iid_dependence_props::approximate_matrix() { const float learning_rate = 0.01f; const int max_iterations = 1000; From a1831a32a3d81800a15690d3d436233818db9e60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Wed, 18 Sep 2024 21:53:16 +0200 Subject: [PATCH 023/144] feat: small changes --- src/fuzzing/include/fuzzing/fuzzer.hpp | 2 +- src/fuzzing/src/fuzzer.cpp | 26 +++++++++++++++++++++----- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/fuzzing/include/fuzzing/fuzzer.hpp b/src/fuzzing/include/fuzzing/fuzzer.hpp index 061bc67b..367ce5c4 100644 --- a/src/fuzzing/include/fuzzing/fuzzer.hpp +++ b/src/fuzzing/include/fuzzing/fuzzer.hpp @@ -236,7 +236,7 @@ struct fuzzer final bool update_interesting_nodes(branching_node* node); void recompute_matrix(); void add_equation(branching_node* path); - std::vector approximate_matrix(); + std::vector approximate_matrix() const; }; struct iid_node_dependence diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index 38942f5e..de06c7c2 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -1077,7 +1077,8 @@ void fuzzing::fuzzer::iid_dependence_props::add_equation(branching_node *path) while (prev_node != nullptr) { node_navigation nav = {prev_node->get_location_id(), prev_node->successor(true).pointer == node}; - directions_in_path[nav]++; + if (interesting_nodes.contains(nav)) + directions_in_path[nav]++; node = prev_node; prev_node = node->predecessor; @@ -1093,16 +1094,16 @@ void fuzzing::fuzzer::iid_dependence_props::add_equation(branching_node *path) best_values.push_back(node->best_coverage_value); } -std::vector fuzzing::fuzzer::iid_dependence_props::approximate_matrix() +std::vector fuzzing::fuzzer::iid_dependence_props::approximate_matrix() const { - const float learning_rate = 0.01f; + const float learning_rate = 0.001f; const int max_iterations = 1000; const float tolerance = 1e-6f; size_t m = matrix.size(); size_t n = matrix[0].size(); - std::vector weights(n, 0.0f); // Initialize weights to zero + std::vector weights(n, 1.0f); // Initialize weights to zero for (int iter = 0; iter < max_iterations; ++iter) { @@ -1119,7 +1120,7 @@ std::vector fuzzing::fuzzer::iid_dependence_props::approximate_matrix() float error = dot_product - best_values[i]; for (size_t j = 0; j < n; ++j) { - gradient[j] += matrix[i][j] * error; + gradient[j] += (matrix[i][j] * error) / m; } } @@ -1773,6 +1774,21 @@ branching_node* fuzzer::select_iid_coverage_target() const INVARIANT(winner != nullptr); + const iid_dependence_props& props = iid_dependences.id_to_equation_map.at(winner->get_location_id()); + std::vector weights = props.approximate_matrix(); + + std::cout << weights.size() << " " << props.interesting_nodes.size() << std::endl; + + std::cout << "ID: " << winner->get_location_id().id << std::endl; + for (const auto& node : props.interesting_nodes){ + std::cout << node.node_id.id << " " << node.direction << std::endl; + } + std::cout << std::endl; + for (const auto& weight : weights) { + std::cout << weight << " "; + } + std::cout << std::endl; + return winner; } From 59e8f36fc28b5c3d8d358d594a86f82de4e427e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sun, 22 Sep 2024 16:41:55 +0200 Subject: [PATCH 024/144] feat: doxygen comment for new functions --- src/fuzzing/src/fuzzer.cpp | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index de06c7c2..a884c5d1 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -980,10 +980,22 @@ auto fuzzing::fuzzer::node_navigation::operator<=>(node_navigation const &other) return direction <=> other.direction; } +/** + * @brief Processes the dependencies of a branching node. + * + * This function checks if the given node's location ID is contained within + * the non-IID nodes. If it is, the function returns early. Otherwise, it + * retrieves the properties associated with the node's location ID and updates + * the list of all paths with the given node. If the node is deemed interesting, + * the function recomputes the matrix. Finally, it adds an equation for the node. + * + * @param node A pointer to the branching node whose dependencies are to be processed. + */ void fuzzer::process_node_dependance(branching_node *node) { if (iid_dependences.non_iid_nodes.contains(node->get_location_id())) return; + iid_dependence_props &props = iid_dependences.id_to_equation_map[node->get_location_id()]; props.all_paths.push_back(node); @@ -996,6 +1008,16 @@ void fuzzer::process_node_dependance(branching_node *node) props.add_equation(node); } +/** + * @brief Updates the set of interesting nodes based on the given branching node. + * + * This function traverses the paths from the given branching node and updates the set of interesting nodes + * by comparing the paths with all existing paths. If new interesting nodes are found, the set is updated + * and the function returns true. + * + * @param node A pointer to the branching node from which to start the path traversal. + * @return true if the set of interesting nodes was updated, false otherwise. + */ bool fuzzing::fuzzer::iid_dependence_props::update_interesting_nodes(branching_node *node) { bool set_changed = false; @@ -1054,6 +1076,13 @@ bool fuzzing::fuzzer::iid_dependence_props::update_interesting_nodes(branching_n return set_changed; } +/** + * @brief Recomputes the matrix based on the current paths. + * + * This function clears the existing matrix and then iterates over all paths, + * adding an equation for each path to the matrix. If there are no paths, + * the function returns immediately without modifying the matrix. + */ void fuzzing::fuzzer::iid_dependence_props::recompute_matrix() { if (all_paths.empty()) @@ -1067,6 +1096,14 @@ void fuzzing::fuzzer::iid_dependence_props::recompute_matrix() } } +/** + * @brief Adds an equation to the IID dependence properties based on the given branching path. + * + * This function traverses the given branching path, collects the directions of interesting nodes, + * and updates the internal matrix and best values with the collected data. + * + * @param path A pointer to the branching node representing the path to be analyzed. + */ void fuzzing::fuzzer::iid_dependence_props::add_equation(branching_node *path) { std::map directions_in_path; From 098fe2cdb1a5b84a4c69acc8d064c7de1950e0e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sun, 22 Sep 2024 16:42:21 +0200 Subject: [PATCH 025/144] feat: doxygen comment for new functions --- src/fuzzing/src/fuzzer.cpp | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index a884c5d1..b6578b2f 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -1131,6 +1131,27 @@ void fuzzing::fuzzer::iid_dependence_props::add_equation(branching_node *path) best_values.push_back(node->best_coverage_value); } +/** + * @brief Approximates the weight matrix using gradient descent. + * + * This function performs gradient descent to approximate the weight matrix + * that minimizes the error between the dot product of the input matrix and + * the best values. The gradient descent algorithm iteratively updates the + * weights until convergence or until the maximum number of iterations is reached. + * + * @return A vector of floats representing the approximated weights. + * + * The function uses the following parameters: + * - learning_rate: The rate at which the weights are updated during each iteration. + * - max_iterations: The maximum number of iterations for the gradient descent algorithm. + * - tolerance: The threshold for convergence. If the maximum change in weights is less than this value, the algorithm stops. + * + * The function performs the following steps: + * 1. Initializes the weights to 1.0. + * 2. Iteratively computes the gradient of the error with respect to the weights. + * 3. Updates the weights using the computed gradient and the learning rate. + * 4. Checks for convergence by comparing the maximum change in weights to the tolerance. + */ std::vector fuzzing::fuzzer::iid_dependence_props::approximate_matrix() const { const float learning_rate = 0.001f; From 08d9f6bafd0965a4b6653ca709326e12f25b1875 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sun, 22 Sep 2024 22:34:51 +0200 Subject: [PATCH 026/144] feat: small changes and output operator --- src/fuzzing/include/fuzzing/fuzzer.hpp | 5 +++++ src/fuzzing/src/fuzzer.cpp | 30 +++++++++++++++++++++----- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/src/fuzzing/include/fuzzing/fuzzer.hpp b/src/fuzzing/include/fuzzing/fuzzer.hpp index 367ce5c4..712d6062 100644 --- a/src/fuzzing/include/fuzzing/fuzzer.hpp +++ b/src/fuzzing/include/fuzzing/fuzzer.hpp @@ -224,6 +224,11 @@ struct fuzzer final bool direction; auto operator<=>(node_navigation const& other) const; + friend std::ostream& operator<<(std::ostream& os, node_navigation const& nn) + { + os << nn.node_id.id << " " << (nn.direction ? "right" : "left"); + return os; + } }; struct iid_dependence_props diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index b6578b2f..486079de 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -972,7 +972,20 @@ void fuzzer::generate_next_input(vecb& stdin_bits) UNREACHABLE(); } -auto fuzzing::fuzzer::node_navigation::operator<=>(node_navigation const &other) const +/** + * @brief Compares this node_navigation object with another node_navigation object. + * + * This operator performs a three-way comparison between the current node_navigation + * object and another node_navigation object. The comparison is first based on the + * node_id.id. If the node_id.id values are equal, it then compares the direction. + * + * @param other The other node_navigation object to compare with. + * @return std::strong_ordering Result of the comparison: + * - std::strong_ordering::less if this object is less than the other. + * - std::strong_ordering::equal if this object is equal to the other. + * - std::strong_ordering::greater if this object is greater than the other. + */ +auto fuzzer::node_navigation::operator<=>(node_navigation const &other) const { if (auto const cmp = node_id.id <=> other.node_id.id; cmp != 0) return cmp; @@ -980,6 +993,7 @@ auto fuzzing::fuzzer::node_navigation::operator<=>(node_navigation const &other) return direction <=> other.direction; } + /** * @brief Processes the dependencies of a branching node. * @@ -1008,6 +1022,7 @@ void fuzzer::process_node_dependance(branching_node *node) props.add_equation(node); } + /** * @brief Updates the set of interesting nodes based on the given branching node. * @@ -1076,6 +1091,7 @@ bool fuzzing::fuzzer::iid_dependence_props::update_interesting_nodes(branching_n return set_changed; } + /** * @brief Recomputes the matrix based on the current paths. * @@ -1096,6 +1112,7 @@ void fuzzing::fuzzer::iid_dependence_props::recompute_matrix() } } + /** * @brief Adds an equation to the IID dependence properties based on the given branching path. * @@ -1107,6 +1124,10 @@ void fuzzing::fuzzer::iid_dependence_props::recompute_matrix() void fuzzing::fuzzer::iid_dependence_props::add_equation(branching_node *path) { std::map directions_in_path; + for (const node_navigation& navigation : interesting_nodes) + { + directions_in_path[navigation] = 0; + } branching_node *node = path; branching_node *prev_node = node->predecessor; @@ -1131,6 +1152,7 @@ void fuzzing::fuzzer::iid_dependence_props::add_equation(branching_node *path) best_values.push_back(node->best_coverage_value); } + /** * @brief Approximates the weight matrix using gradient descent. * @@ -1835,11 +1857,9 @@ branching_node* fuzzer::select_iid_coverage_target() const const iid_dependence_props& props = iid_dependences.id_to_equation_map.at(winner->get_location_id()); std::vector weights = props.approximate_matrix(); - std::cout << weights.size() << " " << props.interesting_nodes.size() << std::endl; - std::cout << "ID: " << winner->get_location_id().id << std::endl; - for (const auto& node : props.interesting_nodes){ - std::cout << node.node_id.id << " " << node.direction << std::endl; + for (const auto& nav : props.interesting_nodes) { + std::cout << nav << " "; } std::cout << std::endl; for (const auto& weight : weights) { From b9f28c049a7580535adc8ddaa50fd2264ec4ad9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sun, 22 Sep 2024 22:35:58 +0200 Subject: [PATCH 027/144] feat: add timings --- src/fuzzing/src/fuzzer.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index 486079de..325382f9 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -1007,6 +1007,8 @@ auto fuzzer::node_navigation::operator<=>(node_navigation const &other) const */ void fuzzer::process_node_dependance(branching_node *node) { + TMPROF_BLOCK(); + if (iid_dependences.non_iid_nodes.contains(node->get_location_id())) return; @@ -1035,6 +1037,8 @@ void fuzzer::process_node_dependance(branching_node *node) */ bool fuzzing::fuzzer::iid_dependence_props::update_interesting_nodes(branching_node *node) { + TMPROF_BLOCK(); + bool set_changed = false; auto get_path = [](branching_node *node) @@ -1101,6 +1105,8 @@ bool fuzzing::fuzzer::iid_dependence_props::update_interesting_nodes(branching_n */ void fuzzing::fuzzer::iid_dependence_props::recompute_matrix() { + TMPROF_BLOCK(); + if (all_paths.empty()) return; @@ -1123,6 +1129,8 @@ void fuzzing::fuzzer::iid_dependence_props::recompute_matrix() */ void fuzzing::fuzzer::iid_dependence_props::add_equation(branching_node *path) { + TMPROF_BLOCK(); + std::map directions_in_path; for (const node_navigation& navigation : interesting_nodes) { @@ -1176,6 +1184,8 @@ void fuzzing::fuzzer::iid_dependence_props::add_equation(branching_node *path) */ std::vector fuzzing::fuzzer::iid_dependence_props::approximate_matrix() const { + TMPROF_BLOCK(); + const float learning_rate = 0.001f; const int max_iterations = 1000; const float tolerance = 1e-6f; From 39322a987fb7f3ec5a992e6afe8caf0976da8074 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Tue, 24 Sep 2024 12:15:23 +0200 Subject: [PATCH 028/144] feat: gradient descent --- src/fuzzing/include/fuzzing/fuzzer.hpp | 2 +- .../include/fuzzing/gradient_descent.hpp | 28 ++ src/fuzzing/src/fuzzer.cpp | 255 ++++++++++-------- src/fuzzing/src/gradient_descent.cpp | 117 ++++++++ 4 files changed, 288 insertions(+), 114 deletions(-) create mode 100644 src/fuzzing/include/fuzzing/gradient_descent.hpp create mode 100644 src/fuzzing/src/gradient_descent.cpp diff --git a/src/fuzzing/include/fuzzing/fuzzer.hpp b/src/fuzzing/include/fuzzing/fuzzer.hpp index 712d6062..45559218 100644 --- a/src/fuzzing/include/fuzzing/fuzzer.hpp +++ b/src/fuzzing/include/fuzzing/fuzzer.hpp @@ -318,7 +318,7 @@ struct fuzzer final ); void generate_next_input(vecb& stdin_bits); - void process_node_dependance(branching_node* node); + void process_node_dependence(branching_node* node); execution_record::execution_flags process_execution_results(); void do_cleanup(); diff --git a/src/fuzzing/include/fuzzing/gradient_descent.hpp b/src/fuzzing/include/fuzzing/gradient_descent.hpp new file mode 100644 index 00000000..e7551719 --- /dev/null +++ b/src/fuzzing/include/fuzzing/gradient_descent.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include +#include + +class GradientDescent { +public: + GradientDescent(float learning_rate = 0.001f, int max_iterations = 10000, float convergence_threshold = 1e-6); + + std::vector optimize(const std::vector>& coefficient_matrix, const std::vector& target_vector); + + // Setters for easy modification of parameters + void setLearningRate(float learning_rate) { _learning_rate = learning_rate; } + void setMaxIterations(int max_iterations) { _max_iterations = max_iterations; }; + void setConvergenceThreshold(float convergence_threshold) { _convergence_threshold = convergence_threshold; }; + + +private: + float _learning_rate; + int _max_iterations; + float _convergence_threshold; + + std::vector computeGradient(const std::vector& current_solution, const std::vector>& coefficient_matrix, const std::vector& target_vector); + float computeMeanSquaredError(const std::vector& current_solution, const std::vector>& coefficient_matrix, const std::vector& target_vector); + std::vector generateRandomWeights(size_t n); + float dotProduct(const std::vector& a, const std::vector& b); + +}; diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index 325382f9..eba7e7b9 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -7,6 +7,8 @@ #include #include #include +#include +#include namespace fuzzing { @@ -974,20 +976,20 @@ void fuzzer::generate_next_input(vecb& stdin_bits) /** * @brief Compares this node_navigation object with another node_navigation object. - * - * This operator performs a three-way comparison between the current node_navigation - * object and another node_navigation object. The comparison is first based on the + * + * This operator performs a three-way comparison between the current node_navigation + * object and another node_navigation object. The comparison is first based on the * node_id.id. If the node_id.id values are equal, it then compares the direction. - * + * * @param other The other node_navigation object to compare with. * @return std::strong_ordering Result of the comparison: * - std::strong_ordering::less if this object is less than the other. * - std::strong_ordering::equal if this object is equal to the other. * - std::strong_ordering::greater if this object is greater than the other. */ -auto fuzzer::node_navigation::operator<=>(node_navigation const &other) const +auto fuzzer::node_navigation::operator<=>( node_navigation const& other ) const { - if (auto const cmp = node_id.id <=> other.node_id.id; cmp != 0) + if ( auto const cmp = node_id.id <=> other.node_id.id; cmp != 0 ) return cmp; return direction <=> other.direction; @@ -996,100 +998,93 @@ auto fuzzer::node_navigation::operator<=>(node_navigation const &other) const /** * @brief Processes the dependencies of a branching node. - * + * * This function checks if the given node's location ID is contained within * the non-IID nodes. If it is, the function returns early. Otherwise, it * retrieves the properties associated with the node's location ID and updates * the list of all paths with the given node. If the node is deemed interesting, * the function recomputes the matrix. Finally, it adds an equation for the node. - * + * * @param node A pointer to the branching node whose dependencies are to be processed. */ -void fuzzer::process_node_dependance(branching_node *node) +void fuzzer::process_node_dependence( branching_node* node ) { TMPROF_BLOCK(); - - if (iid_dependences.non_iid_nodes.contains(node->get_location_id())) + + if ( iid_dependences.non_iid_nodes.contains( node->get_location_id() ) ) return; - - iid_dependence_props &props = iid_dependences.id_to_equation_map[node->get_location_id()]; - props.all_paths.push_back(node); + iid_dependence_props& props = iid_dependences.id_to_equation_map[ node->get_location_id() ]; - if (props.update_interesting_nodes(node)) - { + props.all_paths.push_back( node ); + + if ( props.update_interesting_nodes( node ) ) { props.recompute_matrix(); } - props.add_equation(node); + props.add_equation( node ); } /** * @brief Updates the set of interesting nodes based on the given branching node. * - * This function traverses the paths from the given branching node and updates the set of interesting nodes - * by comparing the paths with all existing paths. If new interesting nodes are found, the set is updated - * and the function returns true. + * This function traverses the paths from the given branching node and updates the set of + * interesting nodes by comparing the paths with all existing paths. If new interesting nodes are + * found, the set is updated and the function returns true. * * @param node A pointer to the branching node from which to start the path traversal. * @return true if the set of interesting nodes was updated, false otherwise. */ -bool fuzzing::fuzzer::iid_dependence_props::update_interesting_nodes(branching_node *node) +bool fuzzing::fuzzer::iid_dependence_props::update_interesting_nodes( branching_node* node ) { TMPROF_BLOCK(); bool set_changed = false; - auto get_path = [](branching_node *node) - { - std::vector path; - branching_node *current = node; - while (current != nullptr) - { - path.push_back(current); + auto get_path = []( branching_node* node ) { + std::vector< branching_node* > path; + branching_node* current = node; + while ( current != nullptr ) { + path.push_back( current ); current = current->predecessor; } return path; }; - auto add_to_interesting = [this, &set_changed](std::vector &nodes, int i) - { - for (; i >= 0; --i) - { - branching_node *node = nodes[i]; - branching_node *predecessor = node->predecessor; + auto add_to_interesting = [ this, &set_changed ]( std::vector< branching_node* >& nodes, int i ) { + for ( ; i >= 0; --i ) { + branching_node* node = nodes[ i ]; + branching_node* predecessor = node->predecessor; - if (predecessor == nullptr) + if ( predecessor == nullptr ) continue; - bool direction = node == predecessor->successor(true).pointer; - auto result = this->interesting_nodes.emplace(node->get_location_id(), direction); - if (result.second) - { + bool direction = node == predecessor->successor( true ).pointer; + auto result = this->interesting_nodes.emplace( node->get_location_id(), !direction ); + result = this->interesting_nodes.emplace( node->get_location_id(), direction ); + if ( result.second ) { set_changed = true; } } }; - for (const auto &path : all_paths) - { - std::vector path_1 = get_path(node); - std::vector path_2 = get_path(path); + for ( const auto& path : all_paths ) { + std::vector< branching_node* > path_1 = get_path( node ); + std::vector< branching_node* > path_2 = get_path( path ); - ASSUMPTION(path_1.back() == path_2.back()); + ASSUMPTION( path_1.back() == path_2.back() ); std::size_t i_1 = path_1.size() - 1; std::size_t i_2 = path_2.size() - 1; - while (i_1 > 0 && i_2 > 0 && path_1[i_1] == path_2[i_2]) - { + while ( i_1 > 0 && i_2 > 0 && path_1[ i_1 ] == path_2[ i_2 ] ) { --i_1; --i_2; } - add_to_interesting(path_1, i_1); - add_to_interesting(path_2, i_2); + add_to_interesting( path_1, i_1 ); + add_to_interesting( path_2, i_2 ); } return set_changed; @@ -1098,23 +1093,23 @@ bool fuzzing::fuzzer::iid_dependence_props::update_interesting_nodes(branching_n /** * @brief Recomputes the matrix based on the current paths. - * + * * This function clears the existing matrix and then iterates over all paths, - * adding an equation for each path to the matrix. If there are no paths, + * adding an equation for each path to the matrix. If there are no paths, * the function returns immediately without modifying the matrix. */ void fuzzing::fuzzer::iid_dependence_props::recompute_matrix() { TMPROF_BLOCK(); - if (all_paths.empty()) + if ( all_paths.empty() ) return; matrix.clear(); // This could be done better, but for now it's fine + best_values.clear(); - for (const auto &path : all_paths) - { - add_equation(path); + for ( const auto& path : all_paths ) { + add_equation( path ); } } @@ -1127,37 +1122,35 @@ void fuzzing::fuzzer::iid_dependence_props::recompute_matrix() * * @param path A pointer to the branching node representing the path to be analyzed. */ -void fuzzing::fuzzer::iid_dependence_props::add_equation(branching_node *path) +void fuzzing::fuzzer::iid_dependence_props::add_equation( branching_node* path ) { TMPROF_BLOCK(); - std::map directions_in_path; - for (const node_navigation& navigation : interesting_nodes) - { - directions_in_path[navigation] = 0; + std::map< node_navigation, int > directions_in_path; + for ( const node_navigation& navigation : interesting_nodes ) { + directions_in_path[ navigation ] = 0; } - branching_node *node = path; - branching_node *prev_node = node->predecessor; + branching_node* node = path; + branching_node* prev_node = node->predecessor; - while (prev_node != nullptr) - { - node_navigation nav = {prev_node->get_location_id(), prev_node->successor(true).pointer == node}; - if (interesting_nodes.contains(nav)) - directions_in_path[nav]++; + while ( prev_node != nullptr ) { + node_navigation nav = { prev_node->get_location_id(), + prev_node->successor( true ).pointer == node }; + if ( interesting_nodes.contains( nav ) ) + directions_in_path[ nav ]++; node = prev_node; prev_node = node->predecessor; } - std::vector values_in_path; - for (const auto &[direction, count] : directions_in_path) - { - values_in_path.push_back(count); + std::vector< float > values_in_path; + for ( const auto& [ direction, count ] : directions_in_path ) { + values_in_path.push_back( count ); } - matrix.push_back(values_in_path); - best_values.push_back(node->best_coverage_value); + matrix.push_back( values_in_path ); + best_values.push_back( node->best_coverage_value ); } @@ -1174,7 +1167,8 @@ void fuzzing::fuzzer::iid_dependence_props::add_equation(branching_node *path) * The function uses the following parameters: * - learning_rate: The rate at which the weights are updated during each iteration. * - max_iterations: The maximum number of iterations for the gradient descent algorithm. - * - tolerance: The threshold for convergence. If the maximum change in weights is less than this value, the algorithm stops. + * - tolerance: The threshold for convergence. If the maximum change in weights is less than this + * value, the algorithm stops. * * The function performs the following steps: * 1. Initializes the weights to 1.0. @@ -1182,52 +1176,85 @@ void fuzzing::fuzzer::iid_dependence_props::add_equation(branching_node *path) * 3. Updates the weights using the computed gradient and the learning rate. * 4. Checks for convergence by comparing the maximum change in weights to the tolerance. */ -std::vector fuzzing::fuzzer::iid_dependence_props::approximate_matrix() const +std::vector< float > fuzzing::fuzzer::iid_dependence_props::approximate_matrix() const { TMPROF_BLOCK(); - const float learning_rate = 0.001f; - const int max_iterations = 1000; + const float initial_learning_rate = 0.01f; + const int max_iterations = 10000; const float tolerance = 1e-6f; + const float momentum = 0.9f; + + float max_value = 0.0f; + float min_value = 0.0f; + for ( const auto& row : matrix ) { + for ( const auto& value : row ) { + max_value = std::max( max_value, value ); + min_value = std::min( min_value, value ); + } + } + auto normalize = [ min_value, max_value ]( float value ) { + return ( value - min_value ) / ( max_value - min_value ); + }; - size_t m = matrix.size(); - size_t n = matrix[0].size(); + std::vector< std::vector< float > > normalized_matrix; + for ( const auto& row : matrix ) { + std::vector< float > normalized_row; + for ( const auto& value : row ) { + normalized_row.push_back( normalize( value ) ); + } + normalized_matrix.push_back( normalized_row ); + } - std::vector weights(n, 1.0f); // Initialize weights to zero + size_t m = normalized_matrix.size(); + size_t n = normalized_matrix[ 0 ].size(); - for (int iter = 0; iter < max_iterations; ++iter) - { - std::vector gradient(n, 0.0f); + // Initialize weights with small random values + std::vector< float > weights( n ); + std::random_device rd; + std::mt19937 gen( rd() ); + std::normal_distribution<> d( 0, 1.0 / std::sqrt( n ) ); + for ( auto& weight : weights ) { + weight = d( gen ); + } + + std::vector< float > velocity( n, 0.0f ); + float adaptive_learning_rate = initial_learning_rate; + + for ( int iter = 0; iter < max_iterations; ++iter ) { + std::vector< float > gradient( n, 0.0f ); + float total_error = 0.0f; // Compute gradient - for (size_t i = 0; i < m; ++i) - { + for ( size_t i = 0; i < m; ++i ) { float dot_product = 0.0f; - for (size_t j = 0; j < n; ++j) - { - dot_product += matrix[i][j] * weights[j]; + for ( size_t j = 0; j < n; ++j ) { + dot_product += normalized_matrix[ i ][ j ] * weights[ j ]; } - float error = dot_product - best_values[i]; - for (size_t j = 0; j < n; ++j) - { - gradient[j] += (matrix[i][j] * error) / m; + float error = dot_product - best_values[ i ]; + total_error += error * error; + for ( size_t j = 0; j < n; ++j ) { + gradient[ j ] += ( normalized_matrix[ i ][ j ] * error ) / m; } } - // Update weights - for (size_t j = 0; j < n; ++j) - { - weights[j] -= learning_rate * gradient[j]; + // Update weights with momentum + for ( size_t j = 0; j < n; ++j ) { + velocity[ j ] = momentum * velocity[ j ] - adaptive_learning_rate * gradient[ j ]; + weights[ j ] += velocity[ j ]; } - // Check for convergence - float max_change = 0.0f; - for (size_t j = 0; j < n; ++j) - { - max_change = std::max(max_change, std::abs(learning_rate * gradient[j])); + // Decay learning rate + adaptive_learning_rate *= 0.99f; + + // Print debug information + if ( iter % 100 == 0 ) { + std::cout << "Iteration " << iter << ", Error: " << total_error / m << std::endl; } - if (max_change < tolerance) - { + + // Check for convergence + if ( total_error / m < tolerance ) { + std::cout << "Converged after " << iter << " iterations." << std::endl; break; } } @@ -1272,7 +1299,7 @@ execution_record::execution_flags fuzzer::process_execution_results() ); construction_props.diverging_node = entry_branching; - process_node_dependance(entry_branching); + process_node_dependence(entry_branching); ++statistics.nodes_created; } @@ -1357,7 +1384,7 @@ execution_record::execution_flags fuzzer::process_execution_results() new_node }); - process_node_dependance(new_node); + process_node_dependence(new_node); ++statistics.nodes_created; @@ -1864,18 +1891,20 @@ branching_node* fuzzer::select_iid_coverage_target() const INVARIANT(winner != nullptr); - const iid_dependence_props& props = iid_dependences.id_to_equation_map.at(winner->get_location_id()); - std::vector weights = props.approximate_matrix(); + instrumentation::location_id loc_id(7); + const iid_dependence_props& props = iid_dependences.id_to_equation_map.at(loc_id); + if (props.matrix.empty()) + return winner; + + // std::vector weights = props.approximate_matrix(); + GradientDescent gd; + std::vector weights = gd.optimize(props.matrix, props.best_values); std::cout << "ID: " << winner->get_location_id().id << std::endl; - for (const auto& nav : props.interesting_nodes) { - std::cout << nav << " "; - } - std::cout << std::endl; - for (const auto& weight : weights) { - std::cout << weight << " "; + + for (size_t i = 0; i < props.interesting_nodes.size() && i < weights.size(); ++i) { + std::cout << *std::next(props.interesting_nodes.begin(), i) << " : " << weights[i] << std::endl; } - std::cout << std::endl; return winner; } diff --git a/src/fuzzing/src/gradient_descent.cpp b/src/fuzzing/src/gradient_descent.cpp new file mode 100644 index 00000000..cd0d01ba --- /dev/null +++ b/src/fuzzing/src/gradient_descent.cpp @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +GradientDescent::GradientDescent( float learning_rate, int max_iterations, float convergence_threshold ) + : _learning_rate( learning_rate ) + , _max_iterations( max_iterations ) + , _convergence_threshold( convergence_threshold ) +{} + +std::vector< float > GradientDescent::optimize( const std::vector< std::vector< float > >& coefficient_matrix, + const std::vector< float >& target_vector ) +{ + TMPROF_BLOCK(); + + if ( coefficient_matrix.empty() || target_vector.empty() || + coefficient_matrix.size() != target_vector.size() ) { + throw std::invalid_argument( "Invalid input dimensions" ); + } + + std::vector< float > current_solution = generateRandomWeights( coefficient_matrix[ 0 ].size() ); + + std::vector< float > gradient; + float prev_cost = std::numeric_limits< float >::max(); + + for ( int iteration = 0; iteration < _max_iterations; ++iteration ) { + gradient = computeGradient( current_solution, coefficient_matrix, target_vector ); + + // Update current_solution + for ( size_t i = 0; i < current_solution.size(); ++i ) { + current_solution[ i ] -= _learning_rate * gradient[ i ]; + } + + // Check for convergence + float current_cost = + computeMeanSquaredError( current_solution, coefficient_matrix, target_vector ); + + // Debug output + if ( iteration % 25 == 0 ) { + std::cout << "Iteration " << iteration << ", Cost: " << current_cost << std::endl; + } + + if ( std::abs( current_cost - prev_cost ) < _convergence_threshold ) { + std::cout << "Converged after " << iteration << " iterations." << std::endl; + break; + } + prev_cost = current_cost; + } + + return current_solution; +} + +std::vector< float > GradientDescent::computeGradient( const std::vector< float >& current_solution, + const std::vector< std::vector< float > >& coefficient_matrix, + const std::vector< float >& target_vector ) +{ + std::vector< float > gradient( current_solution.size(), 0.0f ); + + for ( size_t i = 0; i < coefficient_matrix.size(); ++i ) { + float predicted = dotProduct( current_solution, coefficient_matrix[ i ] ); + float error = predicted - target_vector[ i ]; + + for ( size_t j = 0; j < current_solution.size(); ++j ) { + gradient[ j ] += 2 * error * coefficient_matrix[ i ][ j ]; + } + } + + for ( float& grad : gradient ) { + grad /= coefficient_matrix.size(); + } + + return gradient; +} + +float GradientDescent::computeMeanSquaredError( const std::vector< float >& current_solution, + const std::vector< std::vector< float > >& coefficient_matrix, + const std::vector< float >& target_vector ) +{ + float mse = 0.0f; + + for ( size_t i = 0; i < coefficient_matrix.size(); ++i ) { + float predicted = dotProduct( current_solution, coefficient_matrix[ i ] ); + float error = predicted - target_vector[ i ]; + mse += error * error; + } + + return mse / coefficient_matrix.size(); +} + +float GradientDescent::dotProduct( const std::vector< float >& a, const std::vector< float >& b ) +{ + INVARIANT( a.size() == b.size() ); + + float result = 0.0f; + for ( size_t i = 0; i < a.size(); ++i ) { + result += a[ i ] * b[ i ]; + } + return result; +} + +std::vector< float > GradientDescent::generateRandomWeights( size_t n ) +{ + std::vector< float > weights( n ); + std::random_device rd; + std::mt19937 gen( rd() ); + std::normal_distribution<> d( 0, 1.0 / std::sqrt( n ) ); + for ( auto& weight : weights ) { + weight = d( gen ); + } + + return weights; +} From 0dd48b30e526514b4e9fa4be7fd1279ba004eb4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Tue, 24 Sep 2024 16:31:51 +0200 Subject: [PATCH 029/144] fix: changed add_equation to update best_values correctly --- src/fuzzing/include/fuzzing/fuzzer.hpp | 2 + .../include/fuzzing/gradient_descent.hpp | 3 +- src/fuzzing/src/fuzzer.cpp | 217 ++++++------------ src/fuzzing/src/gradient_descent.cpp | 24 +- 4 files changed, 96 insertions(+), 150 deletions(-) diff --git a/src/fuzzing/include/fuzzing/fuzzer.hpp b/src/fuzzing/include/fuzzing/fuzzer.hpp index 45559218..767ce35f 100644 --- a/src/fuzzing/include/fuzzing/fuzzer.hpp +++ b/src/fuzzing/include/fuzzing/fuzzer.hpp @@ -224,6 +224,7 @@ struct fuzzer final bool direction; auto operator<=>(node_navigation const& other) const; + bool operator==(node_navigation const& other) const; friend std::ostream& operator<<(std::ostream& os, node_navigation const& nn) { os << nn.node_id.id << " " << (nn.direction ? "right" : "left"); @@ -239,6 +240,7 @@ struct fuzzer final std::vector best_values; bool update_interesting_nodes(branching_node* node); + std::vector get_path(branching_node* node); void recompute_matrix(); void add_equation(branching_node* path); std::vector approximate_matrix() const; diff --git a/src/fuzzing/include/fuzzing/gradient_descent.hpp b/src/fuzzing/include/fuzzing/gradient_descent.hpp index e7551719..ce271b66 100644 --- a/src/fuzzing/include/fuzzing/gradient_descent.hpp +++ b/src/fuzzing/include/fuzzing/gradient_descent.hpp @@ -19,10 +19,11 @@ class GradientDescent { float _learning_rate; int _max_iterations; float _convergence_threshold; + bool _debug = false; std::vector computeGradient(const std::vector& current_solution, const std::vector>& coefficient_matrix, const std::vector& target_vector); float computeMeanSquaredError(const std::vector& current_solution, const std::vector>& coefficient_matrix, const std::vector& target_vector); std::vector generateRandomWeights(size_t n); float dotProduct(const std::vector& a, const std::vector& b); - + std::vector normalize(std::vector& values, float min_value, float max_value); }; diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index eba7e7b9..e75438ef 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -995,6 +995,11 @@ auto fuzzer::node_navigation::operator<=>( node_navigation const& other ) const return direction <=> other.direction; } +bool fuzzer::node_navigation::operator==( node_navigation const& other ) const +{ + return node_id.id == other.node_id.id && direction == other.direction; +} + /** * @brief Processes the dependencies of a branching node. @@ -1036,42 +1041,27 @@ void fuzzer::process_node_dependence( branching_node* node ) * @param node A pointer to the branching node from which to start the path traversal. * @return true if the set of interesting nodes was updated, false otherwise. */ -bool fuzzing::fuzzer::iid_dependence_props::update_interesting_nodes( branching_node* node ) +bool fuzzer::iid_dependence_props::update_interesting_nodes( branching_node* node ) { TMPROF_BLOCK(); bool set_changed = false; - auto get_path = []( branching_node* node ) { - std::vector< branching_node* > path; - branching_node* current = node; - while ( current != nullptr ) { - path.push_back( current ); - current = current->predecessor; - } - return path; - }; - - auto add_to_interesting = [ this, &set_changed ]( std::vector< branching_node* >& nodes, int i ) { + auto add_to_interesting = [ this, &set_changed ]( std::vector< node_navigation >& nodes, int i ) { for ( ; i >= 0; --i ) { - branching_node* node = nodes[ i ]; - branching_node* predecessor = node->predecessor; - - if ( predecessor == nullptr ) - continue; - - bool direction = node == predecessor->successor( true ).pointer; - auto result = this->interesting_nodes.emplace( node->get_location_id(), !direction ); - result = this->interesting_nodes.emplace( node->get_location_id(), direction ); + auto result = this->interesting_nodes.emplace( nodes[ i ] ); if ( result.second ) { set_changed = true; } } }; - for ( const auto& path : all_paths ) { - std::vector< branching_node* > path_1 = get_path( node ); - std::vector< branching_node* > path_2 = get_path( path ); + for ( const auto& end_node : all_paths ) { + std::vector< node_navigation > path_1 = get_path( node ); + std::vector< node_navigation > path_2 = get_path( end_node ); + + if ( path_1.empty() || path_2.empty() ) + continue; ASSUMPTION( path_1.back() == path_2.back() ); @@ -1090,6 +1080,22 @@ bool fuzzing::fuzzer::iid_dependence_props::update_interesting_nodes( branching_ return set_changed; } +std::vector< fuzzer::node_navigation > fuzzer::iid_dependence_props::get_path( branching_node* node ) +{ + std::vector< node_navigation > path; + + branching_node* current = node; + while ( current != nullptr ) { + branching_node* predecessor = current->predecessor; + if ( predecessor != nullptr ) { + node_navigation nav = { predecessor->get_location_id(), predecessor->successor_direction( current ) }; + path.push_back( nav ); + } + current = predecessor; + } + + return path; +} /** * @brief Recomputes the matrix based on the current paths. @@ -1098,7 +1104,7 @@ bool fuzzing::fuzzer::iid_dependence_props::update_interesting_nodes( branching_ * adding an equation for each path to the matrix. If there are no paths, * the function returns immediately without modifying the matrix. */ -void fuzzing::fuzzer::iid_dependence_props::recompute_matrix() +void fuzzer::iid_dependence_props::recompute_matrix() { TMPROF_BLOCK(); @@ -1122,7 +1128,7 @@ void fuzzing::fuzzer::iid_dependence_props::recompute_matrix() * * @param path A pointer to the branching node representing the path to be analyzed. */ -void fuzzing::fuzzer::iid_dependence_props::add_equation( branching_node* path ) +void fuzzer::iid_dependence_props::add_equation( branching_node* path ) { TMPROF_BLOCK(); @@ -1131,17 +1137,12 @@ void fuzzing::fuzzer::iid_dependence_props::add_equation( branching_node* path ) directions_in_path[ navigation ] = 0; } - branching_node* node = path; - branching_node* prev_node = node->predecessor; + std::vector< node_navigation > path_nodes = get_path( path ); - while ( prev_node != nullptr ) { - node_navigation nav = { prev_node->get_location_id(), - prev_node->successor( true ).pointer == node }; - if ( interesting_nodes.contains( nav ) ) + for ( const node_navigation& nav : path_nodes ) { + if ( interesting_nodes.contains( nav ) ) { directions_in_path[ nav ]++; - - node = prev_node; - prev_node = node->predecessor; + } } std::vector< float > values_in_path; @@ -1150,118 +1151,25 @@ void fuzzing::fuzzer::iid_dependence_props::add_equation( branching_node* path ) } matrix.push_back( values_in_path ); - best_values.push_back( node->best_coverage_value ); + best_values.push_back( path->best_coverage_value ); } /** - * @brief Approximates the weight matrix using gradient descent. - * - * This function performs gradient descent to approximate the weight matrix - * that minimizes the error between the dot product of the input matrix and - * the best values. The gradient descent algorithm iteratively updates the - * weights until convergence or until the maximum number of iterations is reached. - * - * @return A vector of floats representing the approximated weights. + * @brief Approximates a matrix using gradient descent optimization. * - * The function uses the following parameters: - * - learning_rate: The rate at which the weights are updated during each iteration. - * - max_iterations: The maximum number of iterations for the gradient descent algorithm. - * - tolerance: The threshold for convergence. If the maximum change in weights is less than this - * value, the algorithm stops. + * This function utilizes the GradientDescent class to optimize the given matrix + * and best values, returning a vector of optimized weights. * - * The function performs the following steps: - * 1. Initializes the weights to 1.0. - * 2. Iteratively computes the gradient of the error with respect to the weights. - * 3. Updates the weights using the computed gradient and the learning rate. - * 4. Checks for convergence by comparing the maximum change in weights to the tolerance. + * @return A vector of floats representing the optimized weights. */ -std::vector< float > fuzzing::fuzzer::iid_dependence_props::approximate_matrix() const +std::vector< float > fuzzer::iid_dependence_props::approximate_matrix() const { - TMPROF_BLOCK(); - - const float initial_learning_rate = 0.01f; - const int max_iterations = 10000; - const float tolerance = 1e-6f; - const float momentum = 0.9f; - - float max_value = 0.0f; - float min_value = 0.0f; - for ( const auto& row : matrix ) { - for ( const auto& value : row ) { - max_value = std::max( max_value, value ); - min_value = std::min( min_value, value ); - } - } - auto normalize = [ min_value, max_value ]( float value ) { - return ( value - min_value ) / ( max_value - min_value ); - }; - - std::vector< std::vector< float > > normalized_matrix; - for ( const auto& row : matrix ) { - std::vector< float > normalized_row; - for ( const auto& value : row ) { - normalized_row.push_back( normalize( value ) ); - } - normalized_matrix.push_back( normalized_row ); - } - - size_t m = normalized_matrix.size(); - size_t n = normalized_matrix[ 0 ].size(); - - // Initialize weights with small random values - std::vector< float > weights( n ); - std::random_device rd; - std::mt19937 gen( rd() ); - std::normal_distribution<> d( 0, 1.0 / std::sqrt( n ) ); - for ( auto& weight : weights ) { - weight = d( gen ); - } - - std::vector< float > velocity( n, 0.0f ); - float adaptive_learning_rate = initial_learning_rate; - - for ( int iter = 0; iter < max_iterations; ++iter ) { - std::vector< float > gradient( n, 0.0f ); - float total_error = 0.0f; - - // Compute gradient - for ( size_t i = 0; i < m; ++i ) { - float dot_product = 0.0f; - for ( size_t j = 0; j < n; ++j ) { - dot_product += normalized_matrix[ i ][ j ] * weights[ j ]; - } - float error = dot_product - best_values[ i ]; - total_error += error * error; - for ( size_t j = 0; j < n; ++j ) { - gradient[ j ] += ( normalized_matrix[ i ][ j ] * error ) / m; - } - } - - // Update weights with momentum - for ( size_t j = 0; j < n; ++j ) { - velocity[ j ] = momentum * velocity[ j ] - adaptive_learning_rate * gradient[ j ]; - weights[ j ] += velocity[ j ]; - } - - // Decay learning rate - adaptive_learning_rate *= 0.99f; - - // Print debug information - if ( iter % 100 == 0 ) { - std::cout << "Iteration " << iter << ", Error: " << total_error / m << std::endl; - } - - // Check for convergence - if ( total_error / m < tolerance ) { - std::cout << "Converged after " << iter << " iterations." << std::endl; - break; - } - } - - return weights; + GradientDescent gd; + std::vector weights = gd.optimize(matrix, best_values); } + execution_record::execution_flags fuzzer::process_execution_results() { TMPROF_BLOCK(); @@ -1751,6 +1659,19 @@ void fuzzer::collect_iid_pivots_from_sensitivity_results() void fuzzer::select_next_state() { + instrumentation::location_id loc_id(7); + if (iid_dependences.id_to_equation_map.contains(loc_id)) + { + const iid_dependence_props& props = iid_dependences.id_to_equation_map.at(loc_id); + + std::vector weights = props.approximate_matrix(); + + std::cout << "ID: " << loc_id.id << std::endl; + for (size_t i = 0; i < props.interesting_nodes.size() && i < weights.size(); ++i) { + std::cout << *std::next(props.interesting_nodes.begin(), i) << " : " << weights[i] << std::endl; + } + } + TMPROF_BLOCK(); INVARIANT(sensitivity.is_ready() && typed_minimization.is_ready() && minimization.is_ready() && bitshare.is_ready()); @@ -1891,20 +1812,20 @@ branching_node* fuzzer::select_iid_coverage_target() const INVARIANT(winner != nullptr); - instrumentation::location_id loc_id(7); - const iid_dependence_props& props = iid_dependences.id_to_equation_map.at(loc_id); - if (props.matrix.empty()) - return winner; + // instrumentation::location_id loc_id(6); + // const iid_dependence_props& props = iid_dependences.id_to_equation_map.at(loc_id); + // if (props.matrix.empty()) + // return winner; - // std::vector weights = props.approximate_matrix(); - GradientDescent gd; - std::vector weights = gd.optimize(props.matrix, props.best_values); + // // std::vector weights = props.approximate_matrix(); + // GradientDescent gd; + // std::vector weights = gd.optimize(props.matrix, props.best_values); - std::cout << "ID: " << winner->get_location_id().id << std::endl; + // std::cout << "ID: " << loc_id.id << std::endl; - for (size_t i = 0; i < props.interesting_nodes.size() && i < weights.size(); ++i) { - std::cout << *std::next(props.interesting_nodes.begin(), i) << " : " << weights[i] << std::endl; - } + // for (size_t i = 0; i < props.interesting_nodes.size() && i < weights.size(); ++i) { + // std::cout << *std::next(props.interesting_nodes.begin(), i) << " : " << weights[i] << std::endl; + // } return winner; } diff --git a/src/fuzzing/src/gradient_descent.cpp b/src/fuzzing/src/gradient_descent.cpp index cd0d01ba..bc0dd2b5 100644 --- a/src/fuzzing/src/gradient_descent.cpp +++ b/src/fuzzing/src/gradient_descent.cpp @@ -41,17 +41,19 @@ std::vector< float > GradientDescent::optimize( const std::vector< std::vector< computeMeanSquaredError( current_solution, coefficient_matrix, target_vector ); // Debug output - if ( iteration % 25 == 0 ) { + if ( _debug && iteration % 50 == 0 ) { std::cout << "Iteration " << iteration << ", Cost: " << current_cost << std::endl; } if ( std::abs( current_cost - prev_cost ) < _convergence_threshold ) { std::cout << "Converged after " << iteration << " iterations." << std::endl; + std::cout << "Number of equations: " << coefficient_matrix.size() << std::endl; break; } prev_cost = current_cost; } + normalize( current_solution, 0.0f, 100.0f ); return current_solution; } @@ -115,3 +117,23 @@ std::vector< float > GradientDescent::generateRandomWeights( size_t n ) return weights; } + +std::vector< float > GradientDescent::normalize( std::vector< float >& values, + float min_value, + float max_value ) +{ + if (values.empty()) { + return values; + } + + float min_elem = *std::min_element(values.begin(), values.end()); + float max_elem = *std::max_element(values.begin(), values.end()); + + if (min_elem != max_elem) { + for (auto& value : values) { + value = min_value + (value - min_elem) * (max_value - min_value) / (max_elem - min_elem); + } + } + + return values; +} \ No newline at end of file From 4b7c8884075f202d95f535dec4633a3356e72a5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Tue, 24 Sep 2024 16:40:22 +0200 Subject: [PATCH 030/144] chore: formatting and docstring --- .../include/fuzzing/gradient_descent.hpp | 32 ++++++++++++------- src/fuzzing/src/fuzzer.cpp | 13 ++++++-- src/fuzzing/src/gradient_descent.cpp | 19 +++++------ 3 files changed, 42 insertions(+), 22 deletions(-) diff --git a/src/fuzzing/include/fuzzing/gradient_descent.hpp b/src/fuzzing/include/fuzzing/gradient_descent.hpp index ce271b66..85ec1c05 100644 --- a/src/fuzzing/include/fuzzing/gradient_descent.hpp +++ b/src/fuzzing/include/fuzzing/gradient_descent.hpp @@ -1,18 +1,24 @@ #pragma once -#include #include +#include class GradientDescent { public: - GradientDescent(float learning_rate = 0.001f, int max_iterations = 10000, float convergence_threshold = 1e-6); + GradientDescent( float learning_rate = 0.001f, + int max_iterations = 10000, + float convergence_threshold = 1e-6 ); - std::vector optimize(const std::vector>& coefficient_matrix, const std::vector& target_vector); + std::vector< float > optimize( const std::vector< std::vector< float > >& coefficient_matrix, + const std::vector< float >& target_vector ); // Setters for easy modification of parameters - void setLearningRate(float learning_rate) { _learning_rate = learning_rate; } - void setMaxIterations(int max_iterations) { _max_iterations = max_iterations; }; - void setConvergenceThreshold(float convergence_threshold) { _convergence_threshold = convergence_threshold; }; + void setLearningRate( float learning_rate ) { _learning_rate = learning_rate; } + void setMaxIterations( int max_iterations ) { _max_iterations = max_iterations; }; + void setConvergenceThreshold( float convergence_threshold ) + { + _convergence_threshold = convergence_threshold; + }; private: @@ -21,9 +27,13 @@ class GradientDescent { float _convergence_threshold; bool _debug = false; - std::vector computeGradient(const std::vector& current_solution, const std::vector>& coefficient_matrix, const std::vector& target_vector); - float computeMeanSquaredError(const std::vector& current_solution, const std::vector>& coefficient_matrix, const std::vector& target_vector); - std::vector generateRandomWeights(size_t n); - float dotProduct(const std::vector& a, const std::vector& b); - std::vector normalize(std::vector& values, float min_value, float max_value); + std::vector< float > computeGradient( const std::vector< float >& current_solution, + const std::vector< std::vector< float > >& coefficient_matrix, + const std::vector< float >& target_vector ); + float computeMeanSquaredError( const std::vector< float >& current_solution, + const std::vector< std::vector< float > >& coefficient_matrix, + const std::vector< float >& target_vector ); + std::vector< float > generateRandomWeights( size_t n ); + float dotProduct( const std::vector< float >& a, const std::vector< float >& b ); + std::vector< float > normalize( std::vector< float >& values, float min_value, float max_value ); }; diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index e75438ef..74dce9fd 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -995,6 +995,14 @@ auto fuzzer::node_navigation::operator<=>( node_navigation const& other ) const return direction <=> other.direction; } +/** + * @brief Equality operator for node_navigation. + * + * Compares two node_navigation objects for equality based on their node_id and direction. + * + * @param other The other node_navigation object to compare with. + * @return true if both node_id and direction are equal, false otherwise. + */ bool fuzzer::node_navigation::operator==( node_navigation const& other ) const { return node_id.id == other.node_id.id && direction == other.direction; @@ -1088,7 +1096,8 @@ std::vector< fuzzer::node_navigation > fuzzer::iid_dependence_props::get_path( b while ( current != nullptr ) { branching_node* predecessor = current->predecessor; if ( predecessor != nullptr ) { - node_navigation nav = { predecessor->get_location_id(), predecessor->successor_direction( current ) }; + node_navigation nav = { predecessor->get_location_id(), + predecessor->successor_direction( current ) }; path.push_back( nav ); } current = predecessor; @@ -1166,7 +1175,7 @@ void fuzzer::iid_dependence_props::add_equation( branching_node* path ) std::vector< float > fuzzer::iid_dependence_props::approximate_matrix() const { GradientDescent gd; - std::vector weights = gd.optimize(matrix, best_values); + std::vector< float > weights = gd.optimize( matrix, best_values ); } diff --git a/src/fuzzing/src/gradient_descent.cpp b/src/fuzzing/src/gradient_descent.cpp index bc0dd2b5..6f244857 100644 --- a/src/fuzzing/src/gradient_descent.cpp +++ b/src/fuzzing/src/gradient_descent.cpp @@ -1,11 +1,11 @@ #include #include #include -#include -#include #include -#include #include +#include +#include +#include GradientDescent::GradientDescent( float learning_rate, int max_iterations, float convergence_threshold ) : _learning_rate( learning_rate ) @@ -122,16 +122,17 @@ std::vector< float > GradientDescent::normalize( std::vector< float >& values, float min_value, float max_value ) { - if (values.empty()) { + if ( values.empty() ) { return values; } - float min_elem = *std::min_element(values.begin(), values.end()); - float max_elem = *std::max_element(values.begin(), values.end()); + float min_elem = *std::min_element( values.begin(), values.end() ); + float max_elem = *std::max_element( values.begin(), values.end() ); - if (min_elem != max_elem) { - for (auto& value : values) { - value = min_value + (value - min_elem) * (max_value - min_value) / (max_elem - min_elem); + if ( min_elem != max_elem ) { + for ( auto& value : values ) { + value = min_value + + ( value - min_elem ) * ( max_value - min_value ) / ( max_elem - min_elem ); } } From 1f554f0ea95b81512fab8a3a0d30c268c68d7e98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Wed, 25 Sep 2024 22:29:03 +0200 Subject: [PATCH 031/144] feat: compute iid_dependance only for iid_node --- src/fuzzing/include/fuzzing/fuzzer.hpp | 7 +- src/fuzzing/src/fuzzer.cpp | 203 +++++++++++++++++-------- 2 files changed, 145 insertions(+), 65 deletions(-) diff --git a/src/fuzzing/include/fuzzing/fuzzer.hpp b/src/fuzzing/include/fuzzing/fuzzer.hpp index 767ce35f..5be0c6c4 100644 --- a/src/fuzzing/include/fuzzing/fuzzer.hpp +++ b/src/fuzzing/include/fuzzing/fuzzer.hpp @@ -240,16 +240,20 @@ struct fuzzer final std::vector best_values; bool update_interesting_nodes(branching_node* node); - std::vector get_path(branching_node* node); + std::vector get_path(branching_node* node) const; void recompute_matrix(); void add_equation(branching_node* path); std::vector approximate_matrix() const; + std::vector> weights_to_path(std::vector const& weights) const; }; struct iid_node_dependence { std::unordered_map id_to_equation_map; std::set non_iid_nodes; + + void update_non_iid_nodes(sensitivity_analysis& sensitivity); + void process_node_dependence(branching_node* node); }; static void update_close_flags_from(branching_node* node); @@ -320,7 +324,6 @@ struct fuzzer final ); void generate_next_input(vecb& stdin_bits); - void process_node_dependence(branching_node* node); execution_record::execution_flags process_execution_results(); void do_cleanup(); diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index 74dce9fd..d8dd2178 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -974,20 +974,14 @@ void fuzzer::generate_next_input(vecb& stdin_bits) UNREACHABLE(); } + /** - * @brief Compares this node_navigation object with another node_navigation object. - * - * This operator performs a three-way comparison between the current node_navigation - * object and another node_navigation object. The comparison is first based on the - * node_id.id. If the node_id.id values are equal, it then compares the direction. + * @brief Three-way comparison for node_navigation. * * @param other The other node_navigation object to compare with. - * @return std::strong_ordering Result of the comparison: - * - std::strong_ordering::less if this object is less than the other. - * - std::strong_ordering::equal if this object is equal to the other. - * - std::strong_ordering::greater if this object is greater than the other. + * @return std::strong_ordering Result of the comparison. */ -auto fuzzer::node_navigation::operator<=>( node_navigation const& other ) const +auto fuzzing::fuzzer::node_navigation::operator<=>( node_navigation const& other ) const { if ( auto const cmp = node_id.id <=> other.node_id.id; cmp != 0 ) return cmp; @@ -995,50 +989,21 @@ auto fuzzer::node_navigation::operator<=>( node_navigation const& other ) const return direction <=> other.direction; } + /** * @brief Equality operator for node_navigation. - * + * * Compares two node_navigation objects for equality based on their node_id and direction. - * + * * @param other The other node_navigation object to compare with. * @return true if both node_id and direction are equal, false otherwise. */ -bool fuzzer::node_navigation::operator==( node_navigation const& other ) const +bool fuzzing::fuzzer::node_navigation::operator==( node_navigation const& other ) const { return node_id.id == other.node_id.id && direction == other.direction; } -/** - * @brief Processes the dependencies of a branching node. - * - * This function checks if the given node's location ID is contained within - * the non-IID nodes. If it is, the function returns early. Otherwise, it - * retrieves the properties associated with the node's location ID and updates - * the list of all paths with the given node. If the node is deemed interesting, - * the function recomputes the matrix. Finally, it adds an equation for the node. - * - * @param node A pointer to the branching node whose dependencies are to be processed. - */ -void fuzzer::process_node_dependence( branching_node* node ) -{ - TMPROF_BLOCK(); - - if ( iid_dependences.non_iid_nodes.contains( node->get_location_id() ) ) - return; - - iid_dependence_props& props = iid_dependences.id_to_equation_map[ node->get_location_id() ]; - - props.all_paths.push_back( node ); - - if ( props.update_interesting_nodes( node ) ) { - props.recompute_matrix(); - } - - props.add_equation( node ); -} - - /** * @brief Updates the set of interesting nodes based on the given branching node. * @@ -1049,7 +1014,7 @@ void fuzzer::process_node_dependence( branching_node* node ) * @param node A pointer to the branching node from which to start the path traversal. * @return true if the set of interesting nodes was updated, false otherwise. */ -bool fuzzer::iid_dependence_props::update_interesting_nodes( branching_node* node ) +bool fuzzing::fuzzer::iid_dependence_props::update_interesting_nodes( branching_node* node ) { TMPROF_BLOCK(); @@ -1088,7 +1053,20 @@ bool fuzzer::iid_dependence_props::update_interesting_nodes( branching_node* nod return set_changed; } -std::vector< fuzzer::node_navigation > fuzzer::iid_dependence_props::get_path( branching_node* node ) + +/** + * @brief Retrieves the path of node navigations from the given branching node to the root. + * + * This function constructs a vector of `node_navigation` objects representing the path + * from the specified branching node to the root node. Each `node_navigation` object + * contains the location ID of the predecessor node and the direction to the current node. + * + * @param node A pointer to the starting branching node. + * @return A vector of `node_navigation` objects representing the path from the given node to the + * root. + */ +std::vector< fuzzer::node_navigation > +fuzzing::fuzzer::iid_dependence_props::get_path( branching_node* node ) const { std::vector< node_navigation > path; @@ -1106,14 +1084,17 @@ std::vector< fuzzer::node_navigation > fuzzer::iid_dependence_props::get_path( b return path; } + /** - * @brief Recomputes the matrix based on the current paths. + * @brief Recomputes the matrix of IID dependence properties. + * + * This function clears the current matrix and best values, then iterates + * through all paths to add equations to the matrix. If there are no paths, + * the function returns immediately. * - * This function clears the existing matrix and then iterates over all paths, - * adding an equation for each path to the matrix. If there are no paths, - * the function returns immediately without modifying the matrix. + * @note The matrix clearing could be optimized in the future. */ -void fuzzer::iid_dependence_props::recompute_matrix() +void fuzzing::fuzzer::iid_dependence_props::recompute_matrix() { TMPROF_BLOCK(); @@ -1132,12 +1113,12 @@ void fuzzer::iid_dependence_props::recompute_matrix() /** * @brief Adds an equation to the IID dependence properties based on the given branching path. * - * This function traverses the given branching path, collects the directions of interesting nodes, - * and updates the internal matrix and best values with the collected data. + * This function updates the internal matrix and best values with the direction counts of + * interesting nodes encountered in the path. * - * @param path A pointer to the branching node representing the path to be analyzed. + * @param path A pointer to the branching_node representing the path to be processed. */ -void fuzzer::iid_dependence_props::add_equation( branching_node* path ) +void fuzzing::fuzzer::iid_dependence_props::add_equation( branching_node* path ) { TMPROF_BLOCK(); @@ -1172,10 +1153,107 @@ void fuzzer::iid_dependence_props::add_equation( branching_node* path ) * * @return A vector of floats representing the optimized weights. */ -std::vector< float > fuzzer::iid_dependence_props::approximate_matrix() const +std::vector< float > fuzzing::fuzzer::iid_dependence_props::approximate_matrix() const { GradientDescent gd; std::vector< float > weights = gd.optimize( matrix, best_values ); + + + // std::vector> path = weights_to_path( weights ); + // for (const auto& [value, nav] : path) { + // std::cout << "Node ID: " << nav.node_id.id << ", Direction: " << nav.direction << ", + // Value: " << value << std::endl; + // } + + return weights; +} + + +std::vector< std::pair< int, fuzzer::node_navigation > > +fuzzing::fuzzer::iid_dependence_props::weights_to_path( std::vector< float > const& weights ) const +{ + // *std::next(props.interesting_nodes.begin(), i) + + std::vector< std::pair< float, size_t > > positive_values; + + // Step 1: Consider only positive values and store their indices + for ( size_t i = 0; i < weights.size(); ++i ) { + if ( weights[ i ] > 0 ) { + positive_values.emplace_back( weights[ i ], i ); + // std::cout << "Value: " << weights[i] << std::endl; + } + } + + // Sort the positive values + std::sort( positive_values.begin(), positive_values.end() ); + std::vector< std::pair< int, node_navigation > > result; + + if ( !positive_values.empty() ) { + float smallest = positive_values[ 0 ].first; + + // Steps 2-4: Create new values and store as integers + for ( const auto& [ value, index ] : positive_values ) { + int new_value = std::round( value / smallest ); + result.emplace_back( new_value, *std::next( interesting_nodes.begin(), index ) ); + } + } + + return result; +} + + +/** + * @brief Updates the set of non-IID nodes based on sensitivity analysis. + * + * This function iterates through the nodes that have changed according to the + * provided sensitivity analysis. For each node that has undergone branching, + * it retrieves the node's location ID and attempts to insert it into the set + * of non-IID nodes. If the insertion is successful, it also removes the + * corresponding entry from the ID-to-equation map. + * + * @param sensitivity A reference to the sensitivity analysis object that + * provides the changed nodes. + */ +void fuzzing::fuzzer::iid_node_dependence::update_non_iid_nodes( sensitivity_analysis& sensitivity ) +{ + for ( branching_node* node : sensitivity.get_changed_nodes() ) { + if ( node->is_did_branching() ) { + auto location_id = node->get_location_id(); + if ( non_iid_nodes.insert( location_id ).second ) { + id_to_equation_map.erase( location_id ); + } + } + } +} + + +/** + * @brief Processes the dependence of a given branching node. + * + * This function processes the dependence of a branching node by updating + * the internal state and properties associated with the node's location ID. + * It ensures that non-IID nodes are skipped, updates the list of all paths, + * and recomputes the matrix if the node is deemed interesting. Additionally, + * it adds the node to the equation map. + * + * @param node A pointer to the branching node to be processed. + */ +void fuzzing::fuzzer::iid_node_dependence::process_node_dependence( branching_node* node ) +{ + TMPROF_BLOCK(); + + if ( non_iid_nodes.contains( node->get_location_id() ) ) + return; + + iid_dependence_props& props = id_to_equation_map[ node->get_location_id() ]; + + props.all_paths.push_back( node ); + + if ( props.update_interesting_nodes( node ) ) { + props.recompute_matrix(); + } + + props.add_equation( node ); } @@ -1216,7 +1294,7 @@ execution_record::execution_flags fuzzer::process_execution_results() ); construction_props.diverging_node = entry_branching; - process_node_dependence(entry_branching); + iid_dependences.process_node_dependence(entry_branching); ++statistics.nodes_created; } @@ -1301,7 +1379,7 @@ execution_record::execution_flags fuzzer::process_execution_results() new_node }); - process_node_dependence(new_node); + iid_dependences.process_node_dependence(new_node); ++statistics.nodes_created; @@ -1468,6 +1546,7 @@ void fuzzer::do_cleanup() update_close_flags_from(node); break; } + iid_dependences.update_non_iid_nodes(sensitivity); collect_iid_pivots_from_sensitivity_results(); break; case BITSHARE: @@ -1533,8 +1612,6 @@ void fuzzer::collect_iid_pivots_from_sensitivity_results() pivots.push_back({ node, &pivot_it_and_state.first->second }); } - - // iid_dependence_props& deps = iid_dependences[node->get_location_id()]; } if (pivots.empty()) return; @@ -1675,10 +1752,10 @@ void fuzzer::select_next_state() std::vector weights = props.approximate_matrix(); - std::cout << "ID: " << loc_id.id << std::endl; - for (size_t i = 0; i < props.interesting_nodes.size() && i < weights.size(); ++i) { - std::cout << *std::next(props.interesting_nodes.begin(), i) << " : " << weights[i] << std::endl; - } + // std::cout << "ID: " << loc_id.id << std::endl; + // for (size_t i = 0; i < props.interesting_nodes.size() && i < weights.size(); ++i) { + // std::cout << *std::next(props.interesting_nodes.begin(), i) << " : " << weights[i] << std::endl; + // } } TMPROF_BLOCK(); From b775151ac464070c20749c4648cc7985b0326c3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Thu, 26 Sep 2024 23:07:07 +0200 Subject: [PATCH 032/144] feat: refactoring of gradient descent computation --- .../include/fuzzing/gradient_descent.hpp | 38 ++++--- src/fuzzing/src/fuzzer.cpp | 5 +- src/fuzzing/src/gradient_descent.cpp | 106 +++++++++--------- 3 files changed, 74 insertions(+), 75 deletions(-) diff --git a/src/fuzzing/include/fuzzing/gradient_descent.hpp b/src/fuzzing/include/fuzzing/gradient_descent.hpp index 85ec1c05..2d45f3a2 100644 --- a/src/fuzzing/include/fuzzing/gradient_descent.hpp +++ b/src/fuzzing/include/fuzzing/gradient_descent.hpp @@ -1,39 +1,43 @@ #pragma once -#include #include class GradientDescent { + /* + # Improvements + - Learning rate optimization(Adaptive learning rate) + - Feature scaling + - Momentum + - Mini-batch gradient descent + */ public: - GradientDescent( float learning_rate = 0.001f, + GradientDescent( const std::vector< std::vector< float > >& coefficient_matrix, + const std::vector< float >& target_vector, + float learning_rate = 0.001f, int max_iterations = 10000, float convergence_threshold = 1e-6 ); - std::vector< float > optimize( const std::vector< std::vector< float > >& coefficient_matrix, - const std::vector< float >& target_vector ); + std::vector< float > optimize(); // Setters for easy modification of parameters void setLearningRate( float learning_rate ) { _learning_rate = learning_rate; } - void setMaxIterations( int max_iterations ) { _max_iterations = max_iterations; }; + void setMaxIterations( int max_iterations ) { _max_iterations = max_iterations; } void setConvergenceThreshold( float convergence_threshold ) { _convergence_threshold = convergence_threshold; - }; - + } private: + const std::vector< std::vector< float > >& _coefficient_matrix; + const std::vector< float >& _target_vector; float _learning_rate; int _max_iterations; float _convergence_threshold; bool _debug = false; - std::vector< float > computeGradient( const std::vector< float >& current_solution, - const std::vector< std::vector< float > >& coefficient_matrix, - const std::vector< float >& target_vector ); - float computeMeanSquaredError( const std::vector< float >& current_solution, - const std::vector< std::vector< float > >& coefficient_matrix, - const std::vector< float >& target_vector ); - std::vector< float > generateRandomWeights( size_t n ); - float dotProduct( const std::vector< float >& a, const std::vector< float >& b ); - std::vector< float > normalize( std::vector< float >& values, float min_value, float max_value ); -}; + std::vector< float > computeGradient( const std::vector< float >& current_solution ); + float computeMeanSquaredError( const std::vector< float >& current_solution ); + static std::vector< float > generateRandomWeights( size_t n ); + static float dotProduct( const std::vector< float >& a, const std::vector< float >& b ); + static void rescale( std::vector< float >& values, float min_value, float max_value ); +}; \ No newline at end of file diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index d8dd2178..e6633b24 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -1155,8 +1155,8 @@ void fuzzing::fuzzer::iid_dependence_props::add_equation( branching_node* path ) */ std::vector< float > fuzzing::fuzzer::iid_dependence_props::approximate_matrix() const { - GradientDescent gd; - std::vector< float > weights = gd.optimize( matrix, best_values ); + GradientDescent gd( matrix, best_values ); + std::vector< float > weights = gd.optimize(); // std::vector> path = weights_to_path( weights ); @@ -1748,6 +1748,7 @@ void fuzzer::select_next_state() instrumentation::location_id loc_id(7); if (iid_dependences.id_to_equation_map.contains(loc_id)) { + std::cout << "ID: " << loc_id.id << std::endl; const iid_dependence_props& props = iid_dependences.id_to_equation_map.at(loc_id); std::vector weights = props.approximate_matrix(); diff --git a/src/fuzzing/src/gradient_descent.cpp b/src/fuzzing/src/gradient_descent.cpp index 6f244857..2420aff4 100644 --- a/src/fuzzing/src/gradient_descent.cpp +++ b/src/fuzzing/src/gradient_descent.cpp @@ -1,35 +1,37 @@ #include #include -#include #include +#include #include #include -#include -#include +#include -GradientDescent::GradientDescent( float learning_rate, int max_iterations, float convergence_threshold ) - : _learning_rate( learning_rate ) + +GradientDescent::GradientDescent( const std::vector< std::vector< float > >& coefficient_matrix, + const std::vector< float >& target_vector, + float learning_rate, + int max_iterations, + float convergence_threshold ) + : _coefficient_matrix( coefficient_matrix ) + , _target_vector( target_vector ) + , _learning_rate( learning_rate ) , _max_iterations( max_iterations ) , _convergence_threshold( convergence_threshold ) -{} - -std::vector< float > GradientDescent::optimize( const std::vector< std::vector< float > >& coefficient_matrix, - const std::vector< float >& target_vector ) { - TMPROF_BLOCK(); - if ( coefficient_matrix.empty() || target_vector.empty() || coefficient_matrix.size() != target_vector.size() ) { throw std::invalid_argument( "Invalid input dimensions" ); } +} - std::vector< float > current_solution = generateRandomWeights( coefficient_matrix[ 0 ].size() ); +std::vector< float > GradientDescent::optimize() +{ + std::vector< float > current_solution = generateRandomWeights( _coefficient_matrix[ 0 ].size() ); - std::vector< float > gradient; float prev_cost = std::numeric_limits< float >::max(); for ( int iteration = 0; iteration < _max_iterations; ++iteration ) { - gradient = computeGradient( current_solution, coefficient_matrix, target_vector ); + std::vector< float > gradient = computeGradient( current_solution ); // Update current_solution for ( size_t i = 0; i < current_solution.size(); ++i ) { @@ -37,72 +39,70 @@ std::vector< float > GradientDescent::optimize( const std::vector< std::vector< } // Check for convergence - float current_cost = - computeMeanSquaredError( current_solution, coefficient_matrix, target_vector ); + float current_cost = computeMeanSquaredError( current_solution ); // Debug output - if ( _debug && iteration % 50 == 0 ) { + if ( _debug && iteration % 100 == 0 ) { std::cout << "Iteration " << iteration << ", Cost: " << current_cost << std::endl; + std::cout << "Gradient: [ "; + for ( const auto& val : gradient ) { + std::cout << val << " "; + } + std::cout << "]" << std::endl; } if ( std::abs( current_cost - prev_cost ) < _convergence_threshold ) { std::cout << "Converged after " << iteration << " iterations." << std::endl; - std::cout << "Number of equations: " << coefficient_matrix.size() << std::endl; + std::cout << "Number of equations: " << _coefficient_matrix.size() << std::endl; break; } prev_cost = current_cost; } - normalize( current_solution, 0.0f, 100.0f ); + rescale( current_solution, -1.0f, 1.0f ); return current_solution; } -std::vector< float > GradientDescent::computeGradient( const std::vector< float >& current_solution, - const std::vector< std::vector< float > >& coefficient_matrix, - const std::vector< float >& target_vector ) +std::vector< float > GradientDescent::computeGradient( const std::vector< float >& current_solution ) { std::vector< float > gradient( current_solution.size(), 0.0f ); - for ( size_t i = 0; i < coefficient_matrix.size(); ++i ) { - float predicted = dotProduct( current_solution, coefficient_matrix[ i ] ); - float error = predicted - target_vector[ i ]; + for ( size_t i = 0; i < _coefficient_matrix.size(); ++i ) { + float predicted = dotProduct( current_solution, _coefficient_matrix[ i ] ); + float error = predicted - _target_vector[ i ]; for ( size_t j = 0; j < current_solution.size(); ++j ) { - gradient[ j ] += 2 * error * coefficient_matrix[ i ][ j ]; + gradient[ j ] += 2 * error * _coefficient_matrix[ i ][ j ]; } } for ( float& grad : gradient ) { - grad /= coefficient_matrix.size(); + grad /= _coefficient_matrix.size(); } return gradient; } -float GradientDescent::computeMeanSquaredError( const std::vector< float >& current_solution, - const std::vector< std::vector< float > >& coefficient_matrix, - const std::vector< float >& target_vector ) +float GradientDescent::computeMeanSquaredError( const std::vector< float >& current_solution ) { float mse = 0.0f; - for ( size_t i = 0; i < coefficient_matrix.size(); ++i ) { - float predicted = dotProduct( current_solution, coefficient_matrix[ i ] ); - float error = predicted - target_vector[ i ]; + for ( size_t i = 0; i < _coefficient_matrix.size(); ++i ) { + float predicted = dotProduct( current_solution, _coefficient_matrix[ i ] ); + float error = predicted - _target_vector[ i ]; mse += error * error; } - return mse / coefficient_matrix.size(); + return mse / _coefficient_matrix.size(); } float GradientDescent::dotProduct( const std::vector< float >& a, const std::vector< float >& b ) { - INVARIANT( a.size() == b.size() ); - - float result = 0.0f; - for ( size_t i = 0; i < a.size(); ++i ) { - result += a[ i ] * b[ i ]; + if ( a.size() != b.size() ) { + throw std::invalid_argument( "Vectors must have the same size for dot product" ); } - return result; + + return std::inner_product( a.begin(), a.end(), b.begin(), 0.0f ); } std::vector< float > GradientDescent::generateRandomWeights( size_t n ) @@ -111,30 +111,24 @@ std::vector< float > GradientDescent::generateRandomWeights( size_t n ) std::random_device rd; std::mt19937 gen( rd() ); std::normal_distribution<> d( 0, 1.0 / std::sqrt( n ) ); - for ( auto& weight : weights ) { - weight = d( gen ); - } + + std::generate( weights.begin(), weights.end(), [ & ]() { return d( gen ); } ); return weights; } -std::vector< float > GradientDescent::normalize( std::vector< float >& values, - float min_value, - float max_value ) +void GradientDescent::rescale( std::vector< float >& values, float min_value, float max_value ) { if ( values.empty() ) { - return values; + return; } - float min_elem = *std::min_element( values.begin(), values.end() ); - float max_elem = *std::max_element( values.begin(), values.end() ); + auto [ min_elem, max_elem ] = std::minmax_element( values.begin(), values.end() ); - if ( min_elem != max_elem ) { - for ( auto& value : values ) { - value = min_value + - ( value - min_elem ) * ( max_value - min_value ) / ( max_elem - min_elem ); - } + if ( *min_elem != *max_elem ) { + std::transform( values.begin(), values.end(), values.begin(), [ & ]( float value ) { + return min_value + + ( value - *min_elem ) * ( max_value - min_value ) / ( *max_elem - *min_elem ); + } ); } - - return values; } \ No newline at end of file From 1e1ff49b8c589f27609a4269425215380c830ec4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Thu, 26 Sep 2024 23:14:44 +0200 Subject: [PATCH 033/144] feat: added docstring --- src/fuzzing/src/gradient_descent.cpp | 51 ++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/src/fuzzing/src/gradient_descent.cpp b/src/fuzzing/src/gradient_descent.cpp index 2420aff4..7b4ebf3e 100644 --- a/src/fuzzing/src/gradient_descent.cpp +++ b/src/fuzzing/src/gradient_descent.cpp @@ -1,12 +1,21 @@ #include #include +#include #include #include #include #include -#include - +/** + * @brief Constructs a GradientDescent object. + * + * @param coefficient_matrix The matrix of coefficients for the linear system. + * @param target_vector The target vector for the linear system. + * @param learning_rate The learning rate for gradient descent (default: 0.001). + * @param max_iterations The maximum number of iterations (default: 10000). + * @param convergence_threshold The threshold for convergence (default: 1e-6). + * @throws std::invalid_argument if input dimensions are invalid. + */ GradientDescent::GradientDescent( const std::vector< std::vector< float > >& coefficient_matrix, const std::vector< float >& target_vector, float learning_rate, @@ -24,6 +33,11 @@ GradientDescent::GradientDescent( const std::vector< std::vector< float > >& coe } } +/** + * @brief Performs gradient descent optimization. + * + * @return std::vector The optimized solution vector. + */ std::vector< float > GradientDescent::optimize() { std::vector< float > current_solution = generateRandomWeights( _coefficient_matrix[ 0 ].size() ); @@ -63,6 +77,12 @@ std::vector< float > GradientDescent::optimize() return current_solution; } +/** + * @brief Computes the gradient for the current solution. + * + * @param current_solution The current solution vector. + * @return std::vector The computed gradient vector. + */ std::vector< float > GradientDescent::computeGradient( const std::vector< float >& current_solution ) { std::vector< float > gradient( current_solution.size(), 0.0f ); @@ -83,6 +103,12 @@ std::vector< float > GradientDescent::computeGradient( const std::vector< float return gradient; } +/** + * @brief Computes the mean squared error for the current solution. + * + * @param current_solution The current solution vector. + * @return float The computed mean squared error. + */ float GradientDescent::computeMeanSquaredError( const std::vector< float >& current_solution ) { float mse = 0.0f; @@ -96,6 +122,14 @@ float GradientDescent::computeMeanSquaredError( const std::vector< float >& curr return mse / _coefficient_matrix.size(); } +/** + * @brief Computes the dot product of two vectors. + * + * @param a The first vector. + * @param b The second vector. + * @return float The dot product of the two vectors. + * @throws std::invalid_argument if the vectors have different sizes. + */ float GradientDescent::dotProduct( const std::vector< float >& a, const std::vector< float >& b ) { if ( a.size() != b.size() ) { @@ -105,6 +139,12 @@ float GradientDescent::dotProduct( const std::vector< float >& a, const std::vec return std::inner_product( a.begin(), a.end(), b.begin(), 0.0f ); } +/** + * @brief Generates a vector of random weights. + * + * @param n The size of the weight vector to generate. + * @return std::vector A vector of random weights. + */ std::vector< float > GradientDescent::generateRandomWeights( size_t n ) { std::vector< float > weights( n ); @@ -117,6 +157,13 @@ std::vector< float > GradientDescent::generateRandomWeights( size_t n ) return weights; } +/** + * @brief Rescales the values in a vector to a specified range. + * + * @param values The vector of values to rescale. + * @param min_value The minimum value of the new range. + * @param max_value The maximum value of the new range. + */ void GradientDescent::rescale( std::vector< float >& values, float min_value, float max_value ) { if ( values.empty() ) { From 514c58d16630f24084b929e3a268d57533048c3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Fri, 27 Sep 2024 11:05:49 +0200 Subject: [PATCH 034/144] feat: compute mean depth for every best_value --- src/fuzzing/include/fuzzing/fuzzer.hpp | 13 ++++++++ src/fuzzing/src/fuzzer.cpp | 46 +++++++++++++++++++++++++- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/src/fuzzing/include/fuzzing/fuzzer.hpp b/src/fuzzing/include/fuzzing/fuzzer.hpp index 5be0c6c4..0996bd32 100644 --- a/src/fuzzing/include/fuzzing/fuzzer.hpp +++ b/src/fuzzing/include/fuzzing/fuzzer.hpp @@ -18,6 +18,7 @@ # include # include # include +# include namespace fuzzing { @@ -234,10 +235,20 @@ struct fuzzer final struct iid_dependence_props { + private: + struct FloatCompare { + bool operator()(const float& a, const float& b) const { + const float epsilon = 1e-6f; + return std::abs(a - b) > epsilon && a < b; + } + }; + + public: std::vector all_paths; std::set interesting_nodes; std::vector> matrix; std::vector best_values; + std::map, FloatCompare> value_to_mean_depth; bool update_interesting_nodes(branching_node* node); std::vector get_path(branching_node* node) const; @@ -245,6 +256,8 @@ struct fuzzer final void add_equation(branching_node* path); std::vector approximate_matrix() const; std::vector> weights_to_path(std::vector const& weights) const; + int get_node_depth(branching_node* node) const; + void update_value_to_mean_depth(branching_node* node); }; struct iid_node_dependence diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index e6633b24..02039362 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -1202,6 +1202,50 @@ fuzzing::fuzzer::iid_dependence_props::weights_to_path( std::vector< float > con } +/** + * @brief Calculates the depth of a given branching node in the tree. + * + * This function traverses the tree from the given node to the root, + * counting the number of edges (or levels) to determine the depth of the node. + * + * @param node A pointer to the branching_node whose depth is to be calculated. + * @return The depth of the node as an integer. + */ +int fuzzing::fuzzer::iid_dependence_props::get_node_depth( branching_node* node ) const +{ + int depth = 0; + branching_node* current = node; + while ( current != nullptr ) { + current = current->predecessor; + ++depth; + } + + return depth; +} + + +/** + * @brief Updates the mean depth value for a given branching node. + * + * This function calculates the depth of the provided branching node and updates + * the mean depth value associated with the node's best coverage value. The mean + * depth is updated incrementally using the formula: + * + * new_mean = current_mean + (depth - current_mean) / count + * + * where `depth` is the depth of the node, `current_mean` is the current mean depth, + * and `count` is the number of times the mean has been updated. + * + * @param node A pointer to the branching node whose mean depth value is to be updated. + */ +void fuzzing::fuzzer::iid_dependence_props::update_value_to_mean_depth( branching_node* node ) +{ + int depth = get_node_depth( node ); + auto& [ current_mean, count ] = value_to_mean_depth[ node->best_coverage_value ]; + current_mean = current_mean + ( depth - current_mean ) / ++count; +} + + /** * @brief Updates the set of non-IID nodes based on sensitivity analysis. * @@ -1254,6 +1298,7 @@ void fuzzing::fuzzer::iid_node_dependence::process_node_dependence( branching_no } props.add_equation( node ); + props.update_value_to_mean_depth( node ); } @@ -1748,7 +1793,6 @@ void fuzzer::select_next_state() instrumentation::location_id loc_id(7); if (iid_dependences.id_to_equation_map.contains(loc_id)) { - std::cout << "ID: " << loc_id.id << std::endl; const iid_dependence_props& props = iid_dependences.id_to_equation_map.at(loc_id); std::vector weights = props.approximate_matrix(); From e1f81423d511cb745b5bd70066326018a750f35c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sun, 29 Sep 2024 19:59:31 +0200 Subject: [PATCH 035/144] feat: move iid_node_dependencies to separate file --- src/fuzzing/include/fuzzing/fuzzer.hpp | 51 +-- .../include/fuzzing/gradient_descent.hpp | 15 +- .../include/fuzzing/iid_node_dependencies.hpp | 67 ++++ src/fuzzing/src/fuzzer.cpp | 349 +----------------- src/fuzzing/src/gradient_descent.cpp | 41 +- src/fuzzing/src/iid_node_dependencies.cpp | 315 ++++++++++++++++ 6 files changed, 425 insertions(+), 413 deletions(-) create mode 100644 src/fuzzing/include/fuzzing/iid_node_dependencies.hpp create mode 100644 src/fuzzing/src/iid_node_dependencies.cpp diff --git a/src/fuzzing/include/fuzzing/fuzzer.hpp b/src/fuzzing/include/fuzzing/fuzzer.hpp index 0996bd32..2c328e9c 100644 --- a/src/fuzzing/include/fuzzing/fuzzer.hpp +++ b/src/fuzzing/include/fuzzing/fuzzer.hpp @@ -4,6 +4,7 @@ # include # include # include +# include # include # include # include @@ -219,56 +220,6 @@ struct fuzzer final mutable random_generator_for_natural_32_bit generator_for_pivot_selection; }; - struct node_navigation - { - location_id node_id; - bool direction; - - auto operator<=>(node_navigation const& other) const; - bool operator==(node_navigation const& other) const; - friend std::ostream& operator<<(std::ostream& os, node_navigation const& nn) - { - os << nn.node_id.id << " " << (nn.direction ? "right" : "left"); - return os; - } - }; - - struct iid_dependence_props - { - private: - struct FloatCompare { - bool operator()(const float& a, const float& b) const { - const float epsilon = 1e-6f; - return std::abs(a - b) > epsilon && a < b; - } - }; - - public: - std::vector all_paths; - std::set interesting_nodes; - std::vector> matrix; - std::vector best_values; - std::map, FloatCompare> value_to_mean_depth; - - bool update_interesting_nodes(branching_node* node); - std::vector get_path(branching_node* node) const; - void recompute_matrix(); - void add_equation(branching_node* path); - std::vector approximate_matrix() const; - std::vector> weights_to_path(std::vector const& weights) const; - int get_node_depth(branching_node* node) const; - void update_value_to_mean_depth(branching_node* node); - }; - - struct iid_node_dependence - { - std::unordered_map id_to_equation_map; - std::set non_iid_nodes; - - void update_non_iid_nodes(sensitivity_analysis& sensitivity); - void process_node_dependence(branching_node* node); - }; - static void update_close_flags_from(branching_node* node); static std::vector const& get_input_width_classes(); diff --git a/src/fuzzing/include/fuzzing/gradient_descent.hpp b/src/fuzzing/include/fuzzing/gradient_descent.hpp index 2d45f3a2..57db0487 100644 --- a/src/fuzzing/include/fuzzing/gradient_descent.hpp +++ b/src/fuzzing/include/fuzzing/gradient_descent.hpp @@ -20,9 +20,9 @@ class GradientDescent { std::vector< float > optimize(); // Setters for easy modification of parameters - void setLearningRate( float learning_rate ) { _learning_rate = learning_rate; } - void setMaxIterations( int max_iterations ) { _max_iterations = max_iterations; } - void setConvergenceThreshold( float convergence_threshold ) + void set_learning_rate( float learning_rate ) { _learning_rate = learning_rate; } + void set_max_iterations( int max_iterations ) { _max_iterations = max_iterations; } + void set_convergence_threshold( float convergence_threshold ) { _convergence_threshold = convergence_threshold; } @@ -35,9 +35,10 @@ class GradientDescent { float _convergence_threshold; bool _debug = false; - std::vector< float > computeGradient( const std::vector< float >& current_solution ); - float computeMeanSquaredError( const std::vector< float >& current_solution ); - static std::vector< float > generateRandomWeights( size_t n ); - static float dotProduct( const std::vector< float >& a, const std::vector< float >& b ); + std::vector< float > compute_gradient( const std::vector< float >& current_solution ); + float compute_mean_squared_error( const std::vector< float >& current_solution ); + static std::vector< float > generate_random_weights( size_t n ); + static float dot_product( const std::vector< float >& a, const std::vector< float >& b ); static void rescale( std::vector< float >& values, float min_value, float max_value ); + static void add_smallest_value( std::vector< float >& values ); }; \ No newline at end of file diff --git a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp new file mode 100644 index 00000000..5c970b18 --- /dev/null +++ b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp @@ -0,0 +1,67 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + + +namespace fuzzing +{ +struct node_navigation { + location_id node_id; + bool direction; + + auto operator<=>( node_navigation const& other ) const; + bool operator==( node_navigation const& other ) const; + friend std::ostream& operator<<( std::ostream& os, node_navigation const& nn ) + { + os << nn.node_id.id << " " << ( nn.direction ? "right" : "left" ); + return os; + } +}; + +struct iid_dependence_props { +private: + struct FloatCompare { + bool operator()( const float& a, const float& b ) const + { + const float epsilon = 1e-6f; + return std::abs( a - b ) > epsilon && a < b; + } + }; + +public: + std::vector< branching_node* > all_paths; + std::set< node_navigation > interesting_nodes; + std::vector< std::vector< float > > matrix; + std::vector< float > best_values; + std::map< float, std::tuple< float, int >, FloatCompare > value_to_mean_depth; + + bool update_interesting_nodes( branching_node* node ); + std::vector< node_navigation > get_path( branching_node* node ) const; + void recompute_matrix(); + void add_equation( branching_node* path ); + std::vector< float > approximate_matrix() const; + std::vector< std::pair< int, node_navigation > > + weights_to_path( std::vector< float > const& weights ) const; + int get_node_depth( branching_node* node ) const; + void update_value_to_mean_depth( branching_node* node ); +}; + +struct iid_node_dependence { + std::unordered_map< location_id, iid_dependence_props > id_to_equation_map; + std::set< location_id > non_iid_nodes; + + void update_non_iid_nodes( sensitivity_analysis& sensitivity ); + void process_node_dependence( branching_node* node ); +}; +} // namespace fuzzing \ No newline at end of file diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index 02039362..b57f7161 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -974,334 +974,6 @@ void fuzzer::generate_next_input(vecb& stdin_bits) UNREACHABLE(); } - -/** - * @brief Three-way comparison for node_navigation. - * - * @param other The other node_navigation object to compare with. - * @return std::strong_ordering Result of the comparison. - */ -auto fuzzing::fuzzer::node_navigation::operator<=>( node_navigation const& other ) const -{ - if ( auto const cmp = node_id.id <=> other.node_id.id; cmp != 0 ) - return cmp; - - return direction <=> other.direction; -} - - -/** - * @brief Equality operator for node_navigation. - * - * Compares two node_navigation objects for equality based on their node_id and direction. - * - * @param other The other node_navigation object to compare with. - * @return true if both node_id and direction are equal, false otherwise. - */ -bool fuzzing::fuzzer::node_navigation::operator==( node_navigation const& other ) const -{ - return node_id.id == other.node_id.id && direction == other.direction; -} - - -/** - * @brief Updates the set of interesting nodes based on the given branching node. - * - * This function traverses the paths from the given branching node and updates the set of - * interesting nodes by comparing the paths with all existing paths. If new interesting nodes are - * found, the set is updated and the function returns true. - * - * @param node A pointer to the branching node from which to start the path traversal. - * @return true if the set of interesting nodes was updated, false otherwise. - */ -bool fuzzing::fuzzer::iid_dependence_props::update_interesting_nodes( branching_node* node ) -{ - TMPROF_BLOCK(); - - bool set_changed = false; - - auto add_to_interesting = [ this, &set_changed ]( std::vector< node_navigation >& nodes, int i ) { - for ( ; i >= 0; --i ) { - auto result = this->interesting_nodes.emplace( nodes[ i ] ); - if ( result.second ) { - set_changed = true; - } - } - }; - - for ( const auto& end_node : all_paths ) { - std::vector< node_navigation > path_1 = get_path( node ); - std::vector< node_navigation > path_2 = get_path( end_node ); - - if ( path_1.empty() || path_2.empty() ) - continue; - - ASSUMPTION( path_1.back() == path_2.back() ); - - std::size_t i_1 = path_1.size() - 1; - std::size_t i_2 = path_2.size() - 1; - - while ( i_1 > 0 && i_2 > 0 && path_1[ i_1 ] == path_2[ i_2 ] ) { - --i_1; - --i_2; - } - - add_to_interesting( path_1, i_1 ); - add_to_interesting( path_2, i_2 ); - } - - return set_changed; -} - - -/** - * @brief Retrieves the path of node navigations from the given branching node to the root. - * - * This function constructs a vector of `node_navigation` objects representing the path - * from the specified branching node to the root node. Each `node_navigation` object - * contains the location ID of the predecessor node and the direction to the current node. - * - * @param node A pointer to the starting branching node. - * @return A vector of `node_navigation` objects representing the path from the given node to the - * root. - */ -std::vector< fuzzer::node_navigation > -fuzzing::fuzzer::iid_dependence_props::get_path( branching_node* node ) const -{ - std::vector< node_navigation > path; - - branching_node* current = node; - while ( current != nullptr ) { - branching_node* predecessor = current->predecessor; - if ( predecessor != nullptr ) { - node_navigation nav = { predecessor->get_location_id(), - predecessor->successor_direction( current ) }; - path.push_back( nav ); - } - current = predecessor; - } - - return path; -} - - -/** - * @brief Recomputes the matrix of IID dependence properties. - * - * This function clears the current matrix and best values, then iterates - * through all paths to add equations to the matrix. If there are no paths, - * the function returns immediately. - * - * @note The matrix clearing could be optimized in the future. - */ -void fuzzing::fuzzer::iid_dependence_props::recompute_matrix() -{ - TMPROF_BLOCK(); - - if ( all_paths.empty() ) - return; - - matrix.clear(); // This could be done better, but for now it's fine - best_values.clear(); - - for ( const auto& path : all_paths ) { - add_equation( path ); - } -} - - -/** - * @brief Adds an equation to the IID dependence properties based on the given branching path. - * - * This function updates the internal matrix and best values with the direction counts of - * interesting nodes encountered in the path. - * - * @param path A pointer to the branching_node representing the path to be processed. - */ -void fuzzing::fuzzer::iid_dependence_props::add_equation( branching_node* path ) -{ - TMPROF_BLOCK(); - - std::map< node_navigation, int > directions_in_path; - for ( const node_navigation& navigation : interesting_nodes ) { - directions_in_path[ navigation ] = 0; - } - - std::vector< node_navigation > path_nodes = get_path( path ); - - for ( const node_navigation& nav : path_nodes ) { - if ( interesting_nodes.contains( nav ) ) { - directions_in_path[ nav ]++; - } - } - - std::vector< float > values_in_path; - for ( const auto& [ direction, count ] : directions_in_path ) { - values_in_path.push_back( count ); - } - - matrix.push_back( values_in_path ); - best_values.push_back( path->best_coverage_value ); -} - - -/** - * @brief Approximates a matrix using gradient descent optimization. - * - * This function utilizes the GradientDescent class to optimize the given matrix - * and best values, returning a vector of optimized weights. - * - * @return A vector of floats representing the optimized weights. - */ -std::vector< float > fuzzing::fuzzer::iid_dependence_props::approximate_matrix() const -{ - GradientDescent gd( matrix, best_values ); - std::vector< float > weights = gd.optimize(); - - - // std::vector> path = weights_to_path( weights ); - // for (const auto& [value, nav] : path) { - // std::cout << "Node ID: " << nav.node_id.id << ", Direction: " << nav.direction << ", - // Value: " << value << std::endl; - // } - - return weights; -} - - -std::vector< std::pair< int, fuzzer::node_navigation > > -fuzzing::fuzzer::iid_dependence_props::weights_to_path( std::vector< float > const& weights ) const -{ - // *std::next(props.interesting_nodes.begin(), i) - - std::vector< std::pair< float, size_t > > positive_values; - - // Step 1: Consider only positive values and store their indices - for ( size_t i = 0; i < weights.size(); ++i ) { - if ( weights[ i ] > 0 ) { - positive_values.emplace_back( weights[ i ], i ); - // std::cout << "Value: " << weights[i] << std::endl; - } - } - - // Sort the positive values - std::sort( positive_values.begin(), positive_values.end() ); - std::vector< std::pair< int, node_navigation > > result; - - if ( !positive_values.empty() ) { - float smallest = positive_values[ 0 ].first; - - // Steps 2-4: Create new values and store as integers - for ( const auto& [ value, index ] : positive_values ) { - int new_value = std::round( value / smallest ); - result.emplace_back( new_value, *std::next( interesting_nodes.begin(), index ) ); - } - } - - return result; -} - - -/** - * @brief Calculates the depth of a given branching node in the tree. - * - * This function traverses the tree from the given node to the root, - * counting the number of edges (or levels) to determine the depth of the node. - * - * @param node A pointer to the branching_node whose depth is to be calculated. - * @return The depth of the node as an integer. - */ -int fuzzing::fuzzer::iid_dependence_props::get_node_depth( branching_node* node ) const -{ - int depth = 0; - branching_node* current = node; - while ( current != nullptr ) { - current = current->predecessor; - ++depth; - } - - return depth; -} - - -/** - * @brief Updates the mean depth value for a given branching node. - * - * This function calculates the depth of the provided branching node and updates - * the mean depth value associated with the node's best coverage value. The mean - * depth is updated incrementally using the formula: - * - * new_mean = current_mean + (depth - current_mean) / count - * - * where `depth` is the depth of the node, `current_mean` is the current mean depth, - * and `count` is the number of times the mean has been updated. - * - * @param node A pointer to the branching node whose mean depth value is to be updated. - */ -void fuzzing::fuzzer::iid_dependence_props::update_value_to_mean_depth( branching_node* node ) -{ - int depth = get_node_depth( node ); - auto& [ current_mean, count ] = value_to_mean_depth[ node->best_coverage_value ]; - current_mean = current_mean + ( depth - current_mean ) / ++count; -} - - -/** - * @brief Updates the set of non-IID nodes based on sensitivity analysis. - * - * This function iterates through the nodes that have changed according to the - * provided sensitivity analysis. For each node that has undergone branching, - * it retrieves the node's location ID and attempts to insert it into the set - * of non-IID nodes. If the insertion is successful, it also removes the - * corresponding entry from the ID-to-equation map. - * - * @param sensitivity A reference to the sensitivity analysis object that - * provides the changed nodes. - */ -void fuzzing::fuzzer::iid_node_dependence::update_non_iid_nodes( sensitivity_analysis& sensitivity ) -{ - for ( branching_node* node : sensitivity.get_changed_nodes() ) { - if ( node->is_did_branching() ) { - auto location_id = node->get_location_id(); - if ( non_iid_nodes.insert( location_id ).second ) { - id_to_equation_map.erase( location_id ); - } - } - } -} - - -/** - * @brief Processes the dependence of a given branching node. - * - * This function processes the dependence of a branching node by updating - * the internal state and properties associated with the node's location ID. - * It ensures that non-IID nodes are skipped, updates the list of all paths, - * and recomputes the matrix if the node is deemed interesting. Additionally, - * it adds the node to the equation map. - * - * @param node A pointer to the branching node to be processed. - */ -void fuzzing::fuzzer::iid_node_dependence::process_node_dependence( branching_node* node ) -{ - TMPROF_BLOCK(); - - if ( non_iid_nodes.contains( node->get_location_id() ) ) - return; - - iid_dependence_props& props = id_to_equation_map[ node->get_location_id() ]; - - props.all_paths.push_back( node ); - - if ( props.update_interesting_nodes( node ) ) { - props.recompute_matrix(); - } - - props.add_equation( node ); - props.update_value_to_mean_depth( node ); -} - - execution_record::execution_flags fuzzer::process_execution_results() { TMPROF_BLOCK(); @@ -1798,9 +1470,9 @@ void fuzzer::select_next_state() std::vector weights = props.approximate_matrix(); // std::cout << "ID: " << loc_id.id << std::endl; - // for (size_t i = 0; i < props.interesting_nodes.size() && i < weights.size(); ++i) { - // std::cout << *std::next(props.interesting_nodes.begin(), i) << " : " << weights[i] << std::endl; - // } + for (size_t i = 0; i < props.interesting_nodes.size() && i < weights.size(); ++i) { + std::cout << *std::next(props.interesting_nodes.begin(), i) << " : " << weights[i] << std::endl; + } } TMPROF_BLOCK(); @@ -1943,21 +1615,6 @@ branching_node* fuzzer::select_iid_coverage_target() const INVARIANT(winner != nullptr); - // instrumentation::location_id loc_id(6); - // const iid_dependence_props& props = iid_dependences.id_to_equation_map.at(loc_id); - // if (props.matrix.empty()) - // return winner; - - // // std::vector weights = props.approximate_matrix(); - // GradientDescent gd; - // std::vector weights = gd.optimize(props.matrix, props.best_values); - - // std::cout << "ID: " << loc_id.id << std::endl; - - // for (size_t i = 0; i < props.interesting_nodes.size() && i < weights.size(); ++i) { - // std::cout << *std::next(props.interesting_nodes.begin(), i) << " : " << weights[i] << std::endl; - // } - return winner; } diff --git a/src/fuzzing/src/gradient_descent.cpp b/src/fuzzing/src/gradient_descent.cpp index 7b4ebf3e..ce90cbb9 100644 --- a/src/fuzzing/src/gradient_descent.cpp +++ b/src/fuzzing/src/gradient_descent.cpp @@ -40,12 +40,12 @@ GradientDescent::GradientDescent( const std::vector< std::vector< float > >& coe */ std::vector< float > GradientDescent::optimize() { - std::vector< float > current_solution = generateRandomWeights( _coefficient_matrix[ 0 ].size() ); + std::vector< float > current_solution = generate_random_weights( _coefficient_matrix[ 0 ].size() ); float prev_cost = std::numeric_limits< float >::max(); for ( int iteration = 0; iteration < _max_iterations; ++iteration ) { - std::vector< float > gradient = computeGradient( current_solution ); + std::vector< float > gradient = compute_gradient( current_solution ); // Update current_solution for ( size_t i = 0; i < current_solution.size(); ++i ) { @@ -53,7 +53,7 @@ std::vector< float > GradientDescent::optimize() } // Check for convergence - float current_cost = computeMeanSquaredError( current_solution ); + float current_cost = compute_mean_squared_error( current_solution ); // Debug output if ( _debug && iteration % 100 == 0 ) { @@ -73,7 +73,8 @@ std::vector< float > GradientDescent::optimize() prev_cost = current_cost; } - rescale( current_solution, -1.0f, 1.0f ); + add_smallest_value( current_solution ); + rescale( current_solution, 0.0f, 100.0f ); return current_solution; } @@ -83,12 +84,12 @@ std::vector< float > GradientDescent::optimize() * @param current_solution The current solution vector. * @return std::vector The computed gradient vector. */ -std::vector< float > GradientDescent::computeGradient( const std::vector< float >& current_solution ) +std::vector< float > GradientDescent::compute_gradient( const std::vector< float >& current_solution ) { std::vector< float > gradient( current_solution.size(), 0.0f ); for ( size_t i = 0; i < _coefficient_matrix.size(); ++i ) { - float predicted = dotProduct( current_solution, _coefficient_matrix[ i ] ); + float predicted = dot_product( current_solution, _coefficient_matrix[ i ] ); float error = predicted - _target_vector[ i ]; for ( size_t j = 0; j < current_solution.size(); ++j ) { @@ -109,12 +110,12 @@ std::vector< float > GradientDescent::computeGradient( const std::vector< float * @param current_solution The current solution vector. * @return float The computed mean squared error. */ -float GradientDescent::computeMeanSquaredError( const std::vector< float >& current_solution ) +float GradientDescent::compute_mean_squared_error( const std::vector< float >& current_solution ) { float mse = 0.0f; for ( size_t i = 0; i < _coefficient_matrix.size(); ++i ) { - float predicted = dotProduct( current_solution, _coefficient_matrix[ i ] ); + float predicted = dot_product( current_solution, _coefficient_matrix[ i ] ); float error = predicted - _target_vector[ i ]; mse += error * error; } @@ -130,7 +131,7 @@ float GradientDescent::computeMeanSquaredError( const std::vector< float >& curr * @return float The dot product of the two vectors. * @throws std::invalid_argument if the vectors have different sizes. */ -float GradientDescent::dotProduct( const std::vector< float >& a, const std::vector< float >& b ) +float GradientDescent::dot_product( const std::vector< float >& a, const std::vector< float >& b ) { if ( a.size() != b.size() ) { throw std::invalid_argument( "Vectors must have the same size for dot product" ); @@ -145,7 +146,7 @@ float GradientDescent::dotProduct( const std::vector< float >& a, const std::vec * @param n The size of the weight vector to generate. * @return std::vector A vector of random weights. */ -std::vector< float > GradientDescent::generateRandomWeights( size_t n ) +std::vector< float > GradientDescent::generate_random_weights( size_t n ) { std::vector< float > weights( n ); std::random_device rd; @@ -178,4 +179,24 @@ void GradientDescent::rescale( std::vector< float >& values, float min_value, fl ( value - *min_elem ) * ( max_value - min_value ) / ( *max_elem - *min_elem ); } ); } +} + +/** + * @brief Adds the smallest value in the vector to each element of the vector. + * + * This function finds the smallest value in the given vector and adds its absolute value + * to each element of the vector. If the vector is empty, the function does nothing. + * + * @param values A reference to a vector of floats to be modified. + */ +void GradientDescent::add_smallest_value( std::vector< float >& values ) +{ + auto min_elem = std::min_element( values.begin(), values.end() ); + + if ( min_elem != values.end() ) { + float min_value = *min_elem; + std::transform( values.begin(), values.end(), values.begin(), [ & ]( float value ) { + return value + std::abs( min_value ); + } ); + } } \ No newline at end of file diff --git a/src/fuzzing/src/iid_node_dependencies.cpp b/src/fuzzing/src/iid_node_dependencies.cpp new file mode 100644 index 00000000..67b36514 --- /dev/null +++ b/src/fuzzing/src/iid_node_dependencies.cpp @@ -0,0 +1,315 @@ +#include +#include +#include + +/** + * @brief Three-way comparison for node_navigation. + * + * @param other The other node_navigation object to compare with. + * @return std::strong_ordering Result of the comparison. + */ +auto fuzzing::node_navigation::operator<=>( node_navigation const& other ) const +{ + if ( auto const cmp = node_id.id <=> other.node_id.id; cmp != 0 ) + return cmp; + + return direction <=> other.direction; +} + + +/** + * @brief Equality operator for node_navigation. + * + * Compares two node_navigation objects for equality based on their node_id and direction. + * + * @param other The other node_navigation object to compare with. + * @return true if both node_id and direction are equal, false otherwise. + */ +bool fuzzing::node_navigation::operator==( node_navigation const& other ) const +{ + return node_id.id == other.node_id.id && direction == other.direction; +} + + +/** + * @brief Updates the set of interesting nodes based on the given branching node. + * + * This function traverses the paths from the given branching node and updates the set of + * interesting nodes by comparing the paths with all existing paths. If new interesting nodes are + * found, the set is updated and the function returns true. + * + * @param node A pointer to the branching node from which to start the path traversal. + * @return true if the set of interesting nodes was updated, false otherwise. + */ +bool fuzzing::iid_dependence_props::update_interesting_nodes( branching_node* node ) +{ + TMPROF_BLOCK(); + + bool set_changed = false; + + auto add_to_interesting = [ this, &set_changed ]( std::vector< node_navigation >& nodes, int i ) { + for ( ; i >= 0; --i ) { + auto result = this->interesting_nodes.emplace( nodes[ i ] ); + if ( result.second ) { + set_changed = true; + } + } + }; + + for ( const auto& end_node : all_paths ) { + std::vector< node_navigation > path_1 = get_path( node ); + std::vector< node_navigation > path_2 = get_path( end_node ); + + if ( path_1.empty() || path_2.empty() ) + continue; + + ASSUMPTION( path_1.back() == path_2.back() ); + + std::size_t i_1 = path_1.size() - 1; + std::size_t i_2 = path_2.size() - 1; + + while ( i_1 > 0 && i_2 > 0 && path_1[ i_1 ] == path_2[ i_2 ] ) { + --i_1; + --i_2; + } + + add_to_interesting( path_1, i_1 ); + add_to_interesting( path_2, i_2 ); + } + + return set_changed; +} + + +/** + * @brief Retrieves the path of node navigations from the given branching node to the root. + * + * This function constructs a vector of `node_navigation` objects representing the path + * from the specified branching node to the root node. Each `node_navigation` object + * contains the location ID of the predecessor node and the direction to the current node. + * + * @param node A pointer to the starting branching node. + * @return A vector of `node_navigation` objects representing the path from the given node to the + * root. + */ +std::vector< fuzzing::node_navigation > fuzzing::iid_dependence_props::get_path( branching_node* node ) const +{ + std::vector< node_navigation > path; + + branching_node* current = node; + while ( current != nullptr ) { + branching_node* predecessor = current->predecessor; + if ( predecessor != nullptr ) { + node_navigation nav = { predecessor->get_location_id(), + predecessor->successor_direction( current ) }; + path.push_back( nav ); + } + current = predecessor; + } + + return path; +} + + +/** + * @brief Recomputes the matrix of IID dependence properties. + * + * This function clears the current matrix and best values, then iterates + * through all paths to add equations to the matrix. If there are no paths, + * the function returns immediately. + * + * @note The matrix clearing could be optimized in the future. + */ +void fuzzing::iid_dependence_props::recompute_matrix() +{ + TMPROF_BLOCK(); + + if ( all_paths.empty() ) + return; + + matrix.clear(); // This could be done better, but for now it's fine + best_values.clear(); + + for ( const auto& path : all_paths ) { + add_equation( path ); + } +} + + +/** + * @brief Adds an equation to the IID dependence properties based on the given branching path. + * + * This function updates the internal matrix and best values with the direction counts of + * interesting nodes encountered in the path. + * + * @param path A pointer to the branching_node representing the path to be processed. + */ +void fuzzing::iid_dependence_props::add_equation( branching_node* path ) +{ + TMPROF_BLOCK(); + + std::map< node_navigation, int > directions_in_path; + for ( const node_navigation& navigation : interesting_nodes ) { + directions_in_path[ navigation ] = 0; + } + + std::vector< node_navigation > path_nodes = get_path( path ); + + for ( const node_navigation& nav : path_nodes ) { + if ( interesting_nodes.contains( nav ) ) { + directions_in_path[ nav ]++; + } + } + + std::vector< float > values_in_path; + for ( const auto& [ direction, count ] : directions_in_path ) { + values_in_path.push_back( count ); + } + + matrix.push_back( values_in_path ); + best_values.push_back( path->best_coverage_value ); +} + + +/** + * @brief Approximates a matrix using gradient descent optimization. + * + * This function utilizes the GradientDescent class to optimize the given matrix + * and best values, returning a vector of optimized weights. + * + * @return A vector of floats representing the optimized weights. + */ +std::vector< float > fuzzing::iid_dependence_props::approximate_matrix() const +{ + GradientDescent gd( matrix, best_values ); + std::vector< float > weights = gd.optimize(); + + std::vector< std::pair< int, node_navigation > > path = weights_to_path( weights ); + for ( const auto& [ value, nav ] : path ) { + std::cout << "Node ID: " << nav.node_id.id << ", Direction: " << nav.direction + << ", Value: " << value << std::endl; + } + + return weights; +} + + +std::vector< std::pair< int, fuzzing::node_navigation > > +fuzzing::iid_dependence_props::weights_to_path( std::vector< float > const& weights ) const +{ + int path_size = 35; // Get this value from value_to_mean_depth + + if ( path_size == 0 || weights.empty() ) { + return {}; + } + + float weights_sum = std::accumulate( weights.begin(), weights.end(), 0.0f ); + std::vector< std::pair< int, node_navigation > > path; + + for ( int i = 0; i < weights.size(); ++i ) { + float value = static_cast< float >( path_size ) * weights[ i ] / weights_sum; + path.push_back( { static_cast< int >( value ), *std::next( interesting_nodes.begin(), i ) } ); + } + + return path; +} + + +/** + * @brief Calculates the depth of a given branching node in the tree. + * + * This function traverses the tree from the given node to the root, + * counting the number of edges (or levels) to determine the depth of the node. + * + * @param node A pointer to the branching_node whose depth is to be calculated. + * @return The depth of the node as an integer. + */ +int fuzzing::iid_dependence_props::get_node_depth( branching_node* node ) const +{ + int depth = 0; + branching_node* current = node; + while ( current != nullptr ) { + current = current->predecessor; + ++depth; + } + + return depth; +} + + +/** + * @brief Updates the mean depth value for a given branching node. + * + * This function calculates the depth of the provided branching node and updates + * the mean depth value associated with the node's best coverage value. The mean + * depth is updated incrementally using the formula: + * + * new_mean = current_mean + (depth - current_mean) / count + * + * where `depth` is the depth of the node, `current_mean` is the current mean depth, + * and `count` is the number of times the mean has been updated. + * + * @param node A pointer to the branching node whose mean depth value is to be updated. + */ +void fuzzing::iid_dependence_props::update_value_to_mean_depth( branching_node* node ) +{ + int depth = get_node_depth( node ); + auto& [ current_mean, count ] = value_to_mean_depth[ node->best_coverage_value ]; + current_mean = current_mean + ( depth - current_mean ) / ++count; +} + + +/** + * @brief Updates the set of non-IID nodes based on sensitivity analysis. + * + * This function iterates through the nodes that have changed according to the + * provided sensitivity analysis. For each node that has undergone branching, + * it retrieves the node's location ID and attempts to insert it into the set + * of non-IID nodes. If the insertion is successful, it also removes the + * corresponding entry from the ID-to-equation map. + * + * @param sensitivity A reference to the sensitivity analysis object that + * provides the changed nodes. + */ +void fuzzing::iid_node_dependence::update_non_iid_nodes( sensitivity_analysis& sensitivity ) +{ + for ( branching_node* node : sensitivity.get_changed_nodes() ) { + if ( node->is_did_branching() ) { + auto location_id = node->get_location_id(); + if ( non_iid_nodes.insert( location_id ).second ) { + id_to_equation_map.erase( location_id ); + } + } + } +} + + +/** + * @brief Processes the dependence of a given branching node. + * + * This function processes the dependence of a branching node by updating + * the internal state and properties associated with the node's location ID. + * It ensures that non-IID nodes are skipped, updates the list of all paths, + * and recomputes the matrix if the node is deemed interesting. Additionally, + * it adds the node to the equation map. + * + * @param node A pointer to the branching node to be processed. + */ +void fuzzing::iid_node_dependence::process_node_dependence( branching_node* node ) +{ + TMPROF_BLOCK(); + + if ( non_iid_nodes.contains( node->get_location_id() ) ) + return; + + iid_dependence_props& props = id_to_equation_map[ node->get_location_id() ]; + + props.all_paths.push_back( node ); + + if ( props.update_interesting_nodes( node ) ) { + props.recompute_matrix(); + } + + props.add_equation( node ); + props.update_value_to_mean_depth( node ); +} \ No newline at end of file From 9762859f388240978650337b31d01986f45e13d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sun, 29 Sep 2024 22:06:26 +0200 Subject: [PATCH 036/144] feat: get possible depth --- src/fuzzing/include/fuzzing/fuzzer.hpp | 2 +- .../include/fuzzing/iid_node_dependencies.hpp | 43 ++++---- src/fuzzing/src/fuzzer.cpp | 2 +- src/fuzzing/src/iid_node_dependencies.cpp | 100 +++++++++++------- 4 files changed, 90 insertions(+), 57 deletions(-) diff --git a/src/fuzzing/include/fuzzing/fuzzer.hpp b/src/fuzzing/include/fuzzing/fuzzer.hpp index 2c328e9c..e4f15ec0 100644 --- a/src/fuzzing/include/fuzzing/fuzzer.hpp +++ b/src/fuzzing/include/fuzzing/fuzzer.hpp @@ -313,7 +313,7 @@ struct fuzzer final primary_coverage_target_branchings primary_coverage_targets; std::unordered_map iid_pivots; - iid_node_dependence iid_dependences; + iid_dependencies iid_dependences; std::unordered_set coverage_failures_with_hope; diff --git a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp index 5c970b18..9fcc8af2 100644 --- a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp +++ b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp @@ -13,52 +13,57 @@ #include #include +struct FloatCompare { + bool operator()( const float& a, const float& b ) const + { + const float epsilon = 1e-6f; + return std::abs( a - b ) > epsilon && std::abs( a ) < std::abs( b ); + } +}; namespace fuzzing { -struct node_navigation { +struct node_direction { location_id node_id; bool direction; - auto operator<=>( node_navigation const& other ) const; - bool operator==( node_navigation const& other ) const; - friend std::ostream& operator<<( std::ostream& os, node_navigation const& nn ) + auto operator<=>( node_direction const& other ) const; + bool operator==( node_direction const& other ) const; + friend std::ostream& operator<<( std::ostream& os, node_direction const& nn ) { os << nn.node_id.id << " " << ( nn.direction ? "right" : "left" ); return os; } }; -struct iid_dependence_props { -private: - struct FloatCompare { - bool operator()( const float& a, const float& b ) const - { - const float epsilon = 1e-6f; - return std::abs( a - b ) > epsilon && a < b; - } + +struct iid_node_dependence_props { + struct iid_value_props { + int value_counts; + float mean_depth; }; -public: std::vector< branching_node* > all_paths; - std::set< node_navigation > interesting_nodes; + std::set< node_direction > interesting_nodes; std::vector< std::vector< float > > matrix; std::vector< float > best_values; - std::map< float, std::tuple< float, int >, FloatCompare > value_to_mean_depth; + std::map< float, iid_value_props, FloatCompare > value_to_mean_depth; bool update_interesting_nodes( branching_node* node ); - std::vector< node_navigation > get_path( branching_node* node ) const; + std::vector< node_direction > get_path( branching_node* node ) const; void recompute_matrix(); void add_equation( branching_node* path ); std::vector< float > approximate_matrix() const; - std::vector< std::pair< int, node_navigation > > + std::vector< std::pair< int, node_direction > > weights_to_path( std::vector< float > const& weights ) const; int get_node_depth( branching_node* node ) const; + void update_value_to_mean_depth( branching_node* node ); + int get_possible_depth() const; }; -struct iid_node_dependence { - std::unordered_map< location_id, iid_dependence_props > id_to_equation_map; +struct iid_dependencies { + std::unordered_map< location_id, iid_node_dependence_props > id_to_equation_map; std::set< location_id > non_iid_nodes; void update_non_iid_nodes( sensitivity_analysis& sensitivity ); diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index b57f7161..f79e0eb8 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -1465,7 +1465,7 @@ void fuzzer::select_next_state() instrumentation::location_id loc_id(7); if (iid_dependences.id_to_equation_map.contains(loc_id)) { - const iid_dependence_props& props = iid_dependences.id_to_equation_map.at(loc_id); + const iid_node_dependence_props& props = iid_dependences.id_to_equation_map.at(loc_id); std::vector weights = props.approximate_matrix(); diff --git a/src/fuzzing/src/iid_node_dependencies.cpp b/src/fuzzing/src/iid_node_dependencies.cpp index 67b36514..a209bc87 100644 --- a/src/fuzzing/src/iid_node_dependencies.cpp +++ b/src/fuzzing/src/iid_node_dependencies.cpp @@ -3,12 +3,12 @@ #include /** - * @brief Three-way comparison for node_navigation. + * @brief Three-way comparison for node_direction. * - * @param other The other node_navigation object to compare with. + * @param other The other node_direction object to compare with. * @return std::strong_ordering Result of the comparison. */ -auto fuzzing::node_navigation::operator<=>( node_navigation const& other ) const +auto fuzzing::node_direction::operator<=>( node_direction const& other ) const { if ( auto const cmp = node_id.id <=> other.node_id.id; cmp != 0 ) return cmp; @@ -18,14 +18,14 @@ auto fuzzing::node_navigation::operator<=>( node_navigation const& other ) const /** - * @brief Equality operator for node_navigation. + * @brief Equality operator for node_direction. * - * Compares two node_navigation objects for equality based on their node_id and direction. + * Compares two node_direction objects for equality based on their node_id and direction. * - * @param other The other node_navigation object to compare with. + * @param other The other node_direction object to compare with. * @return true if both node_id and direction are equal, false otherwise. */ -bool fuzzing::node_navigation::operator==( node_navigation const& other ) const +bool fuzzing::node_direction::operator==( node_direction const& other ) const { return node_id.id == other.node_id.id && direction == other.direction; } @@ -41,13 +41,13 @@ bool fuzzing::node_navigation::operator==( node_navigation const& other ) const * @param node A pointer to the branching node from which to start the path traversal. * @return true if the set of interesting nodes was updated, false otherwise. */ -bool fuzzing::iid_dependence_props::update_interesting_nodes( branching_node* node ) +bool fuzzing::iid_node_dependence_props::update_interesting_nodes( branching_node* node ) { TMPROF_BLOCK(); bool set_changed = false; - auto add_to_interesting = [ this, &set_changed ]( std::vector< node_navigation >& nodes, int i ) { + auto add_to_interesting = [ this, &set_changed ]( std::vector< node_direction >& nodes, int i ) { for ( ; i >= 0; --i ) { auto result = this->interesting_nodes.emplace( nodes[ i ] ); if ( result.second ) { @@ -57,8 +57,8 @@ bool fuzzing::iid_dependence_props::update_interesting_nodes( branching_node* no }; for ( const auto& end_node : all_paths ) { - std::vector< node_navigation > path_1 = get_path( node ); - std::vector< node_navigation > path_2 = get_path( end_node ); + std::vector< node_direction > path_1 = get_path( node ); + std::vector< node_direction > path_2 = get_path( end_node ); if ( path_1.empty() || path_2.empty() ) continue; @@ -84,23 +84,23 @@ bool fuzzing::iid_dependence_props::update_interesting_nodes( branching_node* no /** * @brief Retrieves the path of node navigations from the given branching node to the root. * - * This function constructs a vector of `node_navigation` objects representing the path - * from the specified branching node to the root node. Each `node_navigation` object + * This function constructs a vector of `node_direction` objects representing the path + * from the specified branching node to the root node. Each `node_direction` object * contains the location ID of the predecessor node and the direction to the current node. * * @param node A pointer to the starting branching node. - * @return A vector of `node_navigation` objects representing the path from the given node to the + * @return A vector of `node_direction` objects representing the path from the given node to the * root. */ -std::vector< fuzzing::node_navigation > fuzzing::iid_dependence_props::get_path( branching_node* node ) const +std::vector< fuzzing::node_direction > fuzzing::iid_node_dependence_props::get_path( branching_node* node ) const { - std::vector< node_navigation > path; + std::vector< node_direction > path; branching_node* current = node; while ( current != nullptr ) { branching_node* predecessor = current->predecessor; if ( predecessor != nullptr ) { - node_navigation nav = { predecessor->get_location_id(), + node_direction nav = { predecessor->get_location_id(), predecessor->successor_direction( current ) }; path.push_back( nav ); } @@ -120,7 +120,7 @@ std::vector< fuzzing::node_navigation > fuzzing::iid_dependence_props::get_path( * * @note The matrix clearing could be optimized in the future. */ -void fuzzing::iid_dependence_props::recompute_matrix() +void fuzzing::iid_node_dependence_props::recompute_matrix() { TMPROF_BLOCK(); @@ -144,18 +144,18 @@ void fuzzing::iid_dependence_props::recompute_matrix() * * @param path A pointer to the branching_node representing the path to be processed. */ -void fuzzing::iid_dependence_props::add_equation( branching_node* path ) +void fuzzing::iid_node_dependence_props::add_equation( branching_node* path ) { TMPROF_BLOCK(); - std::map< node_navigation, int > directions_in_path; - for ( const node_navigation& navigation : interesting_nodes ) { + std::map< node_direction, int > directions_in_path; + for ( const node_direction& navigation : interesting_nodes ) { directions_in_path[ navigation ] = 0; } - std::vector< node_navigation > path_nodes = get_path( path ); + std::vector< node_direction > path_nodes = get_path( path ); - for ( const node_navigation& nav : path_nodes ) { + for ( const node_direction& nav : path_nodes ) { if ( interesting_nodes.contains( nav ) ) { directions_in_path[ nav ]++; } @@ -179,12 +179,12 @@ void fuzzing::iid_dependence_props::add_equation( branching_node* path ) * * @return A vector of floats representing the optimized weights. */ -std::vector< float > fuzzing::iid_dependence_props::approximate_matrix() const +std::vector< float > fuzzing::iid_node_dependence_props::approximate_matrix() const { GradientDescent gd( matrix, best_values ); std::vector< float > weights = gd.optimize(); - std::vector< std::pair< int, node_navigation > > path = weights_to_path( weights ); + std::vector< std::pair< int, node_direction > > path = weights_to_path( weights ); for ( const auto& [ value, nav ] : path ) { std::cout << "Node ID: " << nav.node_id.id << ", Direction: " << nav.direction << ", Value: " << value << std::endl; @@ -194,17 +194,17 @@ std::vector< float > fuzzing::iid_dependence_props::approximate_matrix() const } -std::vector< std::pair< int, fuzzing::node_navigation > > -fuzzing::iid_dependence_props::weights_to_path( std::vector< float > const& weights ) const +std::vector< std::pair< int, fuzzing::node_direction > > +fuzzing::iid_node_dependence_props::weights_to_path( std::vector< float > const& weights ) const { - int path_size = 35; // Get this value from value_to_mean_depth + int path_size = get_possible_depth(); if ( path_size == 0 || weights.empty() ) { return {}; } float weights_sum = std::accumulate( weights.begin(), weights.end(), 0.0f ); - std::vector< std::pair< int, node_navigation > > path; + std::vector< std::pair< int, node_direction > > path; for ( int i = 0; i < weights.size(); ++i ) { float value = static_cast< float >( path_size ) * weights[ i ] / weights_sum; @@ -224,7 +224,7 @@ fuzzing::iid_dependence_props::weights_to_path( std::vector< float > const& weig * @param node A pointer to the branching_node whose depth is to be calculated. * @return The depth of the node as an integer. */ -int fuzzing::iid_dependence_props::get_node_depth( branching_node* node ) const +int fuzzing::iid_node_dependence_props::get_node_depth( branching_node* node ) const { int depth = 0; branching_node* current = node; @@ -251,14 +251,42 @@ int fuzzing::iid_dependence_props::get_node_depth( branching_node* node ) const * * @param node A pointer to the branching node whose mean depth value is to be updated. */ -void fuzzing::iid_dependence_props::update_value_to_mean_depth( branching_node* node ) +void fuzzing::iid_node_dependence_props::update_value_to_mean_depth( branching_node* node ) { int depth = get_node_depth( node ); - auto& [ current_mean, count ] = value_to_mean_depth[ node->best_coverage_value ]; - current_mean = current_mean + ( depth - current_mean ) / ++count; + iid_value_props& value_props = value_to_mean_depth[ node->best_coverage_value ]; + value_props.mean_depth = value_props.mean_depth + ( depth - value_props.mean_depth ) / ++value_props.value_counts; } +int fuzzing::iid_node_dependence_props::get_possible_depth() const +{ + if ( value_to_mean_depth.empty() ) { + return 0; + } + + if (value_to_mean_depth.size() == 1) { + return value_to_mean_depth.begin()->second.mean_depth; + } + + auto it = value_to_mean_depth.begin(); + int first_depth = it->second.mean_depth; + float first_value = it->first; + ++it; + int second_depth = it->second.mean_depth; + float second_value = it->first; + + if (first_value == second_value) { + return first_depth; + } + + float slope = static_cast(second_depth - first_depth) / (second_value - first_value); + float intercept = first_depth - slope * first_value; + + int result = static_cast(slope * 0 + intercept); + return result; +} + /** * @brief Updates the set of non-IID nodes based on sensitivity analysis. * @@ -271,7 +299,7 @@ void fuzzing::iid_dependence_props::update_value_to_mean_depth( branching_node* * @param sensitivity A reference to the sensitivity analysis object that * provides the changed nodes. */ -void fuzzing::iid_node_dependence::update_non_iid_nodes( sensitivity_analysis& sensitivity ) +void fuzzing::iid_dependencies::update_non_iid_nodes( sensitivity_analysis& sensitivity ) { for ( branching_node* node : sensitivity.get_changed_nodes() ) { if ( node->is_did_branching() ) { @@ -295,14 +323,14 @@ void fuzzing::iid_node_dependence::update_non_iid_nodes( sensitivity_analysis& s * * @param node A pointer to the branching node to be processed. */ -void fuzzing::iid_node_dependence::process_node_dependence( branching_node* node ) +void fuzzing::iid_dependencies::process_node_dependence( branching_node* node ) { TMPROF_BLOCK(); if ( non_iid_nodes.contains( node->get_location_id() ) ) return; - iid_dependence_props& props = id_to_equation_map[ node->get_location_id() ]; + iid_node_dependence_props& props = id_to_equation_map[ node->get_location_id() ]; props.all_paths.push_back( node ); From 9ecaa16e82bc04f3aec2ca141e05266652ec8c79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Mon, 30 Sep 2024 20:07:32 +0200 Subject: [PATCH 037/144] feat: compute node counts --- .../include/fuzzing/branching_node.hpp | 2 + src/fuzzing/include/fuzzing/fuzzer.hpp | 1 + .../include/fuzzing/iid_node_dependencies.hpp | 29 ++- src/fuzzing/src/branching_node.cpp | 7 + src/fuzzing/src/fuzzer.cpp | 33 +-- src/fuzzing/src/iid_node_dependencies.cpp | 198 ++++++++++-------- 6 files changed, 164 insertions(+), 106 deletions(-) diff --git a/src/fuzzing/include/fuzzing/branching_node.hpp b/src/fuzzing/include/fuzzing/branching_node.hpp index e1a2a2f7..bdaf863c 100644 --- a/src/fuzzing/include/fuzzing/branching_node.hpp +++ b/src/fuzzing/include/fuzzing/branching_node.hpp @@ -108,6 +108,8 @@ struct branching_node bool is_closed() const { return closed; } void set_closed(bool const state = true) { closed = state; } + int get_depth() const; + guid_type guid() const { return guid__; } location_id id; diff --git a/src/fuzzing/include/fuzzing/fuzzer.hpp b/src/fuzzing/include/fuzzing/fuzzer.hpp index e4f15ec0..29287b5a 100644 --- a/src/fuzzing/include/fuzzing/fuzzer.hpp +++ b/src/fuzzing/include/fuzzing/fuzzer.hpp @@ -294,6 +294,7 @@ struct fuzzer final void collect_iid_pivots_from_sensitivity_results(); void select_next_state(); branching_node* select_iid_coverage_target() const; + branching_node* select_iid_coverage_target_from_dependencies() const; void remove_leaf_branching_node(branching_node* node); bool apply_coverage_failures_with_hope(); diff --git a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp index 9fcc8af2..84fd46f6 100644 --- a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp +++ b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp @@ -36,13 +36,19 @@ struct node_direction { } }; +struct iid_value_props { + int value_counts; + float mean_depth; + std::map< node_direction, std::tuple > direction_statistics; -struct iid_node_dependence_props { - struct iid_value_props { - int value_counts; - float mean_depth; - }; + void process_node( branching_node* node ); + +private: + void update_mean_depth( branching_node* node ); + void update_direction_counts( branching_node* node ); +}; +struct iid_node_dependence_props { std::vector< branching_node* > all_paths; std::set< node_direction > interesting_nodes; std::vector< std::vector< float > > matrix; @@ -50,15 +56,14 @@ struct iid_node_dependence_props { std::map< float, iid_value_props, FloatCompare > value_to_mean_depth; bool update_interesting_nodes( branching_node* node ); - std::vector< node_direction > get_path( branching_node* node ) const; void recompute_matrix(); void add_equation( branching_node* path ); - std::vector< float > approximate_matrix() const; - std::vector< std::pair< int, node_direction > > - weights_to_path( std::vector< float > const& weights ) const; - int get_node_depth( branching_node* node ) const; + std::map< fuzzing::node_direction, int > generate_path() const; - void update_value_to_mean_depth( branching_node* node ); +private: + std::vector< float > approximate_matrix() const; + std::map< fuzzing::node_direction, int > weights_to_path( std::vector< float > const& weights ) const; + int get_possible_depth() const; }; @@ -69,4 +74,6 @@ struct iid_dependencies { void update_non_iid_nodes( sensitivity_analysis& sensitivity ); void process_node_dependence( branching_node* node ); }; + +std::vector< fuzzing::node_direction > get_path( branching_node* node ); } // namespace fuzzing \ No newline at end of file diff --git a/src/fuzzing/src/branching_node.cpp b/src/fuzzing/src/branching_node.cpp index 128456ce..d79d773a 100644 --- a/src/fuzzing/src/branching_node.cpp +++ b/src/fuzzing/src/branching_node.cpp @@ -3,6 +3,13 @@ namespace fuzzing { +int fuzzing::branching_node::get_depth() const +{ + int depth = 0; + for ( branching_node const* node = this; node != nullptr; node = node->predecessor ) + ++depth; + return depth; +} branching_node::guid_type branching_node::get_fresh_guid__() { diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index f79e0eb8..146bd3fd 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -1462,19 +1462,6 @@ void fuzzer::collect_iid_pivots_from_sensitivity_results() void fuzzer::select_next_state() { - instrumentation::location_id loc_id(7); - if (iid_dependences.id_to_equation_map.contains(loc_id)) - { - const iid_node_dependence_props& props = iid_dependences.id_to_equation_map.at(loc_id); - - std::vector weights = props.approximate_matrix(); - - // std::cout << "ID: " << loc_id.id << std::endl; - for (size_t i = 0; i < props.interesting_nodes.size() && i < weights.size(); ++i) { - std::cout << *std::next(props.interesting_nodes.begin(), i) << " : " << weights[i] << std::endl; - } - } - TMPROF_BLOCK(); INVARIANT(sensitivity.is_ready() && typed_minimization.is_ready() && minimization.is_ready() && bitshare.is_ready()); @@ -1543,6 +1530,10 @@ branching_node* fuzzer::select_iid_coverage_target() const if (iid_pivots.empty() || entry_branching->is_closed()) return nullptr; + + branching_node* possible_winner = select_iid_coverage_target_from_dependencies(); + if (possible_winner != nullptr) + return possible_winner; auto const it_loc = std::next( iid_pivots.begin(), @@ -1618,6 +1609,22 @@ branching_node* fuzzer::select_iid_coverage_target() const return winner; } +branching_node* fuzzer::select_iid_coverage_target_from_dependencies() const +{ + instrumentation::location_id loc_id(7); + if (!iid_dependences.id_to_equation_map.contains(loc_id)) { + return nullptr; + } + + const iid_node_dependence_props& props = iid_dependences.id_to_equation_map.at(loc_id); + std::map< fuzzing::node_direction, int > path = props.generate_path(); + + for (const auto& [direction, count] : path) { + std::cout << "Node ID: " << direction.node_id.id << ", Direction: " << direction.direction << ", Count: " << count << std::endl; + } + + return nullptr; +} void fuzzer::remove_leaf_branching_node(branching_node* node) { diff --git a/src/fuzzing/src/iid_node_dependencies.cpp b/src/fuzzing/src/iid_node_dependencies.cpp index a209bc87..3ff2852a 100644 --- a/src/fuzzing/src/iid_node_dependencies.cpp +++ b/src/fuzzing/src/iid_node_dependencies.cpp @@ -2,6 +2,62 @@ #include #include + +/** + * @brief Processes a branching node to update its properties. + * + * This function updates the mean depth and direction counts of the given branching node. + * + * @param node A pointer to the branching node to be processed. + */ +void fuzzing::iid_value_props::process_node( branching_node* node ) +{ + update_mean_depth( node ); + update_direction_counts( node ); +} + + +/** + * @brief Updates the mean depth of a branching node. + * + * This function recalculates the mean depth of a branching node using the + * incremental average formula. It retrieves the depth of the provided node + * and updates the mean depth accordingly. + * + * @param node A pointer to the branching_node whose depth is to be used + * for updating the mean depth. + */ +void fuzzing::iid_value_props::update_mean_depth( branching_node* node ) +{ + int depth = node->get_depth(); + mean_depth = mean_depth + ( depth - mean_depth ) / ++value_counts; +} + + +/** + * @brief Updates the direction counts for a given branching node. + * + * This function calculates the direction counts for the path of the given branching node + * and updates the minimum and maximum statistics for each direction. + * + * @param node A pointer to the branching node for which the direction counts are to be updated. + */ +void fuzzing::iid_value_props::update_direction_counts( branching_node* node ) +{ + std::vector< node_direction > path = get_path( node ); + + std::map< node_direction, int > direction_counts; + for ( const node_direction& nav : path ) { + direction_counts[ nav ]++; + } + + for ( auto& [ direction, stats ] : direction_statistics ) { + std::get< 0 >( stats ) = std::min(direction_counts[ direction ], std::get< 0 >( stats )); + std::get< 1 >( stats ) = std::max(direction_counts[ direction ], std::get< 1 >( stats )); + } +} + + /** * @brief Three-way comparison for node_direction. * @@ -81,36 +137,6 @@ bool fuzzing::iid_node_dependence_props::update_interesting_nodes( branching_nod } -/** - * @brief Retrieves the path of node navigations from the given branching node to the root. - * - * This function constructs a vector of `node_direction` objects representing the path - * from the specified branching node to the root node. Each `node_direction` object - * contains the location ID of the predecessor node and the direction to the current node. - * - * @param node A pointer to the starting branching node. - * @return A vector of `node_direction` objects representing the path from the given node to the - * root. - */ -std::vector< fuzzing::node_direction > fuzzing::iid_node_dependence_props::get_path( branching_node* node ) const -{ - std::vector< node_direction > path; - - branching_node* current = node; - while ( current != nullptr ) { - branching_node* predecessor = current->predecessor; - if ( predecessor != nullptr ) { - node_direction nav = { predecessor->get_location_id(), - predecessor->successor_direction( current ) }; - path.push_back( nav ); - } - current = predecessor; - } - - return path; -} - - /** * @brief Recomputes the matrix of IID dependence properties. * @@ -183,18 +209,11 @@ std::vector< float > fuzzing::iid_node_dependence_props::approximate_matrix() co { GradientDescent gd( matrix, best_values ); std::vector< float > weights = gd.optimize(); - - std::vector< std::pair< int, node_direction > > path = weights_to_path( weights ); - for ( const auto& [ value, nav ] : path ) { - std::cout << "Node ID: " << nav.node_id.id << ", Direction: " << nav.direction - << ", Value: " << value << std::endl; - } - return weights; } -std::vector< std::pair< int, fuzzing::node_direction > > +std::map< fuzzing::node_direction, int > fuzzing::iid_node_dependence_props::weights_to_path( std::vector< float > const& weights ) const { int path_size = get_possible_depth(); @@ -204,11 +223,11 @@ fuzzing::iid_node_dependence_props::weights_to_path( std::vector< float > const& } float weights_sum = std::accumulate( weights.begin(), weights.end(), 0.0f ); - std::vector< std::pair< int, node_direction > > path; + std::map< node_direction, int > path; for ( int i = 0; i < weights.size(); ++i ) { float value = static_cast< float >( path_size ) * weights[ i ] / weights_sum; - path.push_back( { static_cast< int >( value ), *std::next( interesting_nodes.begin(), i ) } ); + path[*std::next( interesting_nodes.begin(), i )] = static_cast< int >( value ); } return path; @@ -216,49 +235,15 @@ fuzzing::iid_node_dependence_props::weights_to_path( std::vector< float > const& /** - * @brief Calculates the depth of a given branching node in the tree. + * @brief Computes the possible depth based on the value to mean depth mapping. * - * This function traverses the tree from the given node to the root, - * counting the number of edges (or levels) to determine the depth of the node. + * This function calculates the possible depth by examining the `value_to_mean_depth` map. + * If the map is empty, it returns 0. If the map contains only one element, it returns the mean depth + * of that element. If the map contains more than one element, it calculates the depth using linear + * interpolation based on the first two elements in the map. * - * @param node A pointer to the branching_node whose depth is to be calculated. - * @return The depth of the node as an integer. + * @return The computed possible depth. */ -int fuzzing::iid_node_dependence_props::get_node_depth( branching_node* node ) const -{ - int depth = 0; - branching_node* current = node; - while ( current != nullptr ) { - current = current->predecessor; - ++depth; - } - - return depth; -} - - -/** - * @brief Updates the mean depth value for a given branching node. - * - * This function calculates the depth of the provided branching node and updates - * the mean depth value associated with the node's best coverage value. The mean - * depth is updated incrementally using the formula: - * - * new_mean = current_mean + (depth - current_mean) / count - * - * where `depth` is the depth of the node, `current_mean` is the current mean depth, - * and `count` is the number of times the mean has been updated. - * - * @param node A pointer to the branching node whose mean depth value is to be updated. - */ -void fuzzing::iid_node_dependence_props::update_value_to_mean_depth( branching_node* node ) -{ - int depth = get_node_depth( node ); - iid_value_props& value_props = value_to_mean_depth[ node->best_coverage_value ]; - value_props.mean_depth = value_props.mean_depth + ( depth - value_props.mean_depth ) / ++value_props.value_counts; -} - - int fuzzing::iid_node_dependence_props::get_possible_depth() const { if ( value_to_mean_depth.empty() ) { @@ -287,6 +272,23 @@ int fuzzing::iid_node_dependence_props::get_possible_depth() const return result; } + +/** + * @brief Generates a path based on node dependencies. + * + * This function approximates a matrix to generate weights and then converts + * these weights into a path represented as a map of node directions to integers. + * + * @return A map where the keys are node directions and the values are integers + * representing the path. + */ +std::map< fuzzing::node_direction, int > fuzzing::iid_node_dependence_props::generate_path() const +{ + std::vector< float > weights = approximate_matrix(); + std::map< fuzzing::node_direction, int > path = weights_to_path( weights ); + return path; +} + /** * @brief Updates the set of non-IID nodes based on sensitivity analysis. * @@ -336,8 +338,40 @@ void fuzzing::iid_dependencies::process_node_dependence( branching_node* node ) if ( props.update_interesting_nodes( node ) ) { props.recompute_matrix(); + } else { + props.add_equation( node ); } - props.add_equation( node ); - props.update_value_to_mean_depth( node ); + iid_value_props& value_props = props.value_to_mean_depth[ node->best_coverage_value ]; + value_props.process_node( node ); +} + + +/** + * @brief Retrieves the path of node navigations from the given branching node to the root. + * + * This function constructs a vector of `node_direction` objects representing the path + * from the specified branching node to the root node. Each `node_direction` object + * contains the location ID of the predecessor node and the direction to the current node. + * + * @param node A pointer to the starting branching node. + * @return A vector of `node_direction` objects representing the path from the given node to the + * root. + */ +std::vector< fuzzing::node_direction > fuzzing::get_path( branching_node* node ) +{ + std::vector< node_direction > path; + + branching_node* current = node; + while ( current != nullptr ) { + branching_node* predecessor = current->predecessor; + if ( predecessor != nullptr ) { + node_direction nav = { predecessor->get_location_id(), + predecessor->successor_direction( current ) }; + path.push_back( nav ); + } + current = predecessor; + } + + return path; } \ No newline at end of file From 3355f43c40e117fc89285db4572b88d189a6c14d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Tue, 1 Oct 2024 09:50:40 +0200 Subject: [PATCH 038/144] feat: better mean computation --- .../include/fuzzing/iid_node_dependencies.hpp | 46 +++++++++++-- src/fuzzing/src/fuzzer.cpp | 14 +++- src/fuzzing/src/iid_node_dependencies.cpp | 65 ++++++++++--------- 3 files changed, 86 insertions(+), 39 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp index 84fd46f6..5fc304e8 100644 --- a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp +++ b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp @@ -21,6 +21,24 @@ struct FloatCompare { } }; + +struct Mean { +public: + float value; + + Mean() + : value( 0 ) + , count( 0 ) + {} + + void add( float new_value ) { value = value + ( new_value - value ) / ++count; } + + friend std::ostream& operator<<( std::ostream& os, Mean const& m ) { return os << m.value; } + +private: + size_t count; +}; + namespace fuzzing { struct node_direction { @@ -31,15 +49,29 @@ struct node_direction { bool operator==( node_direction const& other ) const; friend std::ostream& operator<<( std::ostream& os, node_direction const& nn ) { - os << nn.node_id.id << " " << ( nn.direction ? "right" : "left" ); - return os; + return os << nn.node_id.id << " " << ( nn.direction ? "right" : "left" ); + } +}; + +struct direction_statistics { + int min; + int max; + Mean mean; + + direction_statistics() + : min( std::numeric_limits< int >::max() ) + , max( std::numeric_limits< int >::min() ) + {} + + friend std::ostream& operator<<( std::ostream& os, direction_statistics const& ds ) + { + return os << "min: " << ds.min << " max: " << ds.max << " mean: " << ds.mean; } }; struct iid_value_props { - int value_counts; - float mean_depth; - std::map< node_direction, std::tuple > direction_statistics; + Mean depth; + std::map< node_direction, direction_statistics > direction_statistics; void process_node( branching_node* node ); @@ -53,7 +85,7 @@ struct iid_node_dependence_props { std::set< node_direction > interesting_nodes; std::vector< std::vector< float > > matrix; std::vector< float > best_values; - std::map< float, iid_value_props, FloatCompare > value_to_mean_depth; + std::map< float, iid_value_props, FloatCompare > best_value_props; bool update_interesting_nodes( branching_node* node ); void recompute_matrix(); @@ -63,7 +95,7 @@ struct iid_node_dependence_props { private: std::vector< float > approximate_matrix() const; std::map< fuzzing::node_direction, int > weights_to_path( std::vector< float > const& weights ) const; - + int get_possible_depth() const; }; diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index 146bd3fd..91145411 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -1620,7 +1620,19 @@ branching_node* fuzzer::select_iid_coverage_target_from_dependencies() const std::map< fuzzing::node_direction, int > path = props.generate_path(); for (const auto& [direction, count] : path) { - std::cout << "Node ID: " << direction.node_id.id << ", Direction: " << direction.direction << ", Count: " << count << std::endl; + // std::cout << "Node ID: " << direction.node_id.id << ", Direction: " << direction.direction << ", Count: " << count << std::endl; + } + + for (const auto&[value, value_props] : props.best_value_props) + { + std::cout << "Value: " << value << std::endl; + for (const auto& [direction, stats] : value_props.direction_statistics) + { + std::cout << "\t" << "Direction: " << direction << std::endl; + std::cout << "\t" << "\t" << "Min: " << stats.min << std::endl; + std::cout << "\t" << "\t" << "Average: " << stats.mean << std::endl; + std::cout << "\t" << "\t" << "Max: " << stats.max << std::endl; + } } return nullptr; diff --git a/src/fuzzing/src/iid_node_dependencies.cpp b/src/fuzzing/src/iid_node_dependencies.cpp index 3ff2852a..ba2f7294 100644 --- a/src/fuzzing/src/iid_node_dependencies.cpp +++ b/src/fuzzing/src/iid_node_dependencies.cpp @@ -5,12 +5,12 @@ /** * @brief Processes a branching node to update its properties. - * + * * This function updates the mean depth and direction counts of the given branching node. - * + * * @param node A pointer to the branching node to be processed. */ -void fuzzing::iid_value_props::process_node( branching_node* node ) +void fuzzing::iid_value_props::process_node( branching_node* node ) { update_mean_depth( node ); update_direction_counts( node ); @@ -20,17 +20,17 @@ void fuzzing::iid_value_props::process_node( branching_node* node ) /** * @brief Updates the mean depth of a branching node. * - * This function recalculates the mean depth of a branching node using the - * incremental average formula. It retrieves the depth of the provided node + * This function recalculates the mean depth of a branching node using the + * incremental average formula. It retrieves the depth of the provided node * and updates the mean depth accordingly. * - * @param node A pointer to the branching_node whose depth is to be used + * @param node A pointer to the branching_node whose depth is to be used * for updating the mean depth. */ void fuzzing::iid_value_props::update_mean_depth( branching_node* node ) { - int depth = node->get_depth(); - mean_depth = mean_depth + ( depth - mean_depth ) / ++value_counts; + int node_depth = node->get_depth(); + depth.add( node_depth ); } @@ -42,7 +42,7 @@ void fuzzing::iid_value_props::update_mean_depth( branching_node* node ) * * @param node A pointer to the branching node for which the direction counts are to be updated. */ -void fuzzing::iid_value_props::update_direction_counts( branching_node* node ) +void fuzzing::iid_value_props::update_direction_counts( branching_node* node ) { std::vector< node_direction > path = get_path( node ); @@ -51,9 +51,11 @@ void fuzzing::iid_value_props::update_direction_counts( branching_node* node ) direction_counts[ nav ]++; } - for ( auto& [ direction, stats ] : direction_statistics ) { - std::get< 0 >( stats ) = std::min(direction_counts[ direction ], std::get< 0 >( stats )); - std::get< 1 >( stats ) = std::max(direction_counts[ direction ], std::get< 1 >( stats )); + for ( auto& [ direction, count ] : direction_counts ) { + auto& stats = direction_statistics[ direction ]; + stats.min = std::min( count, stats.min ); + stats.max = std::max( count, stats.max ); + stats.mean.add( count ); } } @@ -227,7 +229,7 @@ fuzzing::iid_node_dependence_props::weights_to_path( std::vector< float > const& for ( int i = 0; i < weights.size(); ++i ) { float value = static_cast< float >( path_size ) * weights[ i ] / weights_sum; - path[*std::next( interesting_nodes.begin(), i )] = static_cast< int >( value ); + path[ *std::next( interesting_nodes.begin(), i ) ] = static_cast< int >( value ); } return path; @@ -237,38 +239,39 @@ fuzzing::iid_node_dependence_props::weights_to_path( std::vector< float > const& /** * @brief Computes the possible depth based on the value to mean depth mapping. * - * This function calculates the possible depth by examining the `value_to_mean_depth` map. - * If the map is empty, it returns 0. If the map contains only one element, it returns the mean depth - * of that element. If the map contains more than one element, it calculates the depth using linear - * interpolation based on the first two elements in the map. + * This function calculates the possible depth by examining the `best_value_props` map. + * If the map is empty, it returns 0. If the map contains only one element, it returns the mean + * depth of that element. If the map contains more than one element, it calculates the depth using + * linear interpolation based on the first two elements in the map. * * @return The computed possible depth. */ -int fuzzing::iid_node_dependence_props::get_possible_depth() const +int fuzzing::iid_node_dependence_props::get_possible_depth() const { - if ( value_to_mean_depth.empty() ) { + if ( best_value_props.empty() ) { return 0; } - if (value_to_mean_depth.size() == 1) { - return value_to_mean_depth.begin()->second.mean_depth; + if ( best_value_props.size() == 1 ) { + return best_value_props.begin()->second.depth.value; } - auto it = value_to_mean_depth.begin(); - int first_depth = it->second.mean_depth; + auto it = best_value_props.begin(); + int first_depth = it->second.depth.value; float first_value = it->first; + ++it; - int second_depth = it->second.mean_depth; + int second_depth = it->second.depth.value; float second_value = it->first; - if (first_value == second_value) { + if ( first_value == second_value ) { return first_depth; } - float slope = static_cast(second_depth - first_depth) / (second_value - first_value); + float slope = static_cast< float >( second_depth - first_depth ) / ( second_value - first_value ); float intercept = first_depth - slope * first_value; - int result = static_cast(slope * 0 + intercept); + int result = static_cast< int >( slope * 0 + intercept ); return result; } @@ -276,10 +279,10 @@ int fuzzing::iid_node_dependence_props::get_possible_depth() const /** * @brief Generates a path based on node dependencies. * - * This function approximates a matrix to generate weights and then converts + * This function approximates a matrix to generate weights and then converts * these weights into a path represented as a map of node directions to integers. * - * @return A map where the keys are node directions and the values are integers + * @return A map where the keys are node directions and the values are integers * representing the path. */ std::map< fuzzing::node_direction, int > fuzzing::iid_node_dependence_props::generate_path() const @@ -342,7 +345,7 @@ void fuzzing::iid_dependencies::process_node_dependence( branching_node* node ) props.add_equation( node ); } - iid_value_props& value_props = props.value_to_mean_depth[ node->best_coverage_value ]; + iid_value_props& value_props = props.best_value_props[ node->best_coverage_value ]; value_props.process_node( node ); } @@ -367,7 +370,7 @@ std::vector< fuzzing::node_direction > fuzzing::get_path( branching_node* node ) branching_node* predecessor = current->predecessor; if ( predecessor != nullptr ) { node_direction nav = { predecessor->get_location_id(), - predecessor->successor_direction( current ) }; + predecessor->successor_direction( current ) }; path.push_back( nav ); } current = predecessor; From 69f615b548b37aa693ed686990a5de7810e17588 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Wed, 2 Oct 2024 15:10:33 +0200 Subject: [PATCH 039/144] feat: path creation --- .../include/fuzzing/iid_node_dependencies.hpp | 6 + src/fuzzing/src/fuzzer.cpp | 17 +-- src/fuzzing/src/gradient_descent.cpp | 6 +- src/fuzzing/src/iid_node_dependencies.cpp | 142 ++++++++++++------ 4 files changed, 104 insertions(+), 67 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp index 5fc304e8..93c1417d 100644 --- a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp +++ b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp @@ -41,6 +41,11 @@ struct Mean { namespace fuzzing { +struct path_part { + +} + + struct node_direction { location_id node_id; bool direction; @@ -85,6 +90,7 @@ struct iid_node_dependence_props { std::set< node_direction > interesting_nodes; std::vector< std::vector< float > > matrix; std::vector< float > best_values; + iid_value_props all_value_props; std::map< float, iid_value_props, FloatCompare > best_value_props; bool update_interesting_nodes( branching_node* node ); diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index 91145411..c3fe181e 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -1619,22 +1619,7 @@ branching_node* fuzzer::select_iid_coverage_target_from_dependencies() const const iid_node_dependence_props& props = iid_dependences.id_to_equation_map.at(loc_id); std::map< fuzzing::node_direction, int > path = props.generate_path(); - for (const auto& [direction, count] : path) { - // std::cout << "Node ID: " << direction.node_id.id << ", Direction: " << direction.direction << ", Count: " << count << std::endl; - } - - for (const auto&[value, value_props] : props.best_value_props) - { - std::cout << "Value: " << value << std::endl; - for (const auto& [direction, stats] : value_props.direction_statistics) - { - std::cout << "\t" << "Direction: " << direction << std::endl; - std::cout << "\t" << "\t" << "Min: " << stats.min << std::endl; - std::cout << "\t" << "\t" << "Average: " << stats.mean << std::endl; - std::cout << "\t" << "\t" << "Max: " << stats.max << std::endl; - } - } - + branching_node* node = entry_branching; return nullptr; } diff --git a/src/fuzzing/src/gradient_descent.cpp b/src/fuzzing/src/gradient_descent.cpp index ce90cbb9..7de07170 100644 --- a/src/fuzzing/src/gradient_descent.cpp +++ b/src/fuzzing/src/gradient_descent.cpp @@ -14,7 +14,7 @@ * @param learning_rate The learning rate for gradient descent (default: 0.001). * @param max_iterations The maximum number of iterations (default: 10000). * @param convergence_threshold The threshold for convergence (default: 1e-6). - * @throws std::invalid_argument if input dimensions are invalid. + * @throws `std::invalid_argument` if input dimensions are invalid. */ GradientDescent::GradientDescent( const std::vector< std::vector< float > >& coefficient_matrix, const std::vector< float >& target_vector, @@ -73,8 +73,8 @@ std::vector< float > GradientDescent::optimize() prev_cost = current_cost; } - add_smallest_value( current_solution ); - rescale( current_solution, 0.0f, 100.0f ); + // add_smallest_value( current_solution ); + rescale( current_solution, 0.0f, 1.0f ); return current_solution; } diff --git a/src/fuzzing/src/iid_node_dependencies.cpp b/src/fuzzing/src/iid_node_dependencies.cpp index ba2f7294..e920d733 100644 --- a/src/fuzzing/src/iid_node_dependencies.cpp +++ b/src/fuzzing/src/iid_node_dependencies.cpp @@ -3,6 +3,35 @@ #include +/** + * @brief Three-way comparison for node_direction. + * + * @param other The other node_direction object to compare with. + * @return std::strong_ordering Result of the comparison. + */ +auto fuzzing::node_direction::operator<=>( node_direction const& other ) const +{ + if ( auto const cmp = node_id.id <=> other.node_id.id; cmp != 0 ) + return cmp; + + return direction <=> other.direction; +} + + +/** + * @brief Equality operator for node_direction. + * + * Compares two node_direction objects for equality based on their node_id and direction. + * + * @param other The other node_direction object to compare with. + * @return true if both node_id and direction are equal, false otherwise. + */ +bool fuzzing::node_direction::operator==( node_direction const& other ) const +{ + return node_id.id == other.node_id.id && direction == other.direction; +} + + /** * @brief Processes a branching node to update its properties. * @@ -60,35 +89,6 @@ void fuzzing::iid_value_props::update_direction_counts( branching_node* node ) } -/** - * @brief Three-way comparison for node_direction. - * - * @param other The other node_direction object to compare with. - * @return std::strong_ordering Result of the comparison. - */ -auto fuzzing::node_direction::operator<=>( node_direction const& other ) const -{ - if ( auto const cmp = node_id.id <=> other.node_id.id; cmp != 0 ) - return cmp; - - return direction <=> other.direction; -} - - -/** - * @brief Equality operator for node_direction. - * - * Compares two node_direction objects for equality based on their node_id and direction. - * - * @param other The other node_direction object to compare with. - * @return true if both node_id and direction are equal, false otherwise. - */ -bool fuzzing::node_direction::operator==( node_direction const& other ) const -{ - return node_id.id == other.node_id.id && direction == other.direction; -} - - /** * @brief Updates the set of interesting nodes based on the given branching node. * @@ -107,6 +107,10 @@ bool fuzzing::iid_node_dependence_props::update_interesting_nodes( branching_nod auto add_to_interesting = [ this, &set_changed ]( std::vector< node_direction >& nodes, int i ) { for ( ; i >= 0; --i ) { + direction_statistics& stats = all_value_props.direction_statistics[ nodes[ i ] ]; + if ( stats.min == stats.max ) + continue; + auto result = this->interesting_nodes.emplace( nodes[ i ] ); if ( result.second ) { set_changed = true; @@ -199,6 +203,56 @@ void fuzzing::iid_node_dependence_props::add_equation( branching_node* path ) } +/** + * @brief Generates a path based on node dependencies. + * + * This function approximates a matrix to generate weights and then converts + * these weights into a path represented as a map of node directions to integers. + * + * @return A map where the keys are node directions and the values are integers + * representing the path. + */ +std::map< fuzzing::node_direction, int > fuzzing::iid_node_dependence_props::generate_path() const +{ + std::vector< float > weights = approximate_matrix(); + + std::map< fuzzing::node_direction, int > path; + int path_size = 0; + int possible_depth = get_possible_depth(); + + for ( const auto& [ dir, stats ] : all_value_props.direction_statistics ) { + if ( stats.min == stats.max ) { + path[ dir ] = stats.min; + path_size += stats.min; + } + } + + int computed_size = 0; + std::map< fuzzing::node_direction, int > computed_path; + for ( size_t i = 0; i < interesting_nodes.size(); ++i ) { + const auto& node = *std::next( interesting_nodes.begin(), i ); + int max_count = all_value_props.direction_statistics.at( node ).max; + int computed_count = static_cast< int >( max_count * weights[ i ] ); + computed_size += computed_count; + computed_path[ node ] = computed_count; + } + + float scale = static_cast< float >( possible_depth - path_size ) / computed_size; + for ( auto& [ node, count ] : computed_path ) { + count = static_cast< int >( count * scale ); + } + + path.insert( computed_path.begin(), computed_path.end() ); + + for (const auto& [node, count] : path) { + // std::cout << "Node ID: " << node.node_id.id << ", Direction: " << node.direction + // << ", Count: " << count << std::endl; + } + + return path; +} + + /** * @brief Approximates a matrix using gradient descent optimization. * @@ -211,6 +265,14 @@ std::vector< float > fuzzing::iid_node_dependence_props::approximate_matrix() co { GradientDescent gd( matrix, best_values ); std::vector< float > weights = gd.optimize(); + + // std::cout << "Interesting nodes: " << std::endl; + for ( size_t i = 0; i < interesting_nodes.size(); ++i ) { + const auto& node = *std::next( interesting_nodes.begin(), i ); + // std::cout << "Node ID: " << node.node_id.id << ", Direction: " << node.direction + // << ", Weight: " << weights[ i ] << std::endl; + } + return weights; } @@ -276,22 +338,6 @@ int fuzzing::iid_node_dependence_props::get_possible_depth() const } -/** - * @brief Generates a path based on node dependencies. - * - * This function approximates a matrix to generate weights and then converts - * these weights into a path represented as a map of node directions to integers. - * - * @return A map where the keys are node directions and the values are integers - * representing the path. - */ -std::map< fuzzing::node_direction, int > fuzzing::iid_node_dependence_props::generate_path() const -{ - std::vector< float > weights = approximate_matrix(); - std::map< fuzzing::node_direction, int > path = weights_to_path( weights ); - return path; -} - /** * @brief Updates the set of non-IID nodes based on sensitivity analysis. * @@ -336,6 +382,9 @@ void fuzzing::iid_dependencies::process_node_dependence( branching_node* node ) return; iid_node_dependence_props& props = id_to_equation_map[ node->get_location_id() ]; + iid_value_props& value_props = props.best_value_props[ node->best_coverage_value ]; + value_props.process_node( node ); + props.all_value_props.process_node( node ); props.all_paths.push_back( node ); @@ -344,9 +393,6 @@ void fuzzing::iid_dependencies::process_node_dependence( branching_node* node ) } else { props.add_equation( node ); } - - iid_value_props& value_props = props.best_value_props[ node->best_coverage_value ]; - value_props.process_node( node ); } From 6316a9f4245a27ea5411264176efd8f1732520a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Thu, 3 Oct 2024 22:11:20 +0200 Subject: [PATCH 040/144] feat: path following --- .../include/fuzzing/iid_node_dependencies.hpp | 33 ++++++++-- .../include/fuzzing/progress_recorder.hpp | 7 ++- src/fuzzing/src/fuzzer.cpp | 15 ++++- src/fuzzing/src/iid_node_dependencies.cpp | 62 ++++++++++++++++--- src/fuzzing/src/progress_recorder.cpp | 4 +- 5 files changed, 104 insertions(+), 17 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp index 93c1417d..ae4730a1 100644 --- a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp +++ b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp @@ -41,9 +41,34 @@ struct Mean { namespace fuzzing { -struct path_part { - -} +struct path_decision { + int left_current; + int left_max; + + int right_current; + int right_max; + + path_decision( int left, int right ) + : left_current( 0 ) + , left_max( left ) + , right_current( 0 ) + , right_max( right ) + {} + + path_decision() + : left_current( 0 ) + , left_max( 0 ) + , right_current( 0 ) + , right_max( 0 ) + {} + + friend std::ostream& operator<<( std::ostream& os, path_decision const& pd ) + { + return os << " left: " << pd.left_current << "/" << pd.left_max << " right: " << pd.right_current << "/" << pd.right_max; + } + + bool get_next_direction(); +}; struct node_direction { @@ -96,7 +121,7 @@ struct iid_node_dependence_props { bool update_interesting_nodes( branching_node* node ); void recompute_matrix(); void add_equation( branching_node* path ); - std::map< fuzzing::node_direction, int > generate_path() const; + std::map< location_id, path_decision > generate_path() const; private: std::vector< float > approximate_matrix() const; diff --git a/src/fuzzing/include/fuzzing/progress_recorder.hpp b/src/fuzzing/include/fuzzing/progress_recorder.hpp index b9bc9aba..1b3e8899 100644 --- a/src/fuzzing/include/fuzzing/progress_recorder.hpp +++ b/src/fuzzing/include/fuzzing/progress_recorder.hpp @@ -94,7 +94,7 @@ struct progress_recorder PRIORITY_STEP_DISTANCE_TO_WIDTH = 5, PRIORITY_STEP_STDIN_SIZE = 6, PRIORITY_STEP_TRACE_INDEX = 7, - PRIORITY_STEP_SUCCESOR_TRACE_INDEX = 8, + PRIORITY_STEP_SUCCESSOR_TRACE_INDEX = 8, BEST_LOOP_HEAD = 9, BEST_SENSITIVE = 10, @@ -111,7 +111,10 @@ struct progress_recorder MONTE_CARLO_STEP_BACKWARD = 18, BEST_MONTE_CARLO_BACKWARD = 19, - STARTUP = 20, + DEPENDENCY_STEP = 20, + DEPENDENCY_END = 21, + + STARTUP = 22, }; void flush_node_choosing_data(); diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index c3fe181e..05ec9bc2 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -275,7 +275,7 @@ branching_node* fuzzer::primary_coverage_target_branchings::get_best( if (node->max_successors_trace_index > other.node->max_successors_trace_index) { - recorder().on_node_chosen(node, fuzzing::progress_recorder::PRIORITY_STEP_SUCCESOR_TRACE_INDEX); + recorder().on_node_chosen(node, fuzzing::progress_recorder::PRIORITY_STEP_SUCCESSOR_TRACE_INDEX); return true; } return false; @@ -1617,9 +1617,20 @@ branching_node* fuzzer::select_iid_coverage_target_from_dependencies() const } const iid_node_dependence_props& props = iid_dependences.id_to_equation_map.at(loc_id); - std::map< fuzzing::node_direction, int > path = props.generate_path(); + std::map< location_id, fuzzing::path_decision > path = props.generate_path(); branching_node* node = entry_branching; + while ( true ) { + bool direction = path[node->get_location_id()].get_next_direction(); + branching_node* next_node = node->successor(direction).pointer; + if (next_node == nullptr) { + break; + } + recorder().on_node_chosen(next_node, fuzzing::progress_recorder::DEPENDENCY_STEP); + node = next_node; + } + recorder().on_node_chosen(node, fuzzing::progress_recorder::DEPENDENCY_END); + return nullptr; } diff --git a/src/fuzzing/src/iid_node_dependencies.cpp b/src/fuzzing/src/iid_node_dependencies.cpp index e920d733..0f355386 100644 --- a/src/fuzzing/src/iid_node_dependencies.cpp +++ b/src/fuzzing/src/iid_node_dependencies.cpp @@ -3,6 +3,31 @@ #include +bool fuzzing::path_decision::get_next_direction() +{ + if ( left_max == 0) { + right_current++; + return true; + } + + if ( right_max == 0) { + left_current++; + return false; + } + + if ( right_max < left_max ) { + left_current++; + return false; + } + + if ( left_max < right_max ) { + right_current++; + return true; + } + + return false; +} + /** * @brief Three-way comparison for node_direction. * @@ -212,7 +237,7 @@ void fuzzing::iid_node_dependence_props::add_equation( branching_node* path ) * @return A map where the keys are node directions and the values are integers * representing the path. */ -std::map< fuzzing::node_direction, int > fuzzing::iid_node_dependence_props::generate_path() const +std::map< location_id, fuzzing::path_decision > fuzzing::iid_node_dependence_props::generate_path() const { std::vector< float > weights = approximate_matrix(); @@ -224,15 +249,16 @@ std::map< fuzzing::node_direction, int > fuzzing::iid_node_dependence_props::gen if ( stats.min == stats.max ) { path[ dir ] = stats.min; path_size += stats.min; - } + } } int computed_size = 0; std::map< fuzzing::node_direction, int > computed_path; for ( size_t i = 0; i < interesting_nodes.size(); ++i ) { const auto& node = *std::next( interesting_nodes.begin(), i ); - int max_count = all_value_props.direction_statistics.at( node ).max; + int max_count = all_value_props.direction_statistics.at( node ).mean.value; int computed_count = static_cast< int >( max_count * weights[ i ] ); + computed_count = std::max( 0, computed_count); computed_size += computed_count; computed_path[ node ] = computed_count; } @@ -244,12 +270,33 @@ std::map< fuzzing::node_direction, int > fuzzing::iid_node_dependence_props::gen path.insert( computed_path.begin(), computed_path.end() ); - for (const auto& [node, count] : path) { + std::map< location_id, path_decision > decisions; + for ( int i = 1; i < path.size(); ++i ) { + auto first_p = std::next( path.begin(), i - 1 ); + auto second_p = std::next( path.begin(), i ); + + if ( first_p->first.node_id == second_p->first.node_id ) { + decisions[ first_p->first.node_id ] = { first_p->second, second_p->second }; + ++i; + } else { + if ( first_p->first.direction ) { + decisions[ first_p->first.node_id ] = { 0, first_p->second }; + } else { + decisions[ first_p->first.node_id ] = { first_p->second, 0 }; + } + } + } + + for ( const auto& [location , decision] : decisions ) { + std::cout << location.id << decision << std::endl; + } + + for ( const auto& [ node, count ] : path ) { // std::cout << "Node ID: " << node.node_id.id << ", Direction: " << node.direction // << ", Count: " << count << std::endl; } - return path; + return decisions; } @@ -266,11 +313,10 @@ std::vector< float > fuzzing::iid_node_dependence_props::approximate_matrix() co GradientDescent gd( matrix, best_values ); std::vector< float > weights = gd.optimize(); - // std::cout << "Interesting nodes: " << std::endl; for ( size_t i = 0; i < interesting_nodes.size(); ++i ) { const auto& node = *std::next( interesting_nodes.begin(), i ); - // std::cout << "Node ID: " << node.node_id.id << ", Direction: " << node.direction - // << ", Weight: " << weights[ i ] << std::endl; + std::cout << "Node ID: " << node.node_id.id << ", Direction: " << node.direction + << ", Weight: " << weights[ i ] << std::endl; } return weights; diff --git a/src/fuzzing/src/progress_recorder.cpp b/src/fuzzing/src/progress_recorder.cpp index 4d25b55e..ecedca0b 100644 --- a/src/fuzzing/src/progress_recorder.cpp +++ b/src/fuzzing/src/progress_recorder.cpp @@ -943,7 +943,7 @@ void progress_recorder::node_chossing_data::save() const case SELECTION_REASON::PRIORITY_STEP_DISTANCE_TO_WIDTH: ostr << "PRIORITY_STEP_DISTANCE_TO_WIDTH"; break; case SELECTION_REASON::PRIORITY_STEP_STDIN_SIZE: ostr << "PRIORITY_STEP_STDIN_SIZE"; break; case SELECTION_REASON::PRIORITY_STEP_TRACE_INDEX: ostr << "PRIORITY_STEP_TRACE_INDEX"; break; - case SELECTION_REASON::PRIORITY_STEP_SUCCESOR_TRACE_INDEX: ostr << "PRIORITY_STEP_SUCCESOR_TRACE_INDEX"; break; + case SELECTION_REASON::PRIORITY_STEP_SUCCESSOR_TRACE_INDEX: ostr << "PRIORITY_STEP_SUCCESSOR_TRACE_INDEX"; break; case SELECTION_REASON::BEST_LOOP_HEAD: ostr << "BEST_LOOP_HEAD"; break; case SELECTION_REASON::BEST_SENSITIVE: ostr << "BEST_SENSITIVE"; break; case SELECTION_REASON::BEST_UNTOUCHED: ostr << "BEST_UNTOUCHED"; break; @@ -955,6 +955,8 @@ void progress_recorder::node_chossing_data::save() const case SELECTION_REASON::START_MONTE_CARLO_BACKWARD: ostr << "START_MONTE_CARLO_BACKWARD"; break; case SELECTION_REASON::MONTE_CARLO_STEP_BACKWARD: ostr << "MONTE_CARLO_STEP_BACKWARD"; break; case SELECTION_REASON::BEST_MONTE_CARLO_BACKWARD: ostr << "BEST_MONTE_CARLO_BACKWARD"; break; + case SELECTION_REASON::DEPENDENCY_STEP: ostr << "DEPENDENCY_STEP"; break; + case SELECTION_REASON::DEPENDENCY_END: ostr << "DEPENDENCY_END"; break; case SELECTION_REASON::STARTUP: ostr << "STARTUP"; break; default: UNREACHABLE(); break; } From c186bbb2d2e4a17caa268d6b6c58eed36a424144 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 12 Oct 2024 12:20:05 +0200 Subject: [PATCH 041/144] feat: update depth --- .../include/fuzzing/gradient_descent.hpp | 2 +- .../include/fuzzing/iid_node_dependencies.hpp | 18 +++- src/fuzzing/src/fuzzer.cpp | 12 ++- src/fuzzing/src/gradient_descent.cpp | 20 +++- src/fuzzing/src/iid_node_dependencies.cpp | 91 +++++++++++++------ 5 files changed, 103 insertions(+), 40 deletions(-) diff --git a/src/fuzzing/include/fuzzing/gradient_descent.hpp b/src/fuzzing/include/fuzzing/gradient_descent.hpp index 57db0487..3350b02c 100644 --- a/src/fuzzing/include/fuzzing/gradient_descent.hpp +++ b/src/fuzzing/include/fuzzing/gradient_descent.hpp @@ -33,7 +33,7 @@ class GradientDescent { float _learning_rate; int _max_iterations; float _convergence_threshold; - bool _debug = false; + bool _debug = true; std::vector< float > compute_gradient( const std::vector< float >& current_solution ); float compute_mean_squared_error( const std::vector< float >& current_solution ); diff --git a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp index ae4730a1..5bbfe82b 100644 --- a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp +++ b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp @@ -33,6 +33,10 @@ struct Mean { void add( float new_value ) { value = value + ( new_value - value ) / ++count; } + operator int() const { + return static_cast(value); + } + friend std::ostream& operator<<( std::ostream& os, Mean const& m ) { return os << m.value; } private: @@ -83,25 +87,27 @@ struct node_direction { } }; -struct direction_statistics { +struct number_statistics { int min; int max; Mean mean; - direction_statistics() + number_statistics() : min( std::numeric_limits< int >::max() ) , max( std::numeric_limits< int >::min() ) {} - friend std::ostream& operator<<( std::ostream& os, direction_statistics const& ds ) + friend std::ostream& operator<<( std::ostream& os, number_statistics const& ds ) { return os << "min: " << ds.min << " max: " << ds.max << " mean: " << ds.mean; } + + void add( int value ); }; struct iid_value_props { - Mean depth; - std::map< node_direction, direction_statistics > direction_statistics; + number_statistics depth; + std::map< node_direction, number_statistics > direction_statistics; void process_node( branching_node* node ); @@ -115,6 +121,7 @@ struct iid_node_dependence_props { std::set< node_direction > interesting_nodes; std::vector< std::vector< float > > matrix; std::vector< float > best_values; + iid_value_props all_value_props; std::map< float, iid_value_props, FloatCompare > best_value_props; @@ -139,4 +146,5 @@ struct iid_dependencies { }; std::vector< fuzzing::node_direction > get_path( branching_node* node ); +int linear_interpolation( int x1, int y1, int x2, int y2, int x ); } // namespace fuzzing \ No newline at end of file diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index 05ec9bc2..e168b2a4 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -1531,9 +1531,6 @@ branching_node* fuzzer::select_iid_coverage_target() const if (iid_pivots.empty() || entry_branching->is_closed()) return nullptr; - branching_node* possible_winner = select_iid_coverage_target_from_dependencies(); - if (possible_winner != nullptr) - return possible_winner; auto const it_loc = std::next( iid_pivots.begin(), @@ -1590,13 +1587,17 @@ branching_node* fuzzer::select_iid_coverage_target() const } else { - branching_node* const start_node = select_start_node_for_monte_carlo_search( + branching_node* start_node = select_start_node_for_monte_carlo_search( it_pivot->second.loop_boundaries, it_pivot->second.generator_for_start_node_selection, 0.75f, entry_branching ); + branching_node* possible_start = select_iid_coverage_target_from_dependencies(); + if (possible_start != nullptr) + start_node = possible_start; + recorder().on_node_chosen(start_node, fuzzing::progress_recorder::START_MONTE_CARLO); winner = monte_carlo_search(start_node, histogram, generators, *random_uniform_generator); @@ -1631,7 +1632,8 @@ branching_node* fuzzer::select_iid_coverage_target_from_dependencies() const } recorder().on_node_chosen(node, fuzzing::progress_recorder::DEPENDENCY_END); - return nullptr; + // return nullptr; + return node; } void fuzzer::remove_leaf_branching_node(branching_node* node) diff --git a/src/fuzzing/src/gradient_descent.cpp b/src/fuzzing/src/gradient_descent.cpp index 7de07170..a973d5fe 100644 --- a/src/fuzzing/src/gradient_descent.cpp +++ b/src/fuzzing/src/gradient_descent.cpp @@ -40,6 +40,16 @@ GradientDescent::GradientDescent( const std::vector< std::vector< float > >& coe */ std::vector< float > GradientDescent::optimize() { + if (_debug) { + std::cout << "Coefficient Matrix and Target Vector:" << std::endl; + for (size_t i = 0; i < _coefficient_matrix.size(); ++i) { + for (const auto& val : _coefficient_matrix[i]) { + std::cout << val << " "; + } + std::cout << "| " << _target_vector[i] << std::endl; + } + } + std::vector< float > current_solution = generate_random_weights( _coefficient_matrix[ 0 ].size() ); float prev_cost = std::numeric_limits< float >::max(); @@ -73,7 +83,7 @@ std::vector< float > GradientDescent::optimize() prev_cost = current_cost; } - // add_smallest_value( current_solution ); + add_smallest_value( current_solution ); rescale( current_solution, 0.0f, 1.0f ); return current_solution; } @@ -167,6 +177,10 @@ std::vector< float > GradientDescent::generate_random_weights( size_t n ) */ void GradientDescent::rescale( std::vector< float >& values, float min_value, float max_value ) { + for ( float& value : values ) { + std::cout << value << std::endl; + } + if ( values.empty() ) { return; } @@ -179,6 +193,10 @@ void GradientDescent::rescale( std::vector< float >& values, float min_value, fl ( value - *min_elem ) * ( max_value - min_value ) / ( *max_elem - *min_elem ); } ); } + + for ( float& value : values ) { + std::cout << value << std::endl; + } } /** diff --git a/src/fuzzing/src/iid_node_dependencies.cpp b/src/fuzzing/src/iid_node_dependencies.cpp index 0f355386..cb153817 100644 --- a/src/fuzzing/src/iid_node_dependencies.cpp +++ b/src/fuzzing/src/iid_node_dependencies.cpp @@ -3,14 +3,14 @@ #include -bool fuzzing::path_decision::get_next_direction() -{ - if ( left_max == 0) { +bool fuzzing::path_decision::get_next_direction() +{ + if ( left_max == 0 ) { right_current++; return true; } - if ( right_max == 0) { + if ( right_max == 0 ) { left_current++; return false; } @@ -57,6 +57,22 @@ bool fuzzing::node_direction::operator==( node_direction const& other ) const } +/** + * @brief Adds a value to the number statistics, updating the minimum, maximum, and mean. + * + * This function updates the minimum and maximum values if the provided value is lower or higher, respectively. + * It also adds the value to the mean calculation. + * + * @param value The integer value to be added to the statistics. + */ +void fuzzing::number_statistics::add( int value ) +{ + min = std::min( min, value ); + max = std::max( max, value ); + mean.add( value ); +} + + /** * @brief Processes a branching node to update its properties. * @@ -107,9 +123,7 @@ void fuzzing::iid_value_props::update_direction_counts( branching_node* node ) for ( auto& [ direction, count ] : direction_counts ) { auto& stats = direction_statistics[ direction ]; - stats.min = std::min( count, stats.min ); - stats.max = std::max( count, stats.max ); - stats.mean.add( count ); + stats.add( count ); } } @@ -132,7 +146,7 @@ bool fuzzing::iid_node_dependence_props::update_interesting_nodes( branching_nod auto add_to_interesting = [ this, &set_changed ]( std::vector< node_direction >& nodes, int i ) { for ( ; i >= 0; --i ) { - direction_statistics& stats = all_value_props.direction_statistics[ nodes[ i ] ]; + number_statistics& stats = all_value_props.direction_statistics[ nodes[ i ] ]; if ( stats.min == stats.max ) continue; @@ -256,17 +270,33 @@ std::map< location_id, fuzzing::path_decision > fuzzing::iid_node_dependence_pro std::map< fuzzing::node_direction, int > computed_path; for ( size_t i = 0; i < interesting_nodes.size(); ++i ) { const auto& node = *std::next( interesting_nodes.begin(), i ); - int max_count = all_value_props.direction_statistics.at( node ).mean.value; - int computed_count = static_cast< int >( max_count * weights[ i ] ); - computed_count = std::max( 0, computed_count); + auto it = best_value_props.begin(); + int x_1 = it->first; + int y_1 = it->second.direction_statistics.at( node ).mean; + ++it; + int x_2 = it->first; + int y_2 = it->second.direction_statistics.at( node ).mean; + + int interpolated_y = linear_interpolation( x_1, y_1, x_2, y_2, 0 ); + int computed_count = static_cast< int >( interpolated_y * weights[ i ] ); + computed_count = std::max( 0, computed_count ); computed_size += computed_count; computed_path[ node ] = computed_count; } - float scale = static_cast< float >( possible_depth - path_size ) / computed_size; - for ( auto& [ node, count ] : computed_path ) { - count = static_cast< int >( count * scale ); - } + // for ( size_t i = 0; i < interesting_nodes.size(); ++i ) { + // const auto& node = *std::next( interesting_nodes.begin(), i ); + // int max_count = all_value_props.number_statistics.at( node ).max; + // int computed_count = static_cast< int >( max_count * weights[ i ] ); + // computed_count = std::max( 0, computed_count); + // computed_size += computed_count; + // computed_path[ node ] = computed_count; + // } + + // float scale = static_cast< float >( possible_depth - path_size ) / computed_size; + // for ( auto& [ node, count ] : computed_path ) { + // count = static_cast< int >( count * scale ); + // } path.insert( computed_path.begin(), computed_path.end() ); @@ -287,7 +317,7 @@ std::map< location_id, fuzzing::path_decision > fuzzing::iid_node_dependence_pro } } - for ( const auto& [location , decision] : decisions ) { + for ( const auto& [ location, decision ] : decisions ) { std::cout << location.id << decision << std::endl; } @@ -315,8 +345,8 @@ std::vector< float > fuzzing::iid_node_dependence_props::approximate_matrix() co for ( size_t i = 0; i < interesting_nodes.size(); ++i ) { const auto& node = *std::next( interesting_nodes.begin(), i ); - std::cout << "Node ID: " << node.node_id.id << ", Direction: " << node.direction - << ", Weight: " << weights[ i ] << std::endl; + // std::cout << "Node ID: " << node.node_id.id << ", Direction: " << node.direction + // << ", Weight: " << weights[ i ] << std::endl; } return weights; @@ -361,29 +391,34 @@ int fuzzing::iid_node_dependence_props::get_possible_depth() const } if ( best_value_props.size() == 1 ) { - return best_value_props.begin()->second.depth.value; + return best_value_props.begin()->second.depth.mean; } auto it = best_value_props.begin(); - int first_depth = it->second.depth.value; + int first_depth = it->second.depth.mean; float first_value = it->first; ++it; - int second_depth = it->second.depth.value; + int second_depth = it->second.depth.mean; float second_value = it->first; - if ( first_value == second_value ) { - return first_depth; + return linear_interpolation( first_value, first_depth, second_value, second_depth, 0 ); +} + +int fuzzing::linear_interpolation( int x1, int y1, int x2, int y2, int x ) +{ + if ( x1 == x2 ) { + return y1; } - float slope = static_cast< float >( second_depth - first_depth ) / ( second_value - first_value ); - float intercept = first_depth - slope * first_value; + // Perform linear interpolation + double slope = static_cast< double >( y2 - y1 ) / ( x2 - x1 ); + double y = y1 + slope * ( x - x1 ); - int result = static_cast< int >( slope * 0 + intercept ); - return result; + // Round to nearest integer and return + return static_cast< int >( std::round( y ) ); } - /** * @brief Updates the set of non-IID nodes based on sensitivity analysis. * From 365156ebc34a52af4d870d80a5bbb164ba014d8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 12 Oct 2024 19:54:37 +0200 Subject: [PATCH 042/144] feat: change names --- .../include/fuzzing/iid_node_dependencies.hpp | 14 +-- src/fuzzing/src/iid_node_dependencies.cpp | 107 ++++++++++-------- 2 files changed, 64 insertions(+), 57 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp index 5bbfe82b..6f225f4c 100644 --- a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp +++ b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp @@ -33,10 +33,8 @@ struct Mean { void add( float new_value ) { value = value + ( new_value - value ) / ++count; } - operator int() const { - return static_cast(value); - } - + operator int() const { return static_cast(value); } + friend std::ostream& operator<<( std::ostream& os, Mean const& m ) { return os << m.value; } private: @@ -105,8 +103,8 @@ struct number_statistics { void add( int value ); }; -struct iid_value_props { - number_statistics depth; +struct coverage_value_props { + number_statistics path_depth; std::map< node_direction, number_statistics > direction_statistics; void process_node( branching_node* node ); @@ -122,8 +120,8 @@ struct iid_node_dependence_props { std::vector< std::vector< float > > matrix; std::vector< float > best_values; - iid_value_props all_value_props; - std::map< float, iid_value_props, FloatCompare > best_value_props; + coverage_value_props all_cov_value_props; + std::map< float, coverage_value_props, FloatCompare > cov_values_to_props; bool update_interesting_nodes( branching_node* node ); void recompute_matrix(); diff --git a/src/fuzzing/src/iid_node_dependencies.cpp b/src/fuzzing/src/iid_node_dependencies.cpp index cb153817..8b18f94a 100644 --- a/src/fuzzing/src/iid_node_dependencies.cpp +++ b/src/fuzzing/src/iid_node_dependencies.cpp @@ -59,10 +59,10 @@ bool fuzzing::node_direction::operator==( node_direction const& other ) const /** * @brief Adds a value to the number statistics, updating the minimum, maximum, and mean. - * - * This function updates the minimum and maximum values if the provided value is lower or higher, respectively. - * It also adds the value to the mean calculation. - * + * + * This function updates the minimum and maximum values if the provided value is lower or higher, + * respectively. It also adds the value to the mean calculation. + * * @param value The integer value to be added to the statistics. */ void fuzzing::number_statistics::add( int value ) @@ -80,7 +80,7 @@ void fuzzing::number_statistics::add( int value ) * * @param node A pointer to the branching node to be processed. */ -void fuzzing::iid_value_props::process_node( branching_node* node ) +void fuzzing::coverage_value_props::process_node( branching_node* node ) { update_mean_depth( node ); update_direction_counts( node ); @@ -97,10 +97,10 @@ void fuzzing::iid_value_props::process_node( branching_node* node ) * @param node A pointer to the branching_node whose depth is to be used * for updating the mean depth. */ -void fuzzing::iid_value_props::update_mean_depth( branching_node* node ) +void fuzzing::coverage_value_props::update_mean_depth( branching_node* node ) { int node_depth = node->get_depth(); - depth.add( node_depth ); + path_depth.add( node_depth ); } @@ -112,7 +112,7 @@ void fuzzing::iid_value_props::update_mean_depth( branching_node* node ) * * @param node A pointer to the branching node for which the direction counts are to be updated. */ -void fuzzing::iid_value_props::update_direction_counts( branching_node* node ) +void fuzzing::coverage_value_props::update_direction_counts( branching_node* node ) { std::vector< node_direction > path = get_path( node ); @@ -146,7 +146,7 @@ bool fuzzing::iid_node_dependence_props::update_interesting_nodes( branching_nod auto add_to_interesting = [ this, &set_changed ]( std::vector< node_direction >& nodes, int i ) { for ( ; i >= 0; --i ) { - number_statistics& stats = all_value_props.direction_statistics[ nodes[ i ] ]; + number_statistics& stats = all_cov_value_props.direction_statistics[ nodes[ i ] ]; if ( stats.min == stats.max ) continue; @@ -198,7 +198,7 @@ void fuzzing::iid_node_dependence_props::recompute_matrix() if ( all_paths.empty() ) return; - matrix.clear(); // This could be done better, but for now it's fine + matrix.clear(); best_values.clear(); for ( const auto& path : all_paths ) { @@ -259,7 +259,7 @@ std::map< location_id, fuzzing::path_decision > fuzzing::iid_node_dependence_pro int path_size = 0; int possible_depth = get_possible_depth(); - for ( const auto& [ dir, stats ] : all_value_props.direction_statistics ) { + for ( const auto& [ dir, stats ] : all_cov_value_props.direction_statistics ) { if ( stats.min == stats.max ) { path[ dir ] = stats.min; path_size += stats.min; @@ -270,7 +270,7 @@ std::map< location_id, fuzzing::path_decision > fuzzing::iid_node_dependence_pro std::map< fuzzing::node_direction, int > computed_path; for ( size_t i = 0; i < interesting_nodes.size(); ++i ) { const auto& node = *std::next( interesting_nodes.begin(), i ); - auto it = best_value_props.begin(); + auto it = cov_values_to_props.begin(); int x_1 = it->first; int y_1 = it->second.direction_statistics.at( node ).mean; ++it; @@ -286,17 +286,17 @@ std::map< location_id, fuzzing::path_decision > fuzzing::iid_node_dependence_pro // for ( size_t i = 0; i < interesting_nodes.size(); ++i ) { // const auto& node = *std::next( interesting_nodes.begin(), i ); - // int max_count = all_value_props.number_statistics.at( node ).max; + // int max_count = all_cov_value_props.number_statistics.at( node ).max; // int computed_count = static_cast< int >( max_count * weights[ i ] ); // computed_count = std::max( 0, computed_count); // computed_size += computed_count; // computed_path[ node ] = computed_count; // } - // float scale = static_cast< float >( possible_depth - path_size ) / computed_size; - // for ( auto& [ node, count ] : computed_path ) { - // count = static_cast< int >( count * scale ); - // } + float scale = static_cast< float >( possible_depth - path_size ) / computed_size; + for ( auto& [ node, count ] : computed_path ) { + count = static_cast< int >( count * scale ); + } path.insert( computed_path.begin(), computed_path.end() ); @@ -317,13 +317,17 @@ std::map< location_id, fuzzing::path_decision > fuzzing::iid_node_dependence_pro } } - for ( const auto& [ location, decision ] : decisions ) { - std::cout << location.id << decision << std::endl; + if ( false ) { + for ( const auto& [ location, decision ] : decisions ) { + std::cout << location.id << decision << std::endl; + } } - for ( const auto& [ node, count ] : path ) { - // std::cout << "Node ID: " << node.node_id.id << ", Direction: " << node.direction - // << ", Count: " << count << std::endl; + if ( false ) { + for ( const auto& [ node, count ] : path ) { + std::cout << "Node ID: " << node.node_id.id << ", Direction: " << node.direction + << ", Count: " << count << std::endl; + } } return decisions; @@ -343,10 +347,12 @@ std::vector< float > fuzzing::iid_node_dependence_props::approximate_matrix() co GradientDescent gd( matrix, best_values ); std::vector< float > weights = gd.optimize(); - for ( size_t i = 0; i < interesting_nodes.size(); ++i ) { - const auto& node = *std::next( interesting_nodes.begin(), i ); - // std::cout << "Node ID: " << node.node_id.id << ", Direction: " << node.direction - // << ", Weight: " << weights[ i ] << std::endl; + if ( false ) { + for ( size_t i = 0; i < interesting_nodes.size(); ++i ) { + const auto& node = *std::next( interesting_nodes.begin(), i ); + std::cout << "Node ID: " << node.node_id.id << ", Direction: " << node.direction + << ", Weight: " << weights[ i ] << std::endl; + } } return weights; @@ -377,7 +383,7 @@ fuzzing::iid_node_dependence_props::weights_to_path( std::vector< float > const& /** * @brief Computes the possible depth based on the value to mean depth mapping. * - * This function calculates the possible depth by examining the `best_value_props` map. + * This function calculates the possible depth by examining the `cov_values_to_props` map. * If the map is empty, it returns 0. If the map contains only one element, it returns the mean * depth of that element. If the map contains more than one element, it calculates the depth using * linear interpolation based on the first two elements in the map. @@ -386,38 +392,25 @@ fuzzing::iid_node_dependence_props::weights_to_path( std::vector< float > const& */ int fuzzing::iid_node_dependence_props::get_possible_depth() const { - if ( best_value_props.empty() ) { + if ( cov_values_to_props.empty() ) { return 0; } - if ( best_value_props.size() == 1 ) { - return best_value_props.begin()->second.depth.mean; + if ( cov_values_to_props.size() == 1 ) { + return cov_values_to_props.begin()->second.path_depth.min; } - auto it = best_value_props.begin(); - int first_depth = it->second.depth.mean; + auto it = cov_values_to_props.begin(); + int first_depth = it->second.path_depth.min; float first_value = it->first; ++it; - int second_depth = it->second.depth.mean; + int second_depth = it->second.path_depth.min; float second_value = it->first; return linear_interpolation( first_value, first_depth, second_value, second_depth, 0 ); } -int fuzzing::linear_interpolation( int x1, int y1, int x2, int y2, int x ) -{ - if ( x1 == x2 ) { - return y1; - } - - // Perform linear interpolation - double slope = static_cast< double >( y2 - y1 ) / ( x2 - x1 ); - double y = y1 + slope * ( x - x1 ); - - // Round to nearest integer and return - return static_cast< int >( std::round( y ) ); -} /** * @brief Updates the set of non-IID nodes based on sensitivity analysis. @@ -463,9 +456,10 @@ void fuzzing::iid_dependencies::process_node_dependence( branching_node* node ) return; iid_node_dependence_props& props = id_to_equation_map[ node->get_location_id() ]; - iid_value_props& value_props = props.best_value_props[ node->best_coverage_value ]; - value_props.process_node( node ); - props.all_value_props.process_node( node ); + + props.cov_values_to_props[ node->best_coverage_value ].process_node( node ); + ; + props.all_cov_value_props.process_node( node ); props.all_paths.push_back( node ); @@ -504,4 +498,19 @@ std::vector< fuzzing::node_direction > fuzzing::get_path( branching_node* node ) } return path; +} + + +int fuzzing::linear_interpolation( int x1, int y1, int x2, int y2, int x ) +{ + if ( x1 == x2 ) { + return y1; + } + + // Perform linear interpolation + double slope = static_cast< double >( y2 - y1 ) / ( x2 - x1 ); + double y = y1 + slope * ( x - x1 ); + + // Round to nearest integer and return + return static_cast< int >( std::round( y ) ); } \ No newline at end of file From 75ba932defaff6b83875db312227262ead05bb2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Tue, 15 Oct 2024 10:47:23 +0200 Subject: [PATCH 043/144] feat: add momentum parameter --- .../include/fuzzing/gradient_descent.hpp | 34 ++++++---- src/fuzzing/src/gradient_descent.cpp | 65 ++++++++++++++++--- 2 files changed, 77 insertions(+), 22 deletions(-) diff --git a/src/fuzzing/include/fuzzing/gradient_descent.hpp b/src/fuzzing/include/fuzzing/gradient_descent.hpp index 3350b02c..0e84b01e 100644 --- a/src/fuzzing/include/fuzzing/gradient_descent.hpp +++ b/src/fuzzing/include/fuzzing/gradient_descent.hpp @@ -2,20 +2,22 @@ #include +/* + # Improvements + - Learning rate optimization(Adaptive learning rate) + - Feature scaling + - Momentum + - Mini-batch gradient descent +*/ + class GradientDescent { - /* - # Improvements - - Learning rate optimization(Adaptive learning rate) - - Feature scaling - - Momentum - - Mini-batch gradient descent - */ public: - GradientDescent( const std::vector< std::vector< float > >& coefficient_matrix, - const std::vector< float >& target_vector, + GradientDescent( std::vector< std::vector< float > >& coefficient_matrix, + std::vector< float >& target_vector, float learning_rate = 0.001f, int max_iterations = 10000, - float convergence_threshold = 1e-6 ); + float convergence_threshold = 1e-6, + float momentum = 0.9f); std::vector< float > optimize(); @@ -26,14 +28,16 @@ class GradientDescent { { _convergence_threshold = convergence_threshold; } + void set_momentum(float momentum) { _momentum = momentum; } private: - const std::vector< std::vector< float > >& _coefficient_matrix; - const std::vector< float >& _target_vector; + std::vector< std::vector< float > >& _coefficient_matrix; + std::vector< float >& _target_vector; float _learning_rate; int _max_iterations; float _convergence_threshold; - bool _debug = true; + float _momentum; + bool _debug = false; std::vector< float > compute_gradient( const std::vector< float >& current_solution ); float compute_mean_squared_error( const std::vector< float >& current_solution ); @@ -41,4 +45,6 @@ class GradientDescent { static float dot_product( const std::vector< float >& a, const std::vector< float >& b ); static void rescale( std::vector< float >& values, float min_value, float max_value ); static void add_smallest_value( std::vector< float >& values ); -}; \ No newline at end of file + void min_max_normalize(std::vector>& matrix); + void min_max_normalize_target(std::vector& target); +}; diff --git a/src/fuzzing/src/gradient_descent.cpp b/src/fuzzing/src/gradient_descent.cpp index a973d5fe..6a20c801 100644 --- a/src/fuzzing/src/gradient_descent.cpp +++ b/src/fuzzing/src/gradient_descent.cpp @@ -14,18 +14,21 @@ * @param learning_rate The learning rate for gradient descent (default: 0.001). * @param max_iterations The maximum number of iterations (default: 10000). * @param convergence_threshold The threshold for convergence (default: 1e-6). + * @param momentum The momentum factor for gradient descent (default: 0.9). * @throws `std::invalid_argument` if input dimensions are invalid. */ -GradientDescent::GradientDescent( const std::vector< std::vector< float > >& coefficient_matrix, - const std::vector< float >& target_vector, +GradientDescent::GradientDescent( std::vector< std::vector< float > >& coefficient_matrix, + std::vector< float >& target_vector, float learning_rate, int max_iterations, - float convergence_threshold ) + float convergence_threshold, + float momentum ) : _coefficient_matrix( coefficient_matrix ) , _target_vector( target_vector ) , _learning_rate( learning_rate ) , _max_iterations( max_iterations ) , _convergence_threshold( convergence_threshold ) + , _momentum(momentum) { if ( coefficient_matrix.empty() || target_vector.empty() || coefficient_matrix.size() != target_vector.size() ) { @@ -34,12 +37,15 @@ GradientDescent::GradientDescent( const std::vector< std::vector< float > >& coe } /** - * @brief Performs gradient descent optimization. + * @brief Performs gradient descent optimization with momentum. * * @return std::vector The optimized solution vector. */ std::vector< float > GradientDescent::optimize() { + min_max_normalize(_coefficient_matrix); + min_max_normalize_target(_target_vector); + if (_debug) { std::cout << "Coefficient Matrix and Target Vector:" << std::endl; for (size_t i = 0; i < _coefficient_matrix.size(); ++i) { @@ -51,15 +57,17 @@ std::vector< float > GradientDescent::optimize() } std::vector< float > current_solution = generate_random_weights( _coefficient_matrix[ 0 ].size() ); + std::vector< float > velocity(current_solution.size(), 0.0f); // Initialize velocity to 0 float prev_cost = std::numeric_limits< float >::max(); for ( int iteration = 0; iteration < _max_iterations; ++iteration ) { std::vector< float > gradient = compute_gradient( current_solution ); - // Update current_solution + // Update current_solution with momentum for ( size_t i = 0; i < current_solution.size(); ++i ) { - current_solution[ i ] -= _learning_rate * gradient[ i ]; + velocity[i] = _momentum * velocity[i] - _learning_rate * gradient[i]; + current_solution[i] += velocity[i]; // Update with velocity } // Check for convergence @@ -83,11 +91,13 @@ std::vector< float > GradientDescent::optimize() prev_cost = current_cost; } - add_smallest_value( current_solution ); + // add_smallest_value( current_solution ); rescale( current_solution, 0.0f, 1.0f ); return current_solution; } + + /** * @brief Computes the gradient for the current solution. * @@ -194,9 +204,13 @@ void GradientDescent::rescale( std::vector< float >& values, float min_value, fl } ); } + std::cout << std::endl; + for ( float& value : values ) { std::cout << value << std::endl; } + std::cout << std::endl; + std::cout << std::endl; } /** @@ -217,4 +231,39 @@ void GradientDescent::add_smallest_value( std::vector< float >& values ) return value + std::abs( min_value ); } ); } -} \ No newline at end of file +} + +void GradientDescent::min_max_normalize_target(std::vector& target) { + float min_value = *std::min_element(target.begin(), target.end()); + float max_value = *std::max_element(target.begin(), target.end()); + + for (size_t i = 0; i < target.size(); ++i) { + if (max_value - min_value != 0) { + target[i] = (target[i] - min_value) / (max_value - min_value); + } else { + target[i] = 0.0f; // Handle case where all values are the same + } + } +} + +void GradientDescent::min_max_normalize(std::vector>& matrix) { + for (size_t col = 0; col < matrix[0].size(); ++col) { + // Find the min and max for the column + float min_value = std::numeric_limits::max(); + float max_value = std::numeric_limits::lowest(); + + for (size_t row = 0; row < matrix.size(); ++row) { + min_value = std::min(min_value, matrix[row][col]); + max_value = std::max(max_value, matrix[row][col]); + } + + // Normalize the column values + for (size_t row = 0; row < matrix.size(); ++row) { + if (max_value - min_value != 0) { + matrix[row][col] = (matrix[row][col] - min_value) / (max_value - min_value); + } else { + matrix[row][col] = 0.0f; // Handle case where all values are the same + } + } + } +} From 75c699c4fb3b11bb5d6def7f98940e5c62aa49c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Tue, 15 Oct 2024 11:39:16 +0200 Subject: [PATCH 044/144] feat: mini-batch gradient descent --- .../include/fuzzing/gradient_descent.hpp | 8 +- src/fuzzing/src/gradient_descent.cpp | 98 ++++++++++++++++--- 2 files changed, 89 insertions(+), 17 deletions(-) diff --git a/src/fuzzing/include/fuzzing/gradient_descent.hpp b/src/fuzzing/include/fuzzing/gradient_descent.hpp index 0e84b01e..e3bbf408 100644 --- a/src/fuzzing/include/fuzzing/gradient_descent.hpp +++ b/src/fuzzing/include/fuzzing/gradient_descent.hpp @@ -19,7 +19,7 @@ class GradientDescent { float convergence_threshold = 1e-6, float momentum = 0.9f); - std::vector< float > optimize(); + std::vector< float > optimize(int batch_size = 32); // Added batch_size parameter // Setters for easy modification of parameters void set_learning_rate( float learning_rate ) { _learning_rate = learning_rate; } @@ -47,4 +47,10 @@ class GradientDescent { static void add_smallest_value( std::vector< float >& values ); void min_max_normalize(std::vector>& matrix); void min_max_normalize_target(std::vector& target); + + // New methods for mini-batch gradient descent + void shuffle_data(); + template + std::vector get_batch(const std::vector& data, size_t start, size_t end); + std::vector compute_batch_gradient(const std::vector& current_solution, const std::vector>& batch_coefficients, const std::vector& batch_target); }; diff --git a/src/fuzzing/src/gradient_descent.cpp b/src/fuzzing/src/gradient_descent.cpp index 6a20c801..d6a28f14 100644 --- a/src/fuzzing/src/gradient_descent.cpp +++ b/src/fuzzing/src/gradient_descent.cpp @@ -37,11 +37,12 @@ GradientDescent::GradientDescent( std::vector< std::vector< float > >& coefficie } /** - * @brief Performs gradient descent optimization with momentum. + * @brief Performs mini-batch gradient descent optimization with momentum. * + * @param batch_size The size of each mini-batch. * @return std::vector The optimized solution vector. */ -std::vector< float > GradientDescent::optimize() +std::vector< float > GradientDescent::optimize(int batch_size) { min_max_normalize(_coefficient_matrix); min_max_normalize_target(_target_vector); @@ -50,7 +51,7 @@ std::vector< float > GradientDescent::optimize() std::cout << "Coefficient Matrix and Target Vector:" << std::endl; for (size_t i = 0; i < _coefficient_matrix.size(); ++i) { for (const auto& val : _coefficient_matrix[i]) { - std::cout << val << " "; + std::cout << val << " "; } std::cout << "| " << _target_vector[i] << std::endl; } @@ -60,42 +61,107 @@ std::vector< float > GradientDescent::optimize() std::vector< float > velocity(current_solution.size(), 0.0f); // Initialize velocity to 0 float prev_cost = std::numeric_limits< float >::max(); + size_t num_samples = _coefficient_matrix.size(); for ( int iteration = 0; iteration < _max_iterations; ++iteration ) { - std::vector< float > gradient = compute_gradient( current_solution ); + // Shuffle the dataset at the beginning of each epoch + shuffle_data(); + + // Mini-batch gradient descent loop + for (size_t batch_start = 0; batch_start < num_samples; batch_start += batch_size) { + size_t batch_end = std::min(batch_start + batch_size, num_samples); + std::vector> batch_coefficients = get_batch(_coefficient_matrix, batch_start, batch_end); + std::vector batch_target = get_batch(_target_vector, batch_start, batch_end); + + std::vector< float > gradient = compute_batch_gradient(current_solution, batch_coefficients, batch_target); - // Update current_solution with momentum - for ( size_t i = 0; i < current_solution.size(); ++i ) { - velocity[i] = _momentum * velocity[i] - _learning_rate * gradient[i]; - current_solution[i] += velocity[i]; // Update with velocity + // Update current_solution with momentum + for ( size_t i = 0; i < current_solution.size(); ++i ) { + velocity[i] = _momentum * velocity[i] - _learning_rate * gradient[i]; + current_solution[i] += velocity[i]; // Update with velocity + } } // Check for convergence - float current_cost = compute_mean_squared_error( current_solution ); + float current_cost = compute_mean_squared_error(current_solution); // Debug output if ( _debug && iteration % 100 == 0 ) { std::cout << "Iteration " << iteration << ", Cost: " << current_cost << std::endl; - std::cout << "Gradient: [ "; - for ( const auto& val : gradient ) { - std::cout << val << " "; - } - std::cout << "]" << std::endl; } if ( std::abs( current_cost - prev_cost ) < _convergence_threshold ) { std::cout << "Converged after " << iteration << " iterations." << std::endl; - std::cout << "Number of equations: " << _coefficient_matrix.size() << std::endl; break; } prev_cost = current_cost; } - // add_smallest_value( current_solution ); rescale( current_solution, 0.0f, 1.0f ); return current_solution; } +/** + * @brief Shuffles the dataset (coefficient matrix and target vector). + */ +void GradientDescent::shuffle_data() +{ + std::random_device rd; + std::mt19937 g(rd()); + + // Shuffle the coefficient matrix and target vector together + std::vector indices(_coefficient_matrix.size()); + std::iota(indices.begin(), indices.end(), 0); + std::shuffle(indices.begin(), indices.end(), g); + + std::vector> shuffled_matrix = _coefficient_matrix; + std::vector shuffled_target = _target_vector; + + for (size_t i = 0; i < indices.size(); ++i) { + shuffled_matrix[i] = _coefficient_matrix[indices[i]]; + shuffled_target[i] = _target_vector[indices[i]]; + } + + _coefficient_matrix = shuffled_matrix; + _target_vector = shuffled_target; +} + +/** + * @brief Retrieves a mini-batch from a matrix or vector. + */ +template +std::vector GradientDescent::get_batch(const std::vector& data, size_t start, size_t end) +{ + return std::vector(data.begin() + start, data.begin() + end); +} + +/** + * @brief Computes the gradient for a mini-batch. + * + * @param current_solution The current solution vector. + * @param batch_coefficients The coefficient matrix of the current batch. + * @param batch_target The target vector of the current batch. + * @return std::vector The computed gradient vector. + */ +std::vector GradientDescent::compute_batch_gradient(const std::vector& current_solution, const std::vector>& batch_coefficients, const std::vector& batch_target) +{ + std::vector< float > gradient( current_solution.size(), 0.0f ); + + for ( size_t i = 0; i < batch_coefficients.size(); ++i ) { + float predicted = dot_product( current_solution, batch_coefficients[ i ] ); + float error = predicted - batch_target[ i ]; + + for ( size_t j = 0; j < current_solution.size(); ++j ) { + gradient[ j ] += 2 * error * batch_coefficients[ i ][ j ]; + } + } + + for ( float& grad : gradient ) { + grad /= batch_coefficients.size(); + } + + return gradient; +} /** From da59f6a7bf7b512fb66163bd9e4a76953ba01263 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Wed, 16 Oct 2024 15:29:11 +0200 Subject: [PATCH 045/144] feat: changes from meeting --- src/fuzzing/include/fuzzing/fuzzer.hpp | 4 +- .../include/fuzzing/iid_node_dependencies.hpp | 34 +++-- src/fuzzing/src/fuzzer.cpp | 10 +- src/fuzzing/src/gradient_descent.cpp | 117 ++++++++---------- src/fuzzing/src/iid_node_dependencies.cpp | 61 ++++----- 5 files changed, 104 insertions(+), 122 deletions(-) diff --git a/src/fuzzing/include/fuzzing/fuzzer.hpp b/src/fuzzing/include/fuzzing/fuzzer.hpp index 29287b5a..957f3a56 100644 --- a/src/fuzzing/include/fuzzing/fuzzer.hpp +++ b/src/fuzzing/include/fuzzing/fuzzer.hpp @@ -293,8 +293,8 @@ struct fuzzer final void do_cleanup(); void collect_iid_pivots_from_sensitivity_results(); void select_next_state(); - branching_node* select_iid_coverage_target() const; - branching_node* select_iid_coverage_target_from_dependencies() const; + branching_node* select_iid_coverage_target(); + branching_node* select_iid_coverage_target_from_dependencies(); void remove_leaf_branching_node(branching_node* node); bool apply_coverage_failures_with_hope(); diff --git a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp index 6f225f4c..e190a96f 100644 --- a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp +++ b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp @@ -13,7 +13,7 @@ #include #include -struct FloatCompare { +struct FloatComparator { bool operator()( const float& a, const float& b ) const { const float epsilon = 1e-6f; @@ -22,20 +22,22 @@ struct FloatCompare { }; -struct Mean { -public: +struct Mean_counter { float value; - Mean() + Mean_counter() : value( 0 ) , count( 0 ) {} void add( float new_value ) { value = value + ( new_value - value ) / ++count; } - operator int() const { return static_cast(value); } - - friend std::ostream& operator<<( std::ostream& os, Mean const& m ) { return os << m.value; } + operator int() const { return static_cast< int >( value ); } + + friend std::ostream& operator<<( std::ostream& os, Mean_counter const& m ) + { + return os << m.value; + } private: size_t count; @@ -58,15 +60,13 @@ struct path_decision { {} path_decision() - : left_current( 0 ) - , left_max( 0 ) - , right_current( 0 ) - , right_max( 0 ) + : path_decision( 0, 0 ) {} friend std::ostream& operator<<( std::ostream& os, path_decision const& pd ) { - return os << " left: " << pd.left_current << "/" << pd.left_max << " right: " << pd.right_current << "/" << pd.right_max; + return os << " left: " << pd.left_current << "/" << pd.left_max + << " right: " << pd.right_current << "/" << pd.right_max; } bool get_next_direction(); @@ -88,7 +88,7 @@ struct node_direction { struct number_statistics { int min; int max; - Mean mean; + Mean_counter mean; number_statistics() : min( std::numeric_limits< int >::max() ) @@ -121,17 +121,15 @@ struct iid_node_dependence_props { std::vector< float > best_values; coverage_value_props all_cov_value_props; - std::map< float, coverage_value_props, FloatCompare > cov_values_to_props; + std::map< float, coverage_value_props, FloatComparator > cov_values_to_props; bool update_interesting_nodes( branching_node* node ); void recompute_matrix(); void add_equation( branching_node* path ); - std::map< location_id, path_decision > generate_path() const; + std::map< location_id, path_decision > generate_path(); private: - std::vector< float > approximate_matrix() const; - std::map< fuzzing::node_direction, int > weights_to_path( std::vector< float > const& weights ) const; - + std::vector< float > approximate_matrix(); int get_possible_depth() const; }; diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index e168b2a4..0b7f44eb 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -1524,7 +1524,7 @@ void fuzzer::select_next_state() } -branching_node* fuzzer::select_iid_coverage_target() const +branching_node* fuzzer::select_iid_coverage_target() { TMPROF_BLOCK(); @@ -1610,14 +1610,14 @@ branching_node* fuzzer::select_iid_coverage_target() const return winner; } -branching_node* fuzzer::select_iid_coverage_target_from_dependencies() const +branching_node* fuzzer::select_iid_coverage_target_from_dependencies() { instrumentation::location_id loc_id(7); if (!iid_dependences.id_to_equation_map.contains(loc_id)) { return nullptr; } - const iid_node_dependence_props& props = iid_dependences.id_to_equation_map.at(loc_id); + iid_node_dependence_props& props = iid_dependences.id_to_equation_map.at(loc_id); std::map< location_id, fuzzing::path_decision > path = props.generate_path(); branching_node* node = entry_branching; @@ -1632,8 +1632,8 @@ branching_node* fuzzer::select_iid_coverage_target_from_dependencies() const } recorder().on_node_chosen(node, fuzzing::progress_recorder::DEPENDENCY_END); - // return nullptr; - return node; + return nullptr; + // return node; } void fuzzer::remove_leaf_branching_node(branching_node* node) diff --git a/src/fuzzing/src/gradient_descent.cpp b/src/fuzzing/src/gradient_descent.cpp index d6a28f14..7da07715 100644 --- a/src/fuzzing/src/gradient_descent.cpp +++ b/src/fuzzing/src/gradient_descent.cpp @@ -28,7 +28,7 @@ GradientDescent::GradientDescent( std::vector< std::vector< float > >& coefficie , _learning_rate( learning_rate ) , _max_iterations( max_iterations ) , _convergence_threshold( convergence_threshold ) - , _momentum(momentum) + , _momentum( momentum ) { if ( coefficient_matrix.empty() || target_vector.empty() || coefficient_matrix.size() != target_vector.size() ) { @@ -42,48 +42,48 @@ GradientDescent::GradientDescent( std::vector< std::vector< float > >& coefficie * @param batch_size The size of each mini-batch. * @return std::vector The optimized solution vector. */ -std::vector< float > GradientDescent::optimize(int batch_size) +std::vector< float > GradientDescent::optimize( int batch_size ) { - min_max_normalize(_coefficient_matrix); - min_max_normalize_target(_target_vector); + // min_max_normalize( _coefficient_matrix ); + // min_max_normalize_target( _target_vector ); - if (_debug) { + if ( true ) { std::cout << "Coefficient Matrix and Target Vector:" << std::endl; - for (size_t i = 0; i < _coefficient_matrix.size(); ++i) { - for (const auto& val : _coefficient_matrix[i]) { + for ( size_t i = 0; i < _coefficient_matrix.size(); ++i ) { + for ( const auto& val : _coefficient_matrix[ i ] ) { std::cout << val << " "; } - std::cout << "| " << _target_vector[i] << std::endl; + std::cout << "| " << _target_vector[ i ] << std::endl; } } std::vector< float > current_solution = generate_random_weights( _coefficient_matrix[ 0 ].size() ); - std::vector< float > velocity(current_solution.size(), 0.0f); // Initialize velocity to 0 + std::vector< float > velocity( current_solution.size(), 0.0f ); float prev_cost = std::numeric_limits< float >::max(); size_t num_samples = _coefficient_matrix.size(); for ( int iteration = 0; iteration < _max_iterations; ++iteration ) { - // Shuffle the dataset at the beginning of each epoch shuffle_data(); - // Mini-batch gradient descent loop - for (size_t batch_start = 0; batch_start < num_samples; batch_start += batch_size) { - size_t batch_end = std::min(batch_start + batch_size, num_samples); - std::vector> batch_coefficients = get_batch(_coefficient_matrix, batch_start, batch_end); - std::vector batch_target = get_batch(_target_vector, batch_start, batch_end); + for ( size_t batch_start = 0; batch_start < num_samples; batch_start += batch_size ) { + size_t batch_end = std::min( batch_start + batch_size, num_samples ); + std::vector< std::vector< float > > batch_coefficients = + get_batch( _coefficient_matrix, batch_start, batch_end ); + std::vector< float > batch_target = get_batch( _target_vector, batch_start, batch_end ); - std::vector< float > gradient = compute_batch_gradient(current_solution, batch_coefficients, batch_target); + std::vector< float > gradient = + compute_batch_gradient( current_solution, batch_coefficients, batch_target ); // Update current_solution with momentum for ( size_t i = 0; i < current_solution.size(); ++i ) { - velocity[i] = _momentum * velocity[i] - _learning_rate * gradient[i]; - current_solution[i] += velocity[i]; // Update with velocity + velocity[ i ] = _momentum * velocity[ i ] - _learning_rate * gradient[ i ]; + current_solution[ i ] += velocity[ i ]; } } // Check for convergence - float current_cost = compute_mean_squared_error(current_solution); + float current_cost = compute_mean_squared_error( current_solution ); // Debug output if ( _debug && iteration % 100 == 0 ) { @@ -107,19 +107,19 @@ std::vector< float > GradientDescent::optimize(int batch_size) void GradientDescent::shuffle_data() { std::random_device rd; - std::mt19937 g(rd()); + std::mt19937 g( rd() ); // Shuffle the coefficient matrix and target vector together - std::vector indices(_coefficient_matrix.size()); - std::iota(indices.begin(), indices.end(), 0); - std::shuffle(indices.begin(), indices.end(), g); + std::vector< size_t > indices( _coefficient_matrix.size() ); + std::iota( indices.begin(), indices.end(), 0 ); + std::shuffle( indices.begin(), indices.end(), g ); - std::vector> shuffled_matrix = _coefficient_matrix; - std::vector shuffled_target = _target_vector; + std::vector< std::vector< float > > shuffled_matrix = _coefficient_matrix; + std::vector< float > shuffled_target = _target_vector; - for (size_t i = 0; i < indices.size(); ++i) { - shuffled_matrix[i] = _coefficient_matrix[indices[i]]; - shuffled_target[i] = _target_vector[indices[i]]; + for ( size_t i = 0; i < indices.size(); ++i ) { + shuffled_matrix[ i ] = _coefficient_matrix[ indices[ i ] ]; + shuffled_target[ i ] = _target_vector[ indices[ i ] ]; } _coefficient_matrix = shuffled_matrix; @@ -129,10 +129,10 @@ void GradientDescent::shuffle_data() /** * @brief Retrieves a mini-batch from a matrix or vector. */ -template -std::vector GradientDescent::get_batch(const std::vector& data, size_t start, size_t end) +template < typename T > +std::vector< T > GradientDescent::get_batch( const std::vector< T >& data, size_t start, size_t end ) { - return std::vector(data.begin() + start, data.begin() + end); + return std::vector< T >( data.begin() + start, data.begin() + end ); } /** @@ -143,7 +143,10 @@ std::vector GradientDescent::get_batch(const std::vector& data, size_t sta * @param batch_target The target vector of the current batch. * @return std::vector The computed gradient vector. */ -std::vector GradientDescent::compute_batch_gradient(const std::vector& current_solution, const std::vector>& batch_coefficients, const std::vector& batch_target) +std::vector< float > +GradientDescent::compute_batch_gradient( const std::vector< float >& current_solution, + const std::vector< std::vector< float > >& batch_coefficients, + const std::vector< float >& batch_target ) { std::vector< float > gradient( current_solution.size(), 0.0f ); @@ -253,10 +256,6 @@ std::vector< float > GradientDescent::generate_random_weights( size_t n ) */ void GradientDescent::rescale( std::vector< float >& values, float min_value, float max_value ) { - for ( float& value : values ) { - std::cout << value << std::endl; - } - if ( values.empty() ) { return; } @@ -269,14 +268,6 @@ void GradientDescent::rescale( std::vector< float >& values, float min_value, fl ( value - *min_elem ) * ( max_value - min_value ) / ( *max_elem - *min_elem ); } ); } - - std::cout << std::endl; - - for ( float& value : values ) { - std::cout << value << std::endl; - } - std::cout << std::endl; - std::cout << std::endl; } /** @@ -299,36 +290,38 @@ void GradientDescent::add_smallest_value( std::vector< float >& values ) } } -void GradientDescent::min_max_normalize_target(std::vector& target) { - float min_value = *std::min_element(target.begin(), target.end()); - float max_value = *std::max_element(target.begin(), target.end()); +void GradientDescent::min_max_normalize_target( std::vector< float >& target ) +{ + float min_value = *std::min_element( target.begin(), target.end() ); + float max_value = *std::max_element( target.begin(), target.end() ); - for (size_t i = 0; i < target.size(); ++i) { - if (max_value - min_value != 0) { - target[i] = (target[i] - min_value) / (max_value - min_value); + for ( size_t i = 0; i < target.size(); ++i ) { + if ( max_value - min_value != 0 ) { + target[ i ] = ( target[ i ] - min_value ) / ( max_value - min_value ); } else { - target[i] = 0.0f; // Handle case where all values are the same + target[ i ] = 0.0f; // Handle case where all values are the same } } } -void GradientDescent::min_max_normalize(std::vector>& matrix) { - for (size_t col = 0; col < matrix[0].size(); ++col) { +void GradientDescent::min_max_normalize( std::vector< std::vector< float > >& matrix ) +{ + for ( size_t col = 0; col < matrix[ 0 ].size(); ++col ) { // Find the min and max for the column - float min_value = std::numeric_limits::max(); - float max_value = std::numeric_limits::lowest(); + float min_value = std::numeric_limits< float >::max(); + float max_value = std::numeric_limits< float >::lowest(); - for (size_t row = 0; row < matrix.size(); ++row) { - min_value = std::min(min_value, matrix[row][col]); - max_value = std::max(max_value, matrix[row][col]); + for ( size_t row = 0; row < matrix.size(); ++row ) { + min_value = std::min( min_value, matrix[ row ][ col ] ); + max_value = std::max( max_value, matrix[ row ][ col ] ); } // Normalize the column values - for (size_t row = 0; row < matrix.size(); ++row) { - if (max_value - min_value != 0) { - matrix[row][col] = (matrix[row][col] - min_value) / (max_value - min_value); + for ( size_t row = 0; row < matrix.size(); ++row ) { + if ( max_value - min_value != 0 ) { + matrix[ row ][ col ] = ( matrix[ row ][ col ] - min_value ) / ( max_value - min_value ); } else { - matrix[row][col] = 0.0f; // Handle case where all values are the same + matrix[ row ][ col ] = 0.0f; // Handle case where all values are the same } } } diff --git a/src/fuzzing/src/iid_node_dependencies.cpp b/src/fuzzing/src/iid_node_dependencies.cpp index 8b18f94a..bfdbae5b 100644 --- a/src/fuzzing/src/iid_node_dependencies.cpp +++ b/src/fuzzing/src/iid_node_dependencies.cpp @@ -251,7 +251,7 @@ void fuzzing::iid_node_dependence_props::add_equation( branching_node* path ) * @return A map where the keys are node directions and the values are integers * representing the path. */ -std::map< location_id, fuzzing::path_decision > fuzzing::iid_node_dependence_props::generate_path() const +std::map< location_id, fuzzing::path_decision > fuzzing::iid_node_dependence_props::generate_path() { std::vector< float > weights = approximate_matrix(); @@ -266,36 +266,43 @@ std::map< location_id, fuzzing::path_decision > fuzzing::iid_node_dependence_pro } } + if ( true ) { + auto it = cov_values_to_props.begin(); + std::cout << "Path Depth" << it->second.path_depth << std::endl; + std::cout << "Closest value to 0: " << it->first << std::endl; + for (const auto& [direction, stats] : it->second.direction_statistics) { + std::cout << "Direction: " << direction << ", Min: " << stats.min + << ", Max: " << stats.max << ", Mean: " << stats.mean << std::endl; + } + } + int computed_size = 0; std::map< fuzzing::node_direction, int > computed_path; for ( size_t i = 0; i < interesting_nodes.size(); ++i ) { const auto& node = *std::next( interesting_nodes.begin(), i ); + auto it = cov_values_to_props.begin(); int x_1 = it->first; int y_1 = it->second.direction_statistics.at( node ).mean; + ++it; + int x_2 = it->first; int y_2 = it->second.direction_statistics.at( node ).mean; int interpolated_y = linear_interpolation( x_1, y_1, x_2, y_2, 0 ); int computed_count = static_cast< int >( interpolated_y * weights[ i ] ); + computed_count = interpolated_y; computed_count = std::max( 0, computed_count ); computed_size += computed_count; computed_path[ node ] = computed_count; } - // for ( size_t i = 0; i < interesting_nodes.size(); ++i ) { - // const auto& node = *std::next( interesting_nodes.begin(), i ); - // int max_count = all_cov_value_props.number_statistics.at( node ).max; - // int computed_count = static_cast< int >( max_count * weights[ i ] ); - // computed_count = std::max( 0, computed_count); - // computed_size += computed_count; - // computed_path[ node ] = computed_count; - // } - float scale = static_cast< float >( possible_depth - path_size ) / computed_size; for ( auto& [ node, count ] : computed_path ) { + // std::cout << "Count before scaling: " << count << std::endl; count = static_cast< int >( count * scale ); + // std::cout << "Count after scaling: " << count << std::endl; } path.insert( computed_path.begin(), computed_path.end() ); @@ -317,7 +324,7 @@ std::map< location_id, fuzzing::path_decision > fuzzing::iid_node_dependence_pro } } - if ( false ) { + if ( true ) { for ( const auto& [ location, decision ] : decisions ) { std::cout << location.id << decision << std::endl; } @@ -342,12 +349,12 @@ std::map< location_id, fuzzing::path_decision > fuzzing::iid_node_dependence_pro * * @return A vector of floats representing the optimized weights. */ -std::vector< float > fuzzing::iid_node_dependence_props::approximate_matrix() const +std::vector< float > fuzzing::iid_node_dependence_props::approximate_matrix() { GradientDescent gd( matrix, best_values ); std::vector< float > weights = gd.optimize(); - if ( false ) { + if ( true ) { for ( size_t i = 0; i < interesting_nodes.size(); ++i ) { const auto& node = *std::next( interesting_nodes.begin(), i ); std::cout << "Node ID: " << node.node_id.id << ", Direction: " << node.direction @@ -359,27 +366,6 @@ std::vector< float > fuzzing::iid_node_dependence_props::approximate_matrix() co } -std::map< fuzzing::node_direction, int > -fuzzing::iid_node_dependence_props::weights_to_path( std::vector< float > const& weights ) const -{ - int path_size = get_possible_depth(); - - if ( path_size == 0 || weights.empty() ) { - return {}; - } - - float weights_sum = std::accumulate( weights.begin(), weights.end(), 0.0f ); - std::map< node_direction, int > path; - - for ( int i = 0; i < weights.size(); ++i ) { - float value = static_cast< float >( path_size ) * weights[ i ] / weights_sum; - path[ *std::next( interesting_nodes.begin(), i ) ] = static_cast< int >( value ); - } - - return path; -} - - /** * @brief Computes the possible depth based on the value to mean depth mapping. * @@ -408,7 +394,12 @@ int fuzzing::iid_node_dependence_props::get_possible_depth() const int second_depth = it->second.path_depth.min; float second_value = it->first; - return linear_interpolation( first_value, first_depth, second_value, second_depth, 0 ); + int interpolated_depth = + linear_interpolation( first_value, first_depth, second_value, second_depth, 0 ); + if ( true ) + std::cout << "First Depth: " << first_depth << ", Second Depth: " << second_depth + << ", Interpolated Depth: " << interpolated_depth << std::endl; + return interpolated_depth; } From 8cb3e1f66f2563c787e4efc2bffdd8eaaeffd468 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Fri, 1 Nov 2024 12:53:35 +0100 Subject: [PATCH 046/144] feat: some changes --- src/fuzzing/include/fuzzing/fuzzer.hpp | 3 +- .../include/fuzzing/gradient_descent.hpp | 8 +- .../include/fuzzing/iid_node_dependencies.hpp | 9 +- src/fuzzing/src/gradient_descent.cpp | 15 +- src/fuzzing/src/iid_node_dependencies.cpp | 141 ++++++++++++++---- 5 files changed, 134 insertions(+), 42 deletions(-) diff --git a/src/fuzzing/include/fuzzing/fuzzer.hpp b/src/fuzzing/include/fuzzing/fuzzer.hpp index 957f3a56..48ba2f78 100644 --- a/src/fuzzing/include/fuzzing/fuzzer.hpp +++ b/src/fuzzing/include/fuzzing/fuzzer.hpp @@ -226,12 +226,13 @@ struct fuzzer final static std::unordered_set const& get_input_width_classes_set(); static natural_32_bit get_input_width_class(natural_32_bit num_input_bytes); static natural_32_bit get_input_width_class_index(natural_32_bit num_input_bytes); - +public: static void detect_loops_along_path_to_node( branching_node* const end_node, std::unordered_map >& loop_heads_to_bodies, std::vector* loops ); +private: static void compute_loop_boundaries( std::vector const& loops, std::vector& loop_boundaries diff --git a/src/fuzzing/include/fuzzing/gradient_descent.hpp b/src/fuzzing/include/fuzzing/gradient_descent.hpp index e3bbf408..c7b3c566 100644 --- a/src/fuzzing/include/fuzzing/gradient_descent.hpp +++ b/src/fuzzing/include/fuzzing/gradient_descent.hpp @@ -12,8 +12,8 @@ class GradientDescent { public: - GradientDescent( std::vector< std::vector< float > >& coefficient_matrix, - std::vector< float >& target_vector, + GradientDescent( const std::vector< std::vector< float > >& coefficient_matrix, + const std::vector< float >& target_vector, float learning_rate = 0.001f, int max_iterations = 10000, float convergence_threshold = 1e-6, @@ -31,8 +31,8 @@ class GradientDescent { void set_momentum(float momentum) { _momentum = momentum; } private: - std::vector< std::vector< float > >& _coefficient_matrix; - std::vector< float >& _target_vector; + std::vector< std::vector< float > > _coefficient_matrix; + std::vector< float > _target_vector; float _learning_rate; int _max_iterations; float _convergence_threshold; diff --git a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp index e190a96f..86320813 100644 --- a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp +++ b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp @@ -75,13 +75,13 @@ struct path_decision { struct node_direction { location_id node_id; - bool direction; + bool branching_direction; auto operator<=>( node_direction const& other ) const; bool operator==( node_direction const& other ) const; friend std::ostream& operator<<( std::ostream& os, node_direction const& nn ) { - return os << nn.node_id.id << " " << ( nn.direction ? "right" : "left" ); + return os << nn.node_id.id << " " << ( nn.branching_direction ? "right" : "left" ); } }; @@ -120,6 +120,8 @@ struct iid_node_dependence_props { std::vector< std::vector< float > > matrix; std::vector< float > best_values; + std::unordered_map< location_id, std::set< node_direction > > dependencies_by_loops; + coverage_value_props all_cov_value_props; std::map< float, coverage_value_props, FloatComparator > cov_values_to_props; @@ -131,6 +133,9 @@ struct iid_node_dependence_props { private: std::vector< float > approximate_matrix(); int get_possible_depth() const; + void dependencies_generation(); + std::vector< std::vector< float > > get_matrix( std::set< node_direction > const& subset ) const; + std::vector< std::set< node_direction > > get_subsets( std::set< node_direction > const& all_leafs ); }; struct iid_dependencies { diff --git a/src/fuzzing/src/gradient_descent.cpp b/src/fuzzing/src/gradient_descent.cpp index 7da07715..869b9582 100644 --- a/src/fuzzing/src/gradient_descent.cpp +++ b/src/fuzzing/src/gradient_descent.cpp @@ -17,8 +17,8 @@ * @param momentum The momentum factor for gradient descent (default: 0.9). * @throws `std::invalid_argument` if input dimensions are invalid. */ -GradientDescent::GradientDescent( std::vector< std::vector< float > >& coefficient_matrix, - std::vector< float >& target_vector, +GradientDescent::GradientDescent( const std::vector< std::vector< float > >& coefficient_matrix, + const std::vector< float >& target_vector, float learning_rate, int max_iterations, float convergence_threshold, @@ -34,6 +34,10 @@ GradientDescent::GradientDescent( std::vector< std::vector< float > >& coefficie coefficient_matrix.size() != target_vector.size() ) { throw std::invalid_argument( "Invalid input dimensions" ); } + + for (auto& row : _coefficient_matrix) { + row.push_back(1.0f); + } } /** @@ -44,10 +48,7 @@ GradientDescent::GradientDescent( std::vector< std::vector< float > >& coefficie */ std::vector< float > GradientDescent::optimize( int batch_size ) { - // min_max_normalize( _coefficient_matrix ); - // min_max_normalize_target( _target_vector ); - - if ( true ) { + if ( false ) { std::cout << "Coefficient Matrix and Target Vector:" << std::endl; for ( size_t i = 0; i < _coefficient_matrix.size(); ++i ) { for ( const auto& val : _coefficient_matrix[ i ] ) { @@ -97,7 +98,7 @@ std::vector< float > GradientDescent::optimize( int batch_size ) prev_cost = current_cost; } - rescale( current_solution, 0.0f, 1.0f ); + // rescale( current_solution, 0.0f, 1.0f ); return current_solution; } diff --git a/src/fuzzing/src/iid_node_dependencies.cpp b/src/fuzzing/src/iid_node_dependencies.cpp index bfdbae5b..f7dd4c68 100644 --- a/src/fuzzing/src/iid_node_dependencies.cpp +++ b/src/fuzzing/src/iid_node_dependencies.cpp @@ -1,8 +1,8 @@ +#include #include #include #include - bool fuzzing::path_decision::get_next_direction() { if ( left_max == 0 ) { @@ -39,7 +39,7 @@ auto fuzzing::node_direction::operator<=>( node_direction const& other ) const if ( auto const cmp = node_id.id <=> other.node_id.id; cmp != 0 ) return cmp; - return direction <=> other.direction; + return branching_direction <=> other.branching_direction; } @@ -53,7 +53,7 @@ auto fuzzing::node_direction::operator<=>( node_direction const& other ) const */ bool fuzzing::node_direction::operator==( node_direction const& other ) const { - return node_id.id == other.node_id.id && direction == other.direction; + return node_id.id == other.node_id.id && branching_direction == other.branching_direction; } @@ -242,17 +242,90 @@ void fuzzing::iid_node_dependence_props::add_equation( branching_node* path ) } -/** - * @brief Generates a path based on node dependencies. - * - * This function approximates a matrix to generate weights and then converts - * these weights into a path represented as a map of node directions to integers. - * - * @return A map where the keys are node directions and the values are integers - * representing the path. - */ +std::vector< std::set< fuzzing::node_direction > > +fuzzing::iid_node_dependence_props::get_subsets( std::set< node_direction > const& all_leafs ) +{ + std::vector< std::set< node_direction > > subsets; + std::vector< node_direction > leafs_vector( all_leafs.begin(), all_leafs.end() ); + int n = leafs_vector.size(); + + for ( int i = 1; i < ( 1 << n ); ++i ) { + std::set< node_direction > subset; + for ( int j = 0; j < n; ++j ) { + if ( i & ( 1 << j ) ) { + subset.insert( leafs_vector[ j ] ); + } + } + subsets.push_back( subset ); + } + + return subsets; +} + +std::vector< std::vector< float > > +fuzzing::iid_node_dependence_props::get_matrix( std::set< node_direction > const& subset ) const +{ + std::vector< std::vector< float > > sub_matrix; + for ( const auto& row : matrix ) { + std::vector< float > sub_row; + + for ( const auto& direction : subset ) { + int idx = std::distance( interesting_nodes.begin(), interesting_nodes.find( direction ) ); + sub_row.push_back( row[ idx ] ); + } + + sub_matrix.push_back( sub_row ); + } + + return sub_matrix; +} + +void fuzzing::iid_node_dependence_props::dependencies_generation() +{ + // std::cout << "Coefficient Matrix and Target Vector:" << std::endl; + // for ( size_t i = 0; i < matrix.size(); ++i ) { + // for ( const auto& val : matrix[ i ] ) { + // std::cout << val << " "; + // } + // std::cout << "| " << best_values[ i ] << std::endl; + // } + + std::set< node_direction > all_leafs; + for ( const auto& [ _, loop_bodies ] : dependencies_by_loops ) { + all_leafs.insert( loop_bodies.begin(), loop_bodies.end() ); + } + + std::vector< std::set< node_direction > > subsets = get_subsets( all_leafs ); + + for ( const auto& subset : subsets ) { + std::vector< std::vector< float > > sub_matrix = get_matrix( subset ); + GradientDescent gd( sub_matrix, best_values ); + std::vector< float > weights = gd.optimize(); + + if ( true ) { + for ( const auto& leaf : subset ) { + std::cout << "{Node ID: " << leaf.node_id.id + << ", Direction: " << leaf.branching_direction << "} "; + } + std::cout << std::endl; + + for ( size_t i = 0; i < subset.size(); ++i ) { + const auto& node = *std::next( subset.begin(), i ); + std::cout << "Node ID: " << node.node_id.id + << ", Direction: " << node.branching_direction + << ", Weight: " << weights[ i ] << std::endl; + } + + std::cout << "Last Weight: " << weights.back() << std::endl; + } + } +} + std::map< location_id, fuzzing::path_decision > fuzzing::iid_node_dependence_props::generate_path() { + dependencies_generation(); + return {}; + std::vector< float > weights = approximate_matrix(); std::map< fuzzing::node_direction, int > path; @@ -266,11 +339,11 @@ std::map< location_id, fuzzing::path_decision > fuzzing::iid_node_dependence_pro } } - if ( true ) { + if ( false ) { auto it = cov_values_to_props.begin(); std::cout << "Path Depth" << it->second.path_depth << std::endl; std::cout << "Closest value to 0: " << it->first << std::endl; - for (const auto& [direction, stats] : it->second.direction_statistics) { + for ( const auto& [ direction, stats ] : it->second.direction_statistics ) { std::cout << "Direction: " << direction << ", Min: " << stats.min << ", Max: " << stats.max << ", Mean: " << stats.mean << std::endl; } @@ -316,7 +389,7 @@ std::map< location_id, fuzzing::path_decision > fuzzing::iid_node_dependence_pro decisions[ first_p->first.node_id ] = { first_p->second, second_p->second }; ++i; } else { - if ( first_p->first.direction ) { + if ( first_p->first.branching_direction ) { decisions[ first_p->first.node_id ] = { 0, first_p->second }; } else { decisions[ first_p->first.node_id ] = { first_p->second, 0 }; @@ -324,7 +397,7 @@ std::map< location_id, fuzzing::path_decision > fuzzing::iid_node_dependence_pro } } - if ( true ) { + if ( false ) { for ( const auto& [ location, decision ] : decisions ) { std::cout << location.id << decision << std::endl; } @@ -332,7 +405,7 @@ std::map< location_id, fuzzing::path_decision > fuzzing::iid_node_dependence_pro if ( false ) { for ( const auto& [ node, count ] : path ) { - std::cout << "Node ID: " << node.node_id.id << ", Direction: " << node.direction + std::cout << "Node ID: " << node.node_id.id << ", Direction: " << node.branching_direction << ", Count: " << count << std::endl; } } @@ -354,10 +427,10 @@ std::vector< float > fuzzing::iid_node_dependence_props::approximate_matrix() GradientDescent gd( matrix, best_values ); std::vector< float > weights = gd.optimize(); - if ( true ) { + if ( false ) { for ( size_t i = 0; i < interesting_nodes.size(); ++i ) { const auto& node = *std::next( interesting_nodes.begin(), i ); - std::cout << "Node ID: " << node.node_id.id << ", Direction: " << node.direction + std::cout << "Node ID: " << node.node_id.id << ", Direction: " << node.branching_direction << ", Weight: " << weights[ i ] << std::endl; } } @@ -396,9 +469,7 @@ int fuzzing::iid_node_dependence_props::get_possible_depth() const int interpolated_depth = linear_interpolation( first_value, first_depth, second_value, second_depth, 0 ); - if ( true ) - std::cout << "First Depth: " << first_depth << ", Second Depth: " << second_depth - << ", Interpolated Depth: " << interpolated_depth << std::endl; + return interpolated_depth; } @@ -448,17 +519,31 @@ void fuzzing::iid_dependencies::process_node_dependence( branching_node* node ) iid_node_dependence_props& props = id_to_equation_map[ node->get_location_id() ]; + std::unordered_map< location_id, std::unordered_set< location_id > > loop_heads_to_bodies; + fuzzing::fuzzer::detect_loops_along_path_to_node( node, loop_heads_to_bodies, nullptr ); + + for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { + for ( const auto& body : loop_bodies ) { + props.dependencies_by_loops[ loop_head ].insert( { body, true } ); + props.dependencies_by_loops[ loop_head ].insert( { body, false } ); + } + } + props.cov_values_to_props[ node->best_coverage_value ].process_node( node ); - ; props.all_cov_value_props.process_node( node ); - props.all_paths.push_back( node ); - if ( props.update_interesting_nodes( node ) ) { - props.recompute_matrix(); - } else { - props.add_equation( node ); + std::vector< node_direction > path = get_path( node ); + for ( const node_direction& nav : path ) { + props.interesting_nodes.insert( nav ); } + props.recompute_matrix(); + + // if ( props.update_interesting_nodes( node ) ) { + // props.recompute_matrix(); + // } else { + // props.add_equation( node ); + // } } From 21305573dd4b891b2aa427cdc0441f9d97c1cbd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Mon, 4 Nov 2024 19:40:17 +0100 Subject: [PATCH 047/144] feat: dependencies_by_loading --- .../include/fuzzing/gradient_descent.hpp | 3 +- .../include/fuzzing/iid_node_dependencies.hpp | 10 ++ src/fuzzing/src/fuzzer.cpp | 6 +- src/fuzzing/src/gradient_descent.cpp | 8 +- src/fuzzing/src/iid_node_dependencies.cpp | 146 ++++++++++++++---- 5 files changed, 135 insertions(+), 38 deletions(-) diff --git a/src/fuzzing/include/fuzzing/gradient_descent.hpp b/src/fuzzing/include/fuzzing/gradient_descent.hpp index c7b3c566..92b1ba14 100644 --- a/src/fuzzing/include/fuzzing/gradient_descent.hpp +++ b/src/fuzzing/include/fuzzing/gradient_descent.hpp @@ -19,9 +19,8 @@ class GradientDescent { float convergence_threshold = 1e-6, float momentum = 0.9f); - std::vector< float > optimize(int batch_size = 32); // Added batch_size parameter + std::vector< float > optimize(int batch_size = 32); - // Setters for easy modification of parameters void set_learning_rate( float learning_rate ) { _learning_rate = learning_rate; } void set_max_iterations( int max_iterations ) { _max_iterations = max_iterations; } void set_convergence_threshold( float convergence_threshold ) diff --git a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp index 86320813..07670cbd 100644 --- a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp +++ b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp @@ -13,6 +13,9 @@ #include #include +using loop_to_bodies_t = std::unordered_map< location_id, std::unordered_set< location_id > >; +using loading_loops_t = std::unordered_map< location_id, std::tuple< natural_32_bit, natural_32_bit > >; + struct FloatComparator { bool operator()( const float& a, const float& b ) const { @@ -121,6 +124,7 @@ struct iid_node_dependence_props { std::vector< float > best_values; std::unordered_map< location_id, std::set< node_direction > > dependencies_by_loops; + std::unordered_map< location_id, std::set< node_direction > > dependencies_by_loading; coverage_value_props all_cov_value_props; std::map< float, coverage_value_props, FloatComparator > cov_values_to_props; @@ -130,10 +134,16 @@ struct iid_node_dependence_props { void add_equation( branching_node* path ); std::map< location_id, path_decision > generate_path(); + void compute_dependencies_by_loading( const loop_to_bodies_t& loop_heads_to_bodies, + branching_node* end_node ); + private: std::vector< float > approximate_matrix(); int get_possible_depth() const; void dependencies_generation(); + void print_dependencies(); + void compute_dependencies_by_loading( loading_loops_t& loading_loops, branching_node* end_node ); + std::vector< std::vector< float > > get_matrix( std::set< node_direction > const& subset ) const; std::vector< std::set< node_direction > > get_subsets( std::set< node_direction > const& all_leafs ); }; diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index 0b7f44eb..7042c48c 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -1612,10 +1612,12 @@ branching_node* fuzzer::select_iid_coverage_target() branching_node* fuzzer::select_iid_coverage_target_from_dependencies() { - instrumentation::location_id loc_id(7); - if (!iid_dependences.id_to_equation_map.contains(loc_id)) { + if (iid_dependences.id_to_equation_map.empty()) { return nullptr; } + + instrumentation::location_id loc_id = iid_dependences.id_to_equation_map.begin()->first; + iid_node_dependence_props& props = iid_dependences.id_to_equation_map.at(loc_id); std::map< location_id, fuzzing::path_decision > path = props.generate_path(); diff --git a/src/fuzzing/src/gradient_descent.cpp b/src/fuzzing/src/gradient_descent.cpp index 869b9582..d7526990 100644 --- a/src/fuzzing/src/gradient_descent.cpp +++ b/src/fuzzing/src/gradient_descent.cpp @@ -76,29 +76,27 @@ std::vector< float > GradientDescent::optimize( int batch_size ) std::vector< float > gradient = compute_batch_gradient( current_solution, batch_coefficients, batch_target ); - // Update current_solution with momentum for ( size_t i = 0; i < current_solution.size(); ++i ) { velocity[ i ] = _momentum * velocity[ i ] - _learning_rate * gradient[ i ]; current_solution[ i ] += velocity[ i ]; } } - // Check for convergence float current_cost = compute_mean_squared_error( current_solution ); - // Debug output if ( _debug && iteration % 100 == 0 ) { std::cout << "Iteration " << iteration << ", Cost: " << current_cost << std::endl; } if ( std::abs( current_cost - prev_cost ) < _convergence_threshold ) { - std::cout << "Converged after " << iteration << " iterations." << std::endl; + if ( _debug ) { + std::cout << "Converged after " << iteration << " iterations." << std::endl; + } break; } prev_cost = current_cost; } - // rescale( current_solution, 0.0f, 1.0f ); return current_solution; } diff --git a/src/fuzzing/src/iid_node_dependencies.cpp b/src/fuzzing/src/iid_node_dependencies.cpp index f7dd4c68..65e202ca 100644 --- a/src/fuzzing/src/iid_node_dependencies.cpp +++ b/src/fuzzing/src/iid_node_dependencies.cpp @@ -259,6 +259,12 @@ fuzzing::iid_node_dependence_props::get_subsets( std::set< node_direction > cons subsets.push_back( subset ); } + std::sort( subsets.begin(), + subsets.end(), + []( const std::set< node_direction >& a, const std::set< node_direction >& b ) { + return a.size() < b.size(); + } ); + return subsets; } @@ -280,16 +286,30 @@ fuzzing::iid_node_dependence_props::get_matrix( std::set< node_direction > const return sub_matrix; } -void fuzzing::iid_node_dependence_props::dependencies_generation() + +void fuzzing::iid_node_dependence_props::print_dependencies() { - // std::cout << "Coefficient Matrix and Target Vector:" << std::endl; - // for ( size_t i = 0; i < matrix.size(); ++i ) { - // for ( const auto& val : matrix[ i ] ) { - // std::cout << val << " "; - // } - // std::cout << "| " << best_values[ i ] << std::endl; - // } + std::cout << "# Dependencies:" << std::endl; + std::cout << "## Dependencies by loops:" << std::endl; + for ( const auto& [ loop, nodes ] : dependencies_by_loops ) { + for ( const auto& body : nodes ) { + std::cout << "- " << "`(" << body << ") → " << loop.id << "`" << std::endl; + } + } + + std::cout << "## Dependencies by sensitivity:" << std::endl; + std::cout << "## Dependencies by loading:" << std::endl; + for ( const auto& [ loading, nodes ] : dependencies_by_loading ) { + for ( const auto& body : nodes ) { + std::cout << "- " << "`(" << body << ") → " << loading.id << "`" << std::endl; + } + } +} + +void fuzzing::iid_node_dependence_props::dependencies_generation() +{ + print_dependencies(); std::set< node_direction > all_leafs; for ( const auto& [ _, loop_bodies ] : dependencies_by_loops ) { all_leafs.insert( loop_bodies.begin(), loop_bodies.end() ); @@ -297,28 +317,29 @@ void fuzzing::iid_node_dependence_props::dependencies_generation() std::vector< std::set< node_direction > > subsets = get_subsets( all_leafs ); + std::cout << "# Subsets:" << std::endl; for ( const auto& subset : subsets ) { std::vector< std::vector< float > > sub_matrix = get_matrix( subset ); GradientDescent gd( sub_matrix, best_values ); std::vector< float > weights = gd.optimize(); - if ( true ) { - for ( const auto& leaf : subset ) { - std::cout << "{Node ID: " << leaf.node_id.id - << ", Direction: " << leaf.branching_direction << "} "; - } - std::cout << std::endl; - - for ( size_t i = 0; i < subset.size(); ++i ) { - const auto& node = *std::next( subset.begin(), i ); - std::cout << "Node ID: " << node.node_id.id - << ", Direction: " << node.branching_direction - << ", Weight: " << weights[ i ] << std::endl; - } + std::cout << "## Subset of nodes: `{ "; + auto delimeter = ""; + for ( const auto& leaf : subset ) { + std::cout << delimeter << "(" << leaf << ")"; + delimeter = ", "; + } + std::cout << " }`" << std::endl; - std::cout << "Last Weight: " << weights.back() << std::endl; + for ( size_t i = 0; i < subset.size(); ++i ) { + const auto& node = *std::next( subset.begin(), i ); + std::cout << "- `(" << node << "): " << weights[ i ] << "`" << std::endl; } + + std::cout << "- `Increment: " << weights.back() << "`" << std::endl; } + + std::cout << std::endl; } std::map< location_id, fuzzing::path_decision > fuzzing::iid_node_dependence_props::generate_path() @@ -498,6 +519,67 @@ void fuzzing::iid_dependencies::update_non_iid_nodes( sensitivity_analysis& sens } } +void fuzzing::iid_node_dependence_props::compute_dependencies_by_loading( loading_loops_t& loading_loops, + branching_node* end_node ) +{ + for ( const auto& bit_index : end_node->sensitive_stdin_bits ) { + for ( const auto& [ loop_head, values ] : loading_loops ) { + auto& [ min, max ] = values; + + if ( bit_index >= min && bit_index <= max ) { + dependencies_by_loading[ loop_head ].insert( { end_node->get_location_id(), true } ); + dependencies_by_loading[ loop_head ].insert( { end_node->get_location_id(), false } ); + } + } + } +} + +void fuzzing::iid_node_dependence_props::compute_dependencies_by_loading( const loop_to_bodies_t& loop_heads_to_bodies, + branching_node* end_node ) +{ + branching_node* node = end_node; + loading_loops_t loading_loops; + + for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { + loading_loops[ loop_head ] = { std::numeric_limits< natural_32_bit >::max(), + std::numeric_limits< natural_32_bit >::min() }; + } + + while ( node != nullptr ) { + for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { + if (loop_head.id == node->get_location_id().id) { + natural_32_bit bits_count = node->get_num_stdin_bits(); + + auto& [ min, max ] = loading_loops[ loop_head ]; + min = std::min( min, bits_count ); + max = std::max( max, bits_count ); + } + } + + node = node->predecessor; + } + + node = end_node; + + while ( node != nullptr ) { + for ( const auto& bit_index : node->sensitive_stdin_bits ) { + for ( const auto& [ loop_head, values ] : loading_loops ) { + auto& [ min, max ] = values; + if ( bit_index >= min && bit_index <= max ) { + if ( interesting_nodes.contains( { node->get_location_id(), true } ) ) { + dependencies_by_loading[ loop_head ].insert( { node->get_location_id(), true } ); + } + + if ( interesting_nodes.contains( { node->get_location_id(), false } ) ) { + dependencies_by_loading[ loop_head ].insert( { node->get_location_id(), false } ); + } + } + } + } + + node = node->predecessor; + } +} /** * @brief Processes the dependence of a given branching node. @@ -518,14 +600,25 @@ void fuzzing::iid_dependencies::process_node_dependence( branching_node* node ) return; iid_node_dependence_props& props = id_to_equation_map[ node->get_location_id() ]; + std::vector< node_direction > path = get_path( node ); + for ( const node_direction& nav : path ) { + props.interesting_nodes.insert( nav ); + } + props.recompute_matrix(); std::unordered_map< location_id, std::unordered_set< location_id > > loop_heads_to_bodies; fuzzing::fuzzer::detect_loops_along_path_to_node( node, loop_heads_to_bodies, nullptr ); + props.compute_dependencies_by_loading( loop_heads_to_bodies, node ); for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { for ( const auto& body : loop_bodies ) { - props.dependencies_by_loops[ loop_head ].insert( { body, true } ); - props.dependencies_by_loops[ loop_head ].insert( { body, false } ); + if ( props.interesting_nodes.contains( { body, true } ) ) { + props.dependencies_by_loops[ loop_head ].insert( { body, true } ); + } + + if ( props.interesting_nodes.contains( { body, false } ) ) { + props.dependencies_by_loops[ loop_head ].insert( { body, false } ); + } } } @@ -533,11 +626,6 @@ void fuzzing::iid_dependencies::process_node_dependence( branching_node* node ) props.all_cov_value_props.process_node( node ); props.all_paths.push_back( node ); - std::vector< node_direction > path = get_path( node ); - for ( const node_direction& nav : path ) { - props.interesting_nodes.insert( nav ); - } - props.recompute_matrix(); // if ( props.update_interesting_nodes( node ) ) { // props.recompute_matrix(); From 2df405381fd987bcd648256519e1a95a9a532722 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Tue, 5 Nov 2024 13:32:00 +0100 Subject: [PATCH 048/144] feat: more outputting --- .../include/fuzzing/gradient_descent.hpp | 14 +-- src/fuzzing/src/gradient_descent.cpp | 113 ++++++++---------- src/fuzzing/src/iid_node_dependencies.cpp | 22 +++- 3 files changed, 71 insertions(+), 78 deletions(-) diff --git a/src/fuzzing/include/fuzzing/gradient_descent.hpp b/src/fuzzing/include/fuzzing/gradient_descent.hpp index 92b1ba14..b8d5a610 100644 --- a/src/fuzzing/include/fuzzing/gradient_descent.hpp +++ b/src/fuzzing/include/fuzzing/gradient_descent.hpp @@ -14,12 +14,12 @@ class GradientDescent { public: GradientDescent( const std::vector< std::vector< float > >& coefficient_matrix, const std::vector< float >& target_vector, - float learning_rate = 0.001f, + float learning_rate = 0.01f, int max_iterations = 10000, - float convergence_threshold = 1e-6, - float momentum = 0.9f); + float convergence_threshold = 1e-5, + float momentum = 0.8f); - std::vector< float > optimize(int batch_size = 32); + std::vector< float > optimize(); void set_learning_rate( float learning_rate ) { _learning_rate = learning_rate; } void set_max_iterations( int max_iterations ) { _max_iterations = max_iterations; } @@ -47,9 +47,5 @@ class GradientDescent { void min_max_normalize(std::vector>& matrix); void min_max_normalize_target(std::vector& target); - // New methods for mini-batch gradient descent void shuffle_data(); - template - std::vector get_batch(const std::vector& data, size_t start, size_t end); - std::vector compute_batch_gradient(const std::vector& current_solution, const std::vector>& batch_coefficients, const std::vector& batch_target); -}; +}; \ No newline at end of file diff --git a/src/fuzzing/src/gradient_descent.cpp b/src/fuzzing/src/gradient_descent.cpp index d7526990..f1c67366 100644 --- a/src/fuzzing/src/gradient_descent.cpp +++ b/src/fuzzing/src/gradient_descent.cpp @@ -4,7 +4,9 @@ #include #include #include +#include #include +#include /** * @brief Constructs a GradientDescent object. @@ -30,26 +32,42 @@ GradientDescent::GradientDescent( const std::vector< std::vector< float > >& coe , _convergence_threshold( convergence_threshold ) , _momentum( momentum ) { - if ( coefficient_matrix.empty() || target_vector.empty() || - coefficient_matrix.size() != target_vector.size() ) { + std::set< std::vector< float > > unique_rows; + std::vector< float > new_target_vector; + std::vector< std::vector< float > > new_coefficient_matrix; + + for ( size_t i = 0; i < _coefficient_matrix.size(); ++i ) { + std::vector< float > row = _coefficient_matrix[ i ]; + row.push_back( _target_vector[ i ] ); + + if ( unique_rows.insert( row ).second ) { + new_target_vector.push_back( _target_vector[ i ] ); + new_coefficient_matrix.push_back( _coefficient_matrix[ i ] ); + } + } + + _coefficient_matrix = new_coefficient_matrix; + _target_vector = new_target_vector; + + if ( _coefficient_matrix.empty() || _target_vector.empty() || + _coefficient_matrix.size() != _target_vector.size() ) { throw std::invalid_argument( "Invalid input dimensions" ); } - for (auto& row : _coefficient_matrix) { - row.push_back(1.0f); + for ( auto& row : _coefficient_matrix ) { + row.push_back( 1.0f ); } } /** - * @brief Performs mini-batch gradient descent optimization with momentum. + * @brief Performs gradient descent optimization with momentum. * - * @param batch_size The size of each mini-batch. * @return std::vector The optimized solution vector. */ -std::vector< float > GradientDescent::optimize( int batch_size ) +std::vector< float > GradientDescent::optimize() { if ( false ) { - std::cout << "Coefficient Matrix and Target Vector:" << std::endl; + std::cout << "### Coefficient Matrix and Target Vector:" << std::endl; for ( size_t i = 0; i < _coefficient_matrix.size(); ++i ) { for ( const auto& val : _coefficient_matrix[ i ] ) { std::cout << val << " "; @@ -58,28 +76,38 @@ std::vector< float > GradientDescent::optimize( int batch_size ) } } + if ( true ) { + std::cout << "### Equation Matrix:" << std::endl; + std::cout << "$$\\begin{bmatrix}" << std::endl; + for ( size_t i = 0; i < _coefficient_matrix.size(); ++i ) { + for ( size_t j = 0; j < _coefficient_matrix[ i ].size(); ++j ) { + std::cout << _coefficient_matrix[ i ][ j ]; + if ( j < _coefficient_matrix[ i ].size() - 1 ) { + std::cout << " & "; + } + } + std::cout << " & " << _target_vector[ i ]; + if ( i < _coefficient_matrix.size() - 1 ) { + std::cout << " \\\\"; + } + std::cout << std::endl; + } + std::cout << "\\end{bmatrix}$$" << std::endl; + } + std::vector< float > current_solution = generate_random_weights( _coefficient_matrix[ 0 ].size() ); std::vector< float > velocity( current_solution.size(), 0.0f ); float prev_cost = std::numeric_limits< float >::max(); - size_t num_samples = _coefficient_matrix.size(); for ( int iteration = 0; iteration < _max_iterations; ++iteration ) { shuffle_data(); - for ( size_t batch_start = 0; batch_start < num_samples; batch_start += batch_size ) { - size_t batch_end = std::min( batch_start + batch_size, num_samples ); - std::vector< std::vector< float > > batch_coefficients = - get_batch( _coefficient_matrix, batch_start, batch_end ); - std::vector< float > batch_target = get_batch( _target_vector, batch_start, batch_end ); - - std::vector< float > gradient = - compute_batch_gradient( current_solution, batch_coefficients, batch_target ); + std::vector< float > gradient = compute_gradient( current_solution ); - for ( size_t i = 0; i < current_solution.size(); ++i ) { - velocity[ i ] = _momentum * velocity[ i ] - _learning_rate * gradient[ i ]; - current_solution[ i ] += velocity[ i ]; - } + for ( size_t i = 0; i < current_solution.size(); ++i ) { + velocity[ i ] = _momentum * velocity[ i ] - _learning_rate * gradient[ i ]; + current_solution[ i ] += velocity[ i ]; } float current_cost = compute_mean_squared_error( current_solution ); @@ -125,47 +153,6 @@ void GradientDescent::shuffle_data() _target_vector = shuffled_target; } -/** - * @brief Retrieves a mini-batch from a matrix or vector. - */ -template < typename T > -std::vector< T > GradientDescent::get_batch( const std::vector< T >& data, size_t start, size_t end ) -{ - return std::vector< T >( data.begin() + start, data.begin() + end ); -} - -/** - * @brief Computes the gradient for a mini-batch. - * - * @param current_solution The current solution vector. - * @param batch_coefficients The coefficient matrix of the current batch. - * @param batch_target The target vector of the current batch. - * @return std::vector The computed gradient vector. - */ -std::vector< float > -GradientDescent::compute_batch_gradient( const std::vector< float >& current_solution, - const std::vector< std::vector< float > >& batch_coefficients, - const std::vector< float >& batch_target ) -{ - std::vector< float > gradient( current_solution.size(), 0.0f ); - - for ( size_t i = 0; i < batch_coefficients.size(); ++i ) { - float predicted = dot_product( current_solution, batch_coefficients[ i ] ); - float error = predicted - batch_target[ i ]; - - for ( size_t j = 0; j < current_solution.size(); ++j ) { - gradient[ j ] += 2 * error * batch_coefficients[ i ][ j ]; - } - } - - for ( float& grad : gradient ) { - grad /= batch_coefficients.size(); - } - - return gradient; -} - - /** * @brief Computes the gradient for the current solution. * @@ -324,4 +311,4 @@ void GradientDescent::min_max_normalize( std::vector< std::vector< float > >& ma } } } -} +} \ No newline at end of file diff --git a/src/fuzzing/src/iid_node_dependencies.cpp b/src/fuzzing/src/iid_node_dependencies.cpp index 65e202ca..537970e9 100644 --- a/src/fuzzing/src/iid_node_dependencies.cpp +++ b/src/fuzzing/src/iid_node_dependencies.cpp @@ -320,9 +320,7 @@ void fuzzing::iid_node_dependence_props::dependencies_generation() std::cout << "# Subsets:" << std::endl; for ( const auto& subset : subsets ) { std::vector< std::vector< float > > sub_matrix = get_matrix( subset ); - GradientDescent gd( sub_matrix, best_values ); - std::vector< float > weights = gd.optimize(); - + std::cout << "## Subset of nodes: `{ "; auto delimeter = ""; for ( const auto& leaf : subset ) { @@ -331,9 +329,21 @@ void fuzzing::iid_node_dependence_props::dependencies_generation() } std::cout << " }`" << std::endl; + GradientDescent gd( sub_matrix, best_values ); + std::vector< float > weights = gd.optimize(); + + float dot_product = + std::inner_product( weights.begin(), weights.end() - 1, weights.begin(), 0.0f ); + std::vector< float > node_counts; + for ( auto it = weights.begin(); it != weights.end() - 1; ++it ) { + node_counts.push_back( -weights.back() * ( *it ) / dot_product ); + } + + std::cout << "### Weights and Counts:" << std::endl; for ( size_t i = 0; i < subset.size(); ++i ) { const auto& node = *std::next( subset.begin(), i ); - std::cout << "- `(" << node << "): " << weights[ i ] << "`" << std::endl; + std::cout << "- `(" << node << "): " << weights[ i ] << " (" + << static_cast< int >( std::round( node_counts[ i ] ) ) << ")`" << std::endl; } std::cout << "- `Increment: " << weights.back() << "`" << std::endl; @@ -547,9 +557,9 @@ void fuzzing::iid_node_dependence_props::compute_dependencies_by_loading( const while ( node != nullptr ) { for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { - if (loop_head.id == node->get_location_id().id) { + if ( loop_head.id == node->get_location_id().id ) { natural_32_bit bits_count = node->get_num_stdin_bits(); - + auto& [ min, max ] = loading_loops[ loop_head ]; min = std::min( min, bits_count ); max = std::max( max, bits_count ); From faa30feb9be6e3d844e6c9651372a67ab868f93b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Tue, 5 Nov 2024 14:44:37 +0100 Subject: [PATCH 049/144] feat: return if the gradient descent converged --- .../include/fuzzing/gradient_descent.hpp | 14 ++++++------- src/fuzzing/src/gradient_descent.cpp | 20 +++++++++++++------ src/fuzzing/src/iid_node_dependencies.cpp | 7 ++++--- 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/src/fuzzing/include/fuzzing/gradient_descent.hpp b/src/fuzzing/include/fuzzing/gradient_descent.hpp index b8d5a610..4fc38efd 100644 --- a/src/fuzzing/include/fuzzing/gradient_descent.hpp +++ b/src/fuzzing/include/fuzzing/gradient_descent.hpp @@ -15,11 +15,11 @@ class GradientDescent { GradientDescent( const std::vector< std::vector< float > >& coefficient_matrix, const std::vector< float >& target_vector, float learning_rate = 0.01f, - int max_iterations = 10000, - float convergence_threshold = 1e-5, - float momentum = 0.8f); + int max_iterations = 1000, + float convergence_threshold = 1e-4, + float momentum = 0.9f ); - std::vector< float > optimize(); + std::tuple< std::vector< float >, bool > optimize(); void set_learning_rate( float learning_rate ) { _learning_rate = learning_rate; } void set_max_iterations( int max_iterations ) { _max_iterations = max_iterations; } @@ -27,7 +27,7 @@ class GradientDescent { { _convergence_threshold = convergence_threshold; } - void set_momentum(float momentum) { _momentum = momentum; } + void set_momentum( float momentum ) { _momentum = momentum; } private: std::vector< std::vector< float > > _coefficient_matrix; @@ -44,8 +44,8 @@ class GradientDescent { static float dot_product( const std::vector< float >& a, const std::vector< float >& b ); static void rescale( std::vector< float >& values, float min_value, float max_value ); static void add_smallest_value( std::vector< float >& values ); - void min_max_normalize(std::vector>& matrix); - void min_max_normalize_target(std::vector& target); + void min_max_normalize( std::vector< std::vector< float > >& matrix ); + void min_max_normalize_target( std::vector< float >& target ); void shuffle_data(); }; \ No newline at end of file diff --git a/src/fuzzing/src/gradient_descent.cpp b/src/fuzzing/src/gradient_descent.cpp index f1c67366..27d5241a 100644 --- a/src/fuzzing/src/gradient_descent.cpp +++ b/src/fuzzing/src/gradient_descent.cpp @@ -64,7 +64,7 @@ GradientDescent::GradientDescent( const std::vector< std::vector< float > >& coe * * @return std::vector The optimized solution vector. */ -std::vector< float > GradientDescent::optimize() +std::tuple< std::vector< float >, bool > GradientDescent::optimize() { if ( false ) { std::cout << "### Coefficient Matrix and Target Vector:" << std::endl; @@ -96,13 +96,19 @@ std::vector< float > GradientDescent::optimize() } std::vector< float > current_solution = generate_random_weights( _coefficient_matrix[ 0 ].size() ); + if (_debug) { + std::cout << "Current solution: "; + for (const auto& val : current_solution) { + std::cout << val << " "; + } + std::cout << std::endl; + } std::vector< float > velocity( current_solution.size(), 0.0f ); + bool converged = false; float prev_cost = std::numeric_limits< float >::max(); for ( int iteration = 0; iteration < _max_iterations; ++iteration ) { - shuffle_data(); - std::vector< float > gradient = compute_gradient( current_solution ); for ( size_t i = 0; i < current_solution.size(); ++i ) { @@ -112,7 +118,7 @@ std::vector< float > GradientDescent::optimize() float current_cost = compute_mean_squared_error( current_solution ); - if ( _debug && iteration % 100 == 0 ) { + if ( _debug && iteration % 5 == 0 ) { std::cout << "Iteration " << iteration << ", Cost: " << current_cost << std::endl; } @@ -120,12 +126,14 @@ std::vector< float > GradientDescent::optimize() if ( _debug ) { std::cout << "Converged after " << iteration << " iterations." << std::endl; } + + converged = std::abs( current_cost ) < _convergence_threshold * current_solution.size(); break; } prev_cost = current_cost; } - return current_solution; + return { current_solution, converged }; } /** @@ -226,7 +234,7 @@ std::vector< float > GradientDescent::generate_random_weights( size_t n ) std::vector< float > weights( n ); std::random_device rd; std::mt19937 gen( rd() ); - std::normal_distribution<> d( 0, 1.0 / std::sqrt( n ) ); + std::uniform_real_distribution<> d( -0.01, 0.01 ); std::generate( weights.begin(), weights.end(), [ & ]() { return d( gen ); } ); diff --git a/src/fuzzing/src/iid_node_dependencies.cpp b/src/fuzzing/src/iid_node_dependencies.cpp index 537970e9..b2c4a94d 100644 --- a/src/fuzzing/src/iid_node_dependencies.cpp +++ b/src/fuzzing/src/iid_node_dependencies.cpp @@ -320,7 +320,7 @@ void fuzzing::iid_node_dependence_props::dependencies_generation() std::cout << "# Subsets:" << std::endl; for ( const auto& subset : subsets ) { std::vector< std::vector< float > > sub_matrix = get_matrix( subset ); - + std::cout << "## Subset of nodes: `{ "; auto delimeter = ""; for ( const auto& leaf : subset ) { @@ -330,7 +330,7 @@ void fuzzing::iid_node_dependence_props::dependencies_generation() std::cout << " }`" << std::endl; GradientDescent gd( sub_matrix, best_values ); - std::vector< float > weights = gd.optimize(); + auto [ weights, converged ] = gd.optimize(); float dot_product = std::inner_product( weights.begin(), weights.end() - 1, weights.begin(), 0.0f ); @@ -340,6 +340,7 @@ void fuzzing::iid_node_dependence_props::dependencies_generation() } std::cout << "### Weights and Counts:" << std::endl; + std::cout << "- `Converged: " << ( converged ? "True" : "False" ) << "`" << std::endl; for ( size_t i = 0; i < subset.size(); ++i ) { const auto& node = *std::next( subset.begin(), i ); std::cout << "- `(" << node << "): " << weights[ i ] << " (" @@ -456,7 +457,7 @@ std::map< location_id, fuzzing::path_decision > fuzzing::iid_node_dependence_pro std::vector< float > fuzzing::iid_node_dependence_props::approximate_matrix() { GradientDescent gd( matrix, best_values ); - std::vector< float > weights = gd.optimize(); + auto [ weights, converged ] = gd.optimize(); if ( false ) { for ( size_t i = 0; i < interesting_nodes.size(); ++i ) { From 959a511b8e8f27f19ab774186c689fb13abf3f11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Fri, 15 Nov 2024 19:43:55 +0100 Subject: [PATCH 050/144] feat: new gradient descent --- src/fuzzing/src/gradient_descent.cpp | 44 +++- .../src/gradient_descent_with_convergence.cpp | 223 ++++++++++++++++++ 2 files changed, 262 insertions(+), 5 deletions(-) create mode 100644 src/fuzzing/src/gradient_descent_with_convergence.cpp diff --git a/src/fuzzing/src/gradient_descent.cpp b/src/fuzzing/src/gradient_descent.cpp index 27d5241a..3e9b6699 100644 --- a/src/fuzzing/src/gradient_descent.cpp +++ b/src/fuzzing/src/gradient_descent.cpp @@ -24,13 +24,19 @@ GradientDescent::GradientDescent( const std::vector< std::vector< float > >& coe float learning_rate, int max_iterations, float convergence_threshold, - float momentum ) + float momentum, + int stagnant_threshold, + float improvement_threshold, + float error_threshold ) : _coefficient_matrix( coefficient_matrix ) , _target_vector( target_vector ) , _learning_rate( learning_rate ) , _max_iterations( max_iterations ) , _convergence_threshold( convergence_threshold ) , _momentum( momentum ) + , _stagnant_threshold( stagnant_threshold ) + , _improvement_threshold( improvement_threshold ) + , _error_threshold( error_threshold ) { std::set< std::vector< float > > unique_rows; std::vector< float > new_target_vector; @@ -96,15 +102,16 @@ std::tuple< std::vector< float >, bool > GradientDescent::optimize() } std::vector< float > current_solution = generate_random_weights( _coefficient_matrix[ 0 ].size() ); - if (_debug) { + if ( _debug ) { std::cout << "Current solution: "; - for (const auto& val : current_solution) { + for ( const auto& val : current_solution ) { std::cout << val << " "; } std::cout << std::endl; } std::vector< float > velocity( current_solution.size(), 0.0f ); bool converged = false; + int stagnant_iterations = 0; float prev_cost = std::numeric_limits< float >::max(); @@ -118,16 +125,43 @@ std::tuple< std::vector< float >, bool > GradientDescent::optimize() float current_cost = compute_mean_squared_error( current_solution ); - if ( _debug && iteration % 5 == 0 ) { + if ( _debug && iteration % 25 == 0 ) { std::cout << "Iteration " << iteration << ", Cost: " << current_cost << std::endl; } + if ( std::abs( current_cost ) < std::abs( prev_cost ) && + std::abs( current_cost - prev_cost ) < _improvement_threshold ) { + stagnant_iterations = 0; + } else if ( std::abs( current_cost - prev_cost ) < _improvement_threshold ) { + ++stagnant_iterations; + if ( stagnant_iterations >= _stagnant_threshold ) { + if ( true ) { + std::cout << "Stopped early due to lack of significant improvement after " + << iteration << " iterations." << std::endl; + } + break; + } + } + if ( std::abs( current_cost - prev_cost ) < _convergence_threshold ) { if ( _debug ) { std::cout << "Converged after " << iteration << " iterations." << std::endl; } - converged = std::abs( current_cost ) < _convergence_threshold * current_solution.size(); + bool high_error = false; + for ( size_t i = 0; i < _coefficient_matrix.size(); ++i ) { + float predicted = dot_product( current_solution, _coefficient_matrix[ i ] ); + float error = predicted - _target_vector[ i ]; + if ( std::abs( error ) > 0.1 ) { + high_error = true; + break; + } + } + + if ( !high_error ) { + converged = true; + } + break; } prev_cost = current_cost; diff --git a/src/fuzzing/src/gradient_descent_with_convergence.cpp b/src/fuzzing/src/gradient_descent_with_convergence.cpp new file mode 100644 index 00000000..31f0ba2b --- /dev/null +++ b/src/fuzzing/src/gradient_descent_with_convergence.cpp @@ -0,0 +1,223 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +GradientDescentNew::GradientDescentNew( const std::vector< std::vector< float > >& coefficient_matrix, + const std::vector< float >& target_vector, + float learning_rate, + int max_iterations, + float convergence_threshold, + float momentum ) + : _coefficient_matrix( coefficient_matrix ) + , _target_vector( target_vector ) + , _learning_rate( learning_rate ) + , _max_iterations( max_iterations ) + , _convergence_threshold( convergence_threshold ) + , _momentum( momentum ) +{ + std::set< std::vector< float > > unique_rows; + std::vector< float > new_target_vector; + std::vector< std::vector< float > > new_coefficient_matrix; + + for ( size_t i = 0; i < _coefficient_matrix.size(); ++i ) { + std::vector< float > row = _coefficient_matrix[ i ]; + row.push_back( _target_vector[ i ] ); + + if ( unique_rows.insert( row ).second ) { + new_target_vector.push_back( _target_vector[ i ] ); + new_coefficient_matrix.push_back( _coefficient_matrix[ i ] ); + } + } + + _coefficient_matrix = new_coefficient_matrix; + _target_vector = new_target_vector; + + if ( _coefficient_matrix.empty() || _target_vector.empty() || + _coefficient_matrix.size() != _target_vector.size() ) { + throw std::invalid_argument( "Invalid input dimensions" ); + } + + for ( auto& row : _coefficient_matrix ) { + row.push_back( 1.0f ); + } +} + +GradientDescentResult GradientDescentNew::optimize() +{ + std::vector< float > current_solution = generate_random_weights( _coefficient_matrix[ 0 ].size() ); + std::vector< float > prev_errors = compute_errors( current_solution ); + std::vector< float > velocity( current_solution.size(), 0.0f ); + float prev_cost = std::numeric_limits< float >::max(); + bool converged = false; + int iterations = 0; + + for ( ; iterations < _max_iterations; ++iterations ) { + std::vector< float > gradient = compute_gradient( current_solution ); + + for ( size_t i = 0; i < current_solution.size(); ++i ) { + velocity[ i ] = _momentum * velocity[ i ] - _learning_rate * gradient[ i ]; + current_solution[ i ] += velocity[ i ]; + } + + std::vector< float > errors = compute_errors( current_solution ); + float current_cost = compute_mean_squared_error( errors ); + + if ( std::abs( current_cost - prev_cost ) < _convergence_threshold ) { + break; + } + + prev_cost = current_cost; + prev_errors = errors; + } + + float error_variance = compute_variance( prev_errors ); + float error_mean = std::accumulate( prev_errors.begin(), prev_errors.end(), 0.0f ) / prev_errors.size(); + std::vector< float > counts_per_column = compute_column_count( current_solution ); + + float variance_threshold = 0.0f; + float count_threshold = 0.0f; + + converged = + compute_convergence( counts_per_column, error_variance, error_mean, variance_threshold, count_threshold ); + + return GradientDescentResult{ + .weights = current_solution, + .errors = prev_errors, + .column_count_weighted = counts_per_column, + .iterations = iterations, + .error_mean = error_mean, + .error_variance = error_variance, + .variance_threshold = variance_threshold, + .count_threshold = count_threshold, + .converged = converged + }; +} + +std::vector< float > GradientDescentNew::compute_column_count( const std::vector< float >& current_solution ) +{ + std::vector< float > counts_per_column( current_solution.size(), 0.0f ); + + for ( size_t i = 0; i < current_solution.size(); ++i ) { + for ( size_t j = 0; j < current_solution.size(); ++j ) { + counts_per_column[ i ] += std::abs( current_solution[ i ] * _coefficient_matrix[ j ][ i ] ); + } + } + + + return counts_per_column; +} + +bool GradientDescentNew::compute_convergence( const std::vector< float >& counts_per_column, + float error_variance, + float error_mean, + float& variance_threshold, + float& count_threshold ) +{ + variance_threshold = error_mean * 10.0f; + + if ( std::abs( error_variance ) > variance_threshold ) { + return false; + } + + // float column_count_sum = std::accumulate( counts_per_column.begin(), counts_per_column.end(), 0.0f ); + // float count_threshold = column_count_sum * 0.05f; + + float mean_column_count = std::accumulate( counts_per_column.begin(), counts_per_column.end(), 0.0f ) / + counts_per_column.size(); + + float stddev_column_count = 0.0f; + for ( const auto& count : counts_per_column ) { + stddev_column_count += ( count - mean_column_count ) * ( count - mean_column_count ); + } + stddev_column_count = std::sqrt( stddev_column_count / counts_per_column.size() ); + + count_threshold = mean_column_count + 2 * stddev_column_count; + + for ( const auto& count : counts_per_column ) { + if ( std::abs( count ) > count_threshold ) { + return false; + } + } + + return true; +} + +std::vector< float > GradientDescentNew::compute_gradient( const std::vector< float >& current_solution ) +{ + std::vector< float > gradient( current_solution.size(), 0.0f ); + + for ( size_t i = 0; i < _coefficient_matrix.size(); ++i ) { + float predicted = dot_product( current_solution, _coefficient_matrix[ i ] ); + float error = predicted - _target_vector[ i ]; + + for ( size_t j = 0; j < current_solution.size(); ++j ) { + gradient[ j ] += 2 * error * _coefficient_matrix[ i ][ j ]; + } + } + + return gradient; +} + + +std::vector< float > GradientDescentNew::compute_errors( const std::vector< float >& current_solution ) +{ + std::vector< float > errors( _coefficient_matrix.size(), 0.0f ); + + for ( size_t i = 0; i < _coefficient_matrix.size(); ++i ) { + float predicted = dot_product( current_solution, _coefficient_matrix[ i ] ); + errors[ i ] = predicted - _target_vector[ i ]; + } + + return errors; +} + +float GradientDescentNew::compute_mean_squared_error( const std::vector< float >& errors ) +{ + float mse = 0.0f; + + for ( size_t i = 0; i < errors.size(); ++i ) { + mse += errors[ i ] * errors[ i ]; + } + + return mse / errors.size(); +} + +float GradientDescentNew::dot_product( const std::vector< float >& a, const std::vector< float >& b ) +{ + if ( a.size() != b.size() ) { + throw std::invalid_argument( "Vectors must have the same size for dot product" ); + } + + return std::inner_product( a.begin(), a.end(), b.begin(), 0.0f ); +} + +std::vector< float > GradientDescentNew::generate_random_weights( size_t n ) +{ + std::vector< float > weights( n ); + std::random_device rd; + std::mt19937 gen( rd() ); + std::uniform_real_distribution<> d( -0.01, 0.01 ); + + std::generate( weights.begin(), weights.end(), [ & ]() { return d( gen ); } ); + + return weights; +} + + +float GradientDescentNew::compute_variance( const std::vector< float >& errors ) +{ + float mean = std::accumulate( errors.begin(), errors.end(), 0.0f ) / errors.size(); + float error_variance = 0.0f; + + for ( const auto& error : errors ) { + error_variance += ( error - mean ) * ( error - mean ); + } + + return error_variance / errors.size(); +} From 83a183e216bcec8149754083592ee6d98cac996b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Fri, 15 Nov 2024 19:49:48 +0100 Subject: [PATCH 051/144] feat: new gradient descent --- .../gradient_descent_with_convergence.hpp | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 src/fuzzing/include/fuzzing/gradient_descent_with_convergence.hpp diff --git a/src/fuzzing/include/fuzzing/gradient_descent_with_convergence.hpp b/src/fuzzing/include/fuzzing/gradient_descent_with_convergence.hpp new file mode 100644 index 00000000..49ae4fb9 --- /dev/null +++ b/src/fuzzing/include/fuzzing/gradient_descent_with_convergence.hpp @@ -0,0 +1,50 @@ +#pragma once + +#include + +struct GradientDescentResult { + std::vector< float > weights; + std::vector< float > errors; + std::vector< float > column_count_weighted; + int iterations; + float error_mean; + float error_variance; + float variance_threshold; + float count_threshold; + bool converged; +}; + + +class GradientDescentNew { +public: + GradientDescentNew( const std::vector< std::vector< float > >& coefficient_matrix, + const std::vector< float >& target_vector, + float learning_rate = 0.001f, + int max_iterations = 1000, + float convergence_threshold = 1e-4, + float momentum = 0.9f ); + + GradientDescentResult optimize(); + +private: + std::vector< std::vector< float > > _coefficient_matrix; + std::vector< float > _target_vector; + float _learning_rate; + int _max_iterations; + float _convergence_threshold; + float _momentum; + bool _debug = false; + + std::vector< float > compute_gradient( const std::vector< float >& current_solution ); + std::vector< float > compute_errors( const std::vector< float >& current_solution ); + float compute_mean_squared_error( const std::vector< float >& errors ); + static std::vector< float > generate_random_weights( size_t n ); + static float dot_product( const std::vector< float >& a, const std::vector< float >& b ); + static float compute_variance( const std::vector< float >& errors ); + static bool compute_convergence( const std::vector< float >& counts_per_column_weighted, + float error_variance, + float error_mean, + float& variance_threshold, + float& count_threshold ); + std::vector< float > compute_column_count( const std::vector< float >& current_solution ); +}; \ No newline at end of file From 49bde519cb32209d0663afd2f6324e50e56698b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Fri, 15 Nov 2024 21:01:34 +0100 Subject: [PATCH 052/144] feat: gradient descent convergence computation --- .../gradient_descent_with_convergence.hpp | 3 +- .../src/gradient_descent_with_convergence.cpp | 76 +++++++++++-------- 2 files changed, 45 insertions(+), 34 deletions(-) diff --git a/src/fuzzing/include/fuzzing/gradient_descent_with_convergence.hpp b/src/fuzzing/include/fuzzing/gradient_descent_with_convergence.hpp index 49ae4fb9..2baf045e 100644 --- a/src/fuzzing/include/fuzzing/gradient_descent_with_convergence.hpp +++ b/src/fuzzing/include/fuzzing/gradient_descent_with_convergence.hpp @@ -19,7 +19,7 @@ class GradientDescentNew { public: GradientDescentNew( const std::vector< std::vector< float > >& coefficient_matrix, const std::vector< float >& target_vector, - float learning_rate = 0.001f, + float learning_rate = 0.01f, int max_iterations = 1000, float convergence_threshold = 1e-4, float momentum = 0.9f ); @@ -47,4 +47,5 @@ class GradientDescentNew { float& variance_threshold, float& count_threshold ); std::vector< float > compute_column_count( const std::vector< float >& current_solution ); + void print_input_matrix(); }; \ No newline at end of file diff --git a/src/fuzzing/src/gradient_descent_with_convergence.cpp b/src/fuzzing/src/gradient_descent_with_convergence.cpp index 31f0ba2b..1770c483 100644 --- a/src/fuzzing/src/gradient_descent_with_convergence.cpp +++ b/src/fuzzing/src/gradient_descent_with_convergence.cpp @@ -48,6 +48,26 @@ GradientDescentNew::GradientDescentNew( const std::vector< std::vector< float > } } +void GradientDescentNew::print_input_matrix() +{ + std::cout << "### Equation Matrix:" << std::endl; + std::cout << "$$\\begin{bmatrix}" << std::endl; + for ( size_t i = 0; i < _coefficient_matrix.size(); ++i ) { + for ( size_t j = 0; j < _coefficient_matrix[ i ].size(); ++j ) { + std::cout << _coefficient_matrix[ i ][ j ]; + if ( j < _coefficient_matrix[ i ].size() - 1 ) { + std::cout << " & "; + } + } + std::cout << " & " << _target_vector[ i ]; + if ( i < _coefficient_matrix.size() - 1 ) { + std::cout << " \\\\"; + } + std::cout << std::endl; + } + std::cout << "\\end{bmatrix}$$" << std::endl; +} + GradientDescentResult GradientDescentNew::optimize() { std::vector< float > current_solution = generate_random_weights( _coefficient_matrix[ 0 ].size() ); @@ -86,30 +106,27 @@ GradientDescentResult GradientDescentNew::optimize() converged = compute_convergence( counts_per_column, error_variance, error_mean, variance_threshold, count_threshold ); - return GradientDescentResult{ - .weights = current_solution, - .errors = prev_errors, - .column_count_weighted = counts_per_column, - .iterations = iterations, - .error_mean = error_mean, - .error_variance = error_variance, - .variance_threshold = variance_threshold, - .count_threshold = count_threshold, - .converged = converged - }; + return GradientDescentResult{ .weights = current_solution, + .errors = prev_errors, + .column_count_weighted = counts_per_column, + .iterations = iterations, + .error_mean = error_mean, + .error_variance = error_variance, + .variance_threshold = variance_threshold, + .count_threshold = count_threshold, + .converged = converged }; } std::vector< float > GradientDescentNew::compute_column_count( const std::vector< float >& current_solution ) { std::vector< float > counts_per_column( current_solution.size(), 0.0f ); - for ( size_t i = 0; i < current_solution.size(); ++i ) { - for ( size_t j = 0; j < current_solution.size(); ++j ) { - counts_per_column[ i ] += std::abs( current_solution[ i ] * _coefficient_matrix[ j ][ i ] ); + for ( size_t i = 0; i < _coefficient_matrix.size(); ++i ) { + for ( size_t j = 0; j < _coefficient_matrix[ i ].size(); ++j ) { + counts_per_column[ j ] += std::abs( _coefficient_matrix[ i ][ j ] * current_solution[ j ] ); } } - return counts_per_column; } @@ -121,26 +138,16 @@ bool GradientDescentNew::compute_convergence( const std::vector< float >& counts { variance_threshold = error_mean * 10.0f; - if ( std::abs( error_variance ) > variance_threshold ) { - return false; - } - - // float column_count_sum = std::accumulate( counts_per_column.begin(), counts_per_column.end(), 0.0f ); - // float count_threshold = column_count_sum * 0.05f; + float column_count_sum = std::abs( + std::accumulate( counts_per_column.begin(), counts_per_column.end() - 1, 0.0f ) ); + count_threshold = column_count_sum * 0.05f; - float mean_column_count = std::accumulate( counts_per_column.begin(), counts_per_column.end(), 0.0f ) / - counts_per_column.size(); - - float stddev_column_count = 0.0f; - for ( const auto& count : counts_per_column ) { - stddev_column_count += ( count - mean_column_count ) * ( count - mean_column_count ); + if ( error_variance > variance_threshold ) { + return false; } - stddev_column_count = std::sqrt( stddev_column_count / counts_per_column.size() ); - count_threshold = mean_column_count + 2 * stddev_column_count; - - for ( const auto& count : counts_per_column ) { - if ( std::abs( count ) > count_threshold ) { + for ( size_t i = 0; i < counts_per_column.size() - 1; ++i ) { + if ( std::abs( counts_per_column[ i ] ) < count_threshold ) { return false; } } @@ -161,10 +168,13 @@ std::vector< float > GradientDescentNew::compute_gradient( const std::vector< fl } } + for ( float& grad : gradient ) { + grad /= _coefficient_matrix.size(); + } + return gradient; } - std::vector< float > GradientDescentNew::compute_errors( const std::vector< float >& current_solution ) { std::vector< float > errors( _coefficient_matrix.size(), 0.0f ); From d9d07f490a6622fc1f34e835e8d3a6ddec561d88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sun, 17 Nov 2024 23:01:01 +0100 Subject: [PATCH 053/144] feat: gradient descent convergence computation --- .../gradient_descent_with_convergence.hpp | 5 +++- .../src/gradient_descent_with_convergence.cpp | 23 +++++++++++++------ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/fuzzing/include/fuzzing/gradient_descent_with_convergence.hpp b/src/fuzzing/include/fuzzing/gradient_descent_with_convergence.hpp index 2baf045e..2827a3f1 100644 --- a/src/fuzzing/include/fuzzing/gradient_descent_with_convergence.hpp +++ b/src/fuzzing/include/fuzzing/gradient_descent_with_convergence.hpp @@ -8,6 +8,8 @@ struct GradientDescentResult { std::vector< float > column_count_weighted; int iterations; float error_mean; + float error_square_of_mean; + float error_mean_of_squares; float error_variance; float variance_threshold; float count_threshold; @@ -25,6 +27,7 @@ class GradientDescentNew { float momentum = 0.9f ); GradientDescentResult optimize(); + void print_input_matrix(); private: std::vector< std::vector< float > > _coefficient_matrix; @@ -46,6 +49,6 @@ class GradientDescentNew { float error_mean, float& variance_threshold, float& count_threshold ); + static float compute_mean( const std::vector< float >& errors ); std::vector< float > compute_column_count( const std::vector< float >& current_solution ); - void print_input_matrix(); }; \ No newline at end of file diff --git a/src/fuzzing/src/gradient_descent_with_convergence.cpp b/src/fuzzing/src/gradient_descent_with_convergence.cpp index 1770c483..24648f93 100644 --- a/src/fuzzing/src/gradient_descent_with_convergence.cpp +++ b/src/fuzzing/src/gradient_descent_with_convergence.cpp @@ -97,7 +97,7 @@ GradientDescentResult GradientDescentNew::optimize() } float error_variance = compute_variance( prev_errors ); - float error_mean = std::accumulate( prev_errors.begin(), prev_errors.end(), 0.0f ) / prev_errors.size(); + float error_mean = compute_mean( prev_errors ); std::vector< float > counts_per_column = compute_column_count( current_solution ); float variance_threshold = 0.0f; @@ -111,6 +111,8 @@ GradientDescentResult GradientDescentNew::optimize() .column_count_weighted = counts_per_column, .iterations = iterations, .error_mean = error_mean, + .error_square_of_mean = float( std::pow( error_mean, 2 ) ), + .error_mean_of_squares = prev_cost, .error_variance = error_variance, .variance_threshold = variance_threshold, .count_threshold = count_threshold, @@ -136,13 +138,15 @@ bool GradientDescentNew::compute_convergence( const std::vector< float >& counts float& variance_threshold, float& count_threshold ) { - variance_threshold = error_mean * 10.0f; + constexpr float VARIANCE_SCALING_FACTOR = 10.0f; + + variance_threshold = std::abs( error_mean * VARIANCE_SCALING_FACTOR ); float column_count_sum = std::abs( std::accumulate( counts_per_column.begin(), counts_per_column.end() - 1, 0.0f ) ); - count_threshold = column_count_sum * 0.05f; + count_threshold = column_count_sum * 0.01f; - if ( error_variance > variance_threshold ) { + if ( error_variance > variance_threshold ) { return false; } @@ -192,7 +196,7 @@ float GradientDescentNew::compute_mean_squared_error( const std::vector< float > float mse = 0.0f; for ( size_t i = 0; i < errors.size(); ++i ) { - mse += errors[ i ] * errors[ i ]; + mse += std::pow( errors[ i ], 2 ); } return mse / errors.size(); @@ -222,12 +226,17 @@ std::vector< float > GradientDescentNew::generate_random_weights( size_t n ) float GradientDescentNew::compute_variance( const std::vector< float >& errors ) { - float mean = std::accumulate( errors.begin(), errors.end(), 0.0f ) / errors.size(); + float mean = compute_mean( errors ); float error_variance = 0.0f; for ( const auto& error : errors ) { - error_variance += ( error - mean ) * ( error - mean ); + error_variance += std::pow( error - mean, 2 ); } return error_variance / errors.size(); } + +float GradientDescentNew::compute_mean( const std::vector< float >& errors ) +{ + return std::accumulate( errors.begin(), errors.end(), 0.0f ) / errors.size(); +} \ No newline at end of file From 8b05d0d9858ed68909dec047f59e9127506d1544 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Mon, 18 Nov 2024 19:19:52 +0100 Subject: [PATCH 054/144] feat: add locked columns --- .../gradient_descent_with_convergence.hpp | 4 ++++ .../src/gradient_descent_with_convergence.cpp | 19 ++++++++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/fuzzing/include/fuzzing/gradient_descent_with_convergence.hpp b/src/fuzzing/include/fuzzing/gradient_descent_with_convergence.hpp index 2827a3f1..37b3e312 100644 --- a/src/fuzzing/include/fuzzing/gradient_descent_with_convergence.hpp +++ b/src/fuzzing/include/fuzzing/gradient_descent_with_convergence.hpp @@ -1,5 +1,7 @@ #pragma once +#include +#include #include struct GradientDescentResult { @@ -21,6 +23,7 @@ class GradientDescentNew { public: GradientDescentNew( const std::vector< std::vector< float > >& coefficient_matrix, const std::vector< float >& target_vector, + std::map< size_t, float > locked_columns = {}, float learning_rate = 0.01f, int max_iterations = 1000, float convergence_threshold = 1e-4, @@ -32,6 +35,7 @@ class GradientDescentNew { private: std::vector< std::vector< float > > _coefficient_matrix; std::vector< float > _target_vector; + std::vector< std::optional< float > > _locked_columns; float _learning_rate; int _max_iterations; float _convergence_threshold; diff --git a/src/fuzzing/src/gradient_descent_with_convergence.cpp b/src/fuzzing/src/gradient_descent_with_convergence.cpp index 24648f93..88b90384 100644 --- a/src/fuzzing/src/gradient_descent_with_convergence.cpp +++ b/src/fuzzing/src/gradient_descent_with_convergence.cpp @@ -10,6 +10,7 @@ GradientDescentNew::GradientDescentNew( const std::vector< std::vector< float > >& coefficient_matrix, const std::vector< float >& target_vector, + std::map< size_t, float > locked_columns, float learning_rate, int max_iterations, float convergence_threshold, @@ -46,6 +47,14 @@ GradientDescentNew::GradientDescentNew( const std::vector< std::vector< float > for ( auto& row : _coefficient_matrix ) { row.push_back( 1.0f ); } + + for ( int i = 0; i < _coefficient_matrix[ 0 ].size(); ++i ) { + if ( locked_columns.find( i ) != locked_columns.end() ) { + _locked_columns.push_back( locked_columns.at( i ) ); + } else { + _locked_columns.push_back( std::nullopt ); + } + } } void GradientDescentNew::print_input_matrix() @@ -81,8 +90,12 @@ GradientDescentResult GradientDescentNew::optimize() std::vector< float > gradient = compute_gradient( current_solution ); for ( size_t i = 0; i < current_solution.size(); ++i ) { - velocity[ i ] = _momentum * velocity[ i ] - _learning_rate * gradient[ i ]; - current_solution[ i ] += velocity[ i ]; + if ( _locked_columns[ i ] ) { + current_solution[ i ] = _locked_columns[ i ].value(); + } else { + velocity[ i ] = _momentum * velocity[ i ] - _learning_rate * gradient[ i ]; + current_solution[ i ] += velocity[ i ]; + } } std::vector< float > errors = compute_errors( current_solution ); @@ -146,7 +159,7 @@ bool GradientDescentNew::compute_convergence( const std::vector< float >& counts std::accumulate( counts_per_column.begin(), counts_per_column.end() - 1, 0.0f ) ); count_threshold = column_count_sum * 0.01f; - if ( error_variance > variance_threshold ) { + if ( error_variance > variance_threshold ) { return false; } From 25f3198b5771be0836a4ef5cc15bcdfc7931d1e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Mon, 18 Nov 2024 19:50:45 +0100 Subject: [PATCH 055/144] feat: print gradient descent information --- .../include/fuzzing/iid_node_dependencies.hpp | 20 +-- src/fuzzing/src/iid_node_dependencies.cpp | 136 ++++++++++++++---- 2 files changed, 117 insertions(+), 39 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp index 07670cbd..d7f4be4c 100644 --- a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp +++ b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp @@ -10,6 +10,7 @@ #include #include +#include #include #include @@ -37,10 +38,7 @@ struct Mean_counter { operator int() const { return static_cast< int >( value ); } - friend std::ostream& operator<<( std::ostream& os, Mean_counter const& m ) - { - return os << m.value; - } + friend std::ostream& operator<<( std::ostream& os, Mean_counter const& m ) { return os << m.value; } private: size_t count; @@ -68,8 +66,8 @@ struct path_decision { friend std::ostream& operator<<( std::ostream& os, path_decision const& pd ) { - return os << " left: " << pd.left_current << "/" << pd.left_max - << " right: " << pd.right_current << "/" << pd.right_max; + return os << " left: " << pd.left_current << "/" << pd.left_max << " right: " << pd.right_current + << "/" << pd.right_max; } bool get_next_direction(); @@ -134,14 +132,20 @@ struct iid_node_dependence_props { void add_equation( branching_node* path ); std::map< location_id, path_decision > generate_path(); - void compute_dependencies_by_loading( const loop_to_bodies_t& loop_heads_to_bodies, - branching_node* end_node ); + void compute_dependencies_by_loading( const loop_to_bodies_t& loop_heads_to_bodies, branching_node* end_node ); private: std::vector< float > approximate_matrix(); int get_possible_depth() const; void dependencies_generation(); + void get_best_subset( std::vector< std::vector< std::optional< float > > > const& table, + std::vector< std::set< node_direction > > const& subsets ); void print_dependencies(); + void print_subsets( std::set< node_direction > const& subset, + GradientDescentResult const& result, + std::vector< float > const& node_counts ); + void print_table( std::set< node_direction > const& all_leafs, + std::vector< std::vector< std::optional< float > > > const& table ); void compute_dependencies_by_loading( loading_loops_t& loading_loops, branching_node* end_node ); std::vector< std::vector< float > > get_matrix( std::set< node_direction > const& subset ) const; diff --git a/src/fuzzing/src/iid_node_dependencies.cpp b/src/fuzzing/src/iid_node_dependencies.cpp index b2c4a94d..d07152cf 100644 --- a/src/fuzzing/src/iid_node_dependencies.cpp +++ b/src/fuzzing/src/iid_node_dependencies.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include @@ -305,6 +305,78 @@ void fuzzing::iid_node_dependence_props::print_dependencies() std::cout << "- " << "`(" << body << ") → " << loading.id << "`" << std::endl; } } + + std::cout << "# Subsets:" << std::endl; +} + +void fuzzing::iid_node_dependence_props::print_subsets( std::set< node_direction > const& subset, + GradientDescentResult const& result, + std::vector< float > const& node_counts ) +{ + std::cout << "## Subset of nodes: `{ "; + auto delimeter = ""; + for ( const auto& leaf : subset ) { + std::cout << delimeter << "(" << leaf << ")"; + delimeter = ", "; + } + std::cout << " }`" << std::endl; + + std::cout << "### Weights and Counts:" << std::endl; + std::cout << "- `Iterations: " << result.iterations << "`" << std::endl; + std::cout << "- `Error variance: " << result.error_variance << "`" << std::endl; + std::cout << "- `Error mean: " << result.error_mean << "`" << std::endl; + std::cout << "- `Error square of mean: " << result.error_square_of_mean << "`" << std::endl; + std::cout << "- `Error mean of squares: " << result.error_mean_of_squares << "`" << std::endl; + std::cout << "- `Variance threshold: " << result.variance_threshold << "`" << std::endl; + std::cout << "- `Count threshold: " << result.count_threshold << "`" << std::endl; + std::cout << "- `Converged: " << ( result.converged ? "True" : "False" ) << "`" << std::endl; + + for ( size_t i = 0; i < subset.size(); ++i ) { + const auto& node = *std::next( subset.begin(), i ); + std::cout << "- `(" << node << "):`" << std::endl; + std::cout << " - `Weight: " << result.weights[ i ] << "`" << std::endl; + std::cout << " - `Count: " << static_cast< int >( std::round( node_counts[ i ] ) ) << "`" << std::endl; + std::cout << " - `Column count weighted: " << result.column_count_weighted[ i ] << "`" << std::endl; + } + + std::cout << "- `Increment:`\n - `Weight: " << result.weights.back() + << "`\n - `Column count weighted: " << result.column_count_weighted.back() << "`" << std::endl; +} + +void fuzzing::iid_node_dependence_props::print_table( std::set< node_direction > const& all_leafs, + std::vector< std::vector< std::optional< float > > > const& table ) +{ + std::cout << "## Result table:" << std::endl; + std::cout << "| "; + for ( const auto& node : all_leafs ) { + std::cout << node << " | "; + } + + std::cout << "Increment |" << " Error |" << " Converged |" << std::endl; + + std::cout << "| "; + for ( size_t i = 0; i < all_leafs.size(); ++i ) { + std::cout << "--- | "; + } + std::cout << "--- |" << "--- |" << "--- |" << std::endl; + + for ( const auto& row : table ) { + std::cout << "| "; + for ( size_t i = 0; i < row.size(); ++i ) { + const auto& value = row[ i ]; + if ( value.has_value() ) { + if ( i == row.size() - 1 ) { + std::cout << ( value.value() == 1 ? "True" : "False" ); + } else { + std::cout << value.value(); + } + } + std::cout << " | "; + } + std::cout << std::endl; + } + + std::cout << std::endl; } void fuzzing::iid_node_dependence_props::dependencies_generation() @@ -317,40 +389,44 @@ void fuzzing::iid_node_dependence_props::dependencies_generation() std::vector< std::set< node_direction > > subsets = get_subsets( all_leafs ); - std::cout << "# Subsets:" << std::endl; + std::vector< std::vector< std::optional< float > > > table; + for ( const auto& subset : subsets ) { std::vector< std::vector< float > > sub_matrix = get_matrix( subset ); - std::cout << "## Subset of nodes: `{ "; - auto delimeter = ""; - for ( const auto& leaf : subset ) { - std::cout << delimeter << "(" << leaf << ")"; - delimeter = ", "; - } - std::cout << " }`" << std::endl; + GradientDescentNew gd( sub_matrix, best_values ); + auto result = gd.optimize(); - GradientDescent gd( sub_matrix, best_values ); - auto [ weights, converged ] = gd.optimize(); + std::vector< float >& weights = result.weights; - float dot_product = - std::inner_product( weights.begin(), weights.end() - 1, weights.begin(), 0.0f ); + float dot_product = std::inner_product( weights.begin(), weights.end() - 1, weights.begin(), 0.0f ); std::vector< float > node_counts; for ( auto it = weights.begin(); it != weights.end() - 1; ++it ) { node_counts.push_back( -weights.back() * ( *it ) / dot_product ); } - std::cout << "### Weights and Counts:" << std::endl; - std::cout << "- `Converged: " << ( converged ? "True" : "False" ) << "`" << std::endl; - for ( size_t i = 0; i < subset.size(); ++i ) { - const auto& node = *std::next( subset.begin(), i ); - std::cout << "- `(" << node << "): " << weights[ i ] << " (" - << static_cast< int >( std::round( node_counts[ i ] ) ) << ")`" << std::endl; + + std::vector< std::optional< float > > row; + for ( const auto& node : all_leafs ) { + auto it = std::find( subset.begin(), subset.end(), node ); + if ( it != subset.end() ) { + row.push_back( weights[ std::distance( subset.begin(), it ) ] ); + } else { + row.push_back( std::nullopt ); + } } - std::cout << "- `Increment: " << weights.back() << "`" << std::endl; + row.push_back( weights.back() ); + row.push_back( result.error_mean ); + row.push_back( result.converged ); + + table.push_back( row ); + + print_subsets( subset, result, node_counts ); + // gd.print_input_matrix(); } - std::cout << std::endl; + print_table( all_leafs, table ); } std::map< location_id, fuzzing::path_decision > fuzzing::iid_node_dependence_props::generate_path() @@ -376,8 +452,8 @@ std::map< location_id, fuzzing::path_decision > fuzzing::iid_node_dependence_pro std::cout << "Path Depth" << it->second.path_depth << std::endl; std::cout << "Closest value to 0: " << it->first << std::endl; for ( const auto& [ direction, stats ] : it->second.direction_statistics ) { - std::cout << "Direction: " << direction << ", Min: " << stats.min - << ", Max: " << stats.max << ", Mean: " << stats.mean << std::endl; + std::cout << "Direction: " << direction << ", Min: " << stats.min << ", Max: " << stats.max + << ", Mean: " << stats.mean << std::endl; } } @@ -456,18 +532,18 @@ std::map< location_id, fuzzing::path_decision > fuzzing::iid_node_dependence_pro */ std::vector< float > fuzzing::iid_node_dependence_props::approximate_matrix() { - GradientDescent gd( matrix, best_values ); - auto [ weights, converged ] = gd.optimize(); + GradientDescentNew gd( matrix, best_values ); + GradientDescentResult result = gd.optimize(); if ( false ) { for ( size_t i = 0; i < interesting_nodes.size(); ++i ) { const auto& node = *std::next( interesting_nodes.begin(), i ); std::cout << "Node ID: " << node.node_id.id << ", Direction: " << node.branching_direction - << ", Weight: " << weights[ i ] << std::endl; + << ", Weight: " << result.weights[ i ] << std::endl; } } - return weights; + return result.weights; } @@ -499,8 +575,7 @@ int fuzzing::iid_node_dependence_props::get_possible_depth() const int second_depth = it->second.path_depth.min; float second_value = it->first; - int interpolated_depth = - linear_interpolation( first_value, first_depth, second_value, second_depth, 0 ); + int interpolated_depth = linear_interpolation( first_value, first_depth, second_value, second_depth, 0 ); return interpolated_depth; } @@ -665,8 +740,7 @@ std::vector< fuzzing::node_direction > fuzzing::get_path( branching_node* node ) while ( current != nullptr ) { branching_node* predecessor = current->predecessor; if ( predecessor != nullptr ) { - node_direction nav = { predecessor->get_location_id(), - predecessor->successor_direction( current ) }; + node_direction nav = { predecessor->get_location_id(), predecessor->successor_direction( current ) }; path.push_back( nav ); } current = predecessor; From ce3e4e0d017f0376b4f4c43bd0f8013e1ca4df63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Tue, 26 Nov 2024 22:27:32 +0100 Subject: [PATCH 056/144] feat: outputing --- .../gradient_descent_with_convergence.hpp | 12 +- .../include/fuzzing/iid_node_dependencies.hpp | 28 +++- .../src/gradient_descent_with_convergence.cpp | 9 +- src/fuzzing/src/iid_node_dependencies.cpp | 158 +++++++++++++++--- 4 files changed, 176 insertions(+), 31 deletions(-) diff --git a/src/fuzzing/include/fuzzing/gradient_descent_with_convergence.hpp b/src/fuzzing/include/fuzzing/gradient_descent_with_convergence.hpp index 37b3e312..9184a2d7 100644 --- a/src/fuzzing/include/fuzzing/gradient_descent_with_convergence.hpp +++ b/src/fuzzing/include/fuzzing/gradient_descent_with_convergence.hpp @@ -16,6 +16,16 @@ struct GradientDescentResult { float variance_threshold; float count_threshold; bool converged; + + bool operator==( const GradientDescentResult& other ) const + { + return weights == other.weights && errors == other.errors && + column_count_weighted == other.column_count_weighted && iterations == other.iterations && + error_mean == other.error_mean && error_square_of_mean == other.error_square_of_mean && + error_mean_of_squares == other.error_mean_of_squares && + error_variance == other.error_variance && variance_threshold == other.variance_threshold && + count_threshold == other.count_threshold && converged == other.converged; + } }; @@ -25,7 +35,7 @@ class GradientDescentNew { const std::vector< float >& target_vector, std::map< size_t, float > locked_columns = {}, float learning_rate = 0.01f, - int max_iterations = 1000, + int max_iterations = 500, float convergence_threshold = 1e-4, float momentum = 0.9f ); diff --git a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp index d7f4be4c..7337fab9 100644 --- a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp +++ b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp @@ -17,6 +17,26 @@ using loop_to_bodies_t = std::unordered_map< location_id, std::unordered_set< location_id > >; using loading_loops_t = std::unordered_map< location_id, std::tuple< natural_32_bit, natural_32_bit > >; +struct TableRow { + TableRow( std::vector< std::optional< float > > weights, GradientDescentResult result ) + : weights( std::move( weights ) ) + , result( std::move( result ) ) + {} + + bool operator==( const TableRow& other ) const + { + return weights == other.weights && result == other.result; + } + + int get_non_null_count() const + { + return std::count_if( weights.begin(), weights.end(), []( const auto& w ) { return w.has_value(); } ); + } + + std::vector< std::optional< float > > weights; + GradientDescentResult result; +}; + struct FloatComparator { bool operator()( const float& a, const float& b ) const { @@ -138,14 +158,14 @@ struct iid_node_dependence_props { std::vector< float > approximate_matrix(); int get_possible_depth() const; void dependencies_generation(); - void get_best_subset( std::vector< std::vector< std::optional< float > > > const& table, - std::vector< std::set< node_direction > > const& subsets ); + void get_best_subset( std::vector< TableRow > const& table, + std::vector< std::set< node_direction > > const& subsets, + std::set< node_direction > const& all_leafs ); void print_dependencies(); void print_subsets( std::set< node_direction > const& subset, GradientDescentResult const& result, std::vector< float > const& node_counts ); - void print_table( std::set< node_direction > const& all_leafs, - std::vector< std::vector< std::optional< float > > > const& table ); + void print_table( std::set< node_direction > const& all_leafs, std::vector< TableRow > const& table ); void compute_dependencies_by_loading( loading_loops_t& loading_loops, branching_node* end_node ); std::vector< std::vector< float > > get_matrix( std::set< node_direction > const& subset ) const; diff --git a/src/fuzzing/src/gradient_descent_with_convergence.cpp b/src/fuzzing/src/gradient_descent_with_convergence.cpp index 88b90384..f70e804b 100644 --- a/src/fuzzing/src/gradient_descent_with_convergence.cpp +++ b/src/fuzzing/src/gradient_descent_with_convergence.cpp @@ -83,6 +83,7 @@ GradientDescentResult GradientDescentNew::optimize() std::vector< float > prev_errors = compute_errors( current_solution ); std::vector< float > velocity( current_solution.size(), 0.0f ); float prev_cost = std::numeric_limits< float >::max(); + bool converged = false; int iterations = 0; @@ -101,16 +102,16 @@ GradientDescentResult GradientDescentNew::optimize() std::vector< float > errors = compute_errors( current_solution ); float current_cost = compute_mean_squared_error( errors ); - if ( std::abs( current_cost - prev_cost ) < _convergence_threshold ) { - break; - } + // if ( std::abs( current_cost - prev_cost ) < _convergence_threshold ) { + // break; + // } prev_cost = current_cost; prev_errors = errors; } float error_variance = compute_variance( prev_errors ); - float error_mean = compute_mean( prev_errors ); + float error_mean = std::abs( compute_mean( prev_errors ) ); std::vector< float > counts_per_column = compute_column_count( current_solution ); float variance_threshold = 0.0f; diff --git a/src/fuzzing/src/iid_node_dependencies.cpp b/src/fuzzing/src/iid_node_dependencies.cpp index d07152cf..30ce3423 100644 --- a/src/fuzzing/src/iid_node_dependencies.cpp +++ b/src/fuzzing/src/iid_node_dependencies.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include bool fuzzing::path_decision::get_next_direction() @@ -344,41 +345,154 @@ void fuzzing::iid_node_dependence_props::print_subsets( std::set< node_direction } void fuzzing::iid_node_dependence_props::print_table( std::set< node_direction > const& all_leafs, - std::vector< std::vector< std::optional< float > > > const& table ) + std::vector< TableRow > const& table ) { - std::cout << "## Result table:" << std::endl; + std::unordered_map< std::string, int > headers{ + { "Error mean", 0 }, + { "Error variance", 1 }, + // {"Error square of mean", 2}, + // {"Error mean of squares", 3}, + // {"Variance threshold", 4}, + { "Converged", 5 }, + // { "Iterations", 6 }, + // { "Variance threshold", 7 }, + // { "Count threshold", 8 }, + }; + + if ( std::pow( all_leafs.size(), 2 ) - 1 == table.size() ) { + std::cout << "# Result table:" << std::endl; + } else { + std::cout << "# Best result table:" << std::endl; + } + std::cout << "| "; for ( const auto& node : all_leafs ) { std::cout << node << " | "; } - - std::cout << "Increment |" << " Error |" << " Converged |" << std::endl; + std::cout << "Increment | "; + for ( const auto& header : headers ) { + std::cout << header.first << " | "; + } + std::cout << std::endl; std::cout << "| "; - for ( size_t i = 0; i < all_leafs.size(); ++i ) { + for ( size_t i = 0; i <= all_leafs.size() + headers.size(); ++i ) { std::cout << "--- | "; } - std::cout << "--- |" << "--- |" << "--- |" << std::endl; + std::cout << std::endl; for ( const auto& row : table ) { std::cout << "| "; - for ( size_t i = 0; i < row.size(); ++i ) { - const auto& value = row[ i ]; + for ( size_t i = 0; i < row.weights.size(); ++i ) { + const auto& value = row.weights[ i ]; if ( value.has_value() ) { - if ( i == row.size() - 1 ) { - std::cout << ( value.value() == 1 ? "True" : "False" ); - } else { - std::cout << value.value(); - } + std::cout << value.value(); + } + std::cout << " | "; + } + + for ( const auto& header : headers ) { + int idx = header.second; + switch ( idx ) { + case 0: std::cout << row.result.error_mean; break; + case 1: std::cout << row.result.error_variance; break; + case 2: std::cout << row.result.error_square_of_mean; break; + case 3: std::cout << row.result.error_mean_of_squares; break; + case 4: std::cout << row.result.variance_threshold; break; + case 5: std::cout << ( row.result.converged ? "True" : "False" ); break; + case 6: std::cout << row.result.iterations; break; + case 7: std::cout << row.result.variance_threshold; break; + case 8: std::cout << row.result.count_threshold; break; } std::cout << " | "; } + std::cout << std::endl; } std::cout << std::endl; } +void fuzzing::iid_node_dependence_props::get_best_subset( std::vector< TableRow > const& table, + std::vector< std::set< node_direction > > const& subsets, + std::set< node_direction > const& all_leafs ) +{ + std::vector< TableRow > table_copy = table; + + for ( const auto& row_locked : table_copy ) { + for ( int subset_idx = 0; subset_idx < subsets.size(); ++subset_idx ) { + auto& row_to_change = table_copy[ subset_idx ]; + + if ( row_locked == row_to_change ) { + continue; + } + + bool can_be_locked = true; + for ( size_t i = 0; i < row_locked.weights.size() - 1; ++i ) { + if ( row_locked.weights[ i ].has_value() && !row_to_change.weights[ i ].has_value() ) { + can_be_locked = false; + break; + } + } + + if ( !can_be_locked ) { + continue; + } + + std::set< node_direction > subset = subsets[ subset_idx ]; + + std::map< size_t, float > locked_columns; + int column_idx = 0; + for ( size_t i = 0; i < row_locked.weights.size() - 1; ++i ) { + if ( row_locked.weights[ i ].has_value() ) { + locked_columns[ column_idx ] = row_locked.weights[ i ].value(); + } + + if ( row_to_change.weights[ i ].has_value() ) { + column_idx++; + } + } + + GradientDescentNew gd( get_matrix( subset ), best_values, locked_columns ); + auto result = gd.optimize(); + + if ( result.error_mean < row_to_change.result.error_mean && result.converged ) { + std::cout << "- Error was reduced from `" << row_to_change.result.error_mean << "` to `" + << result.error_mean + << "`, converged: " << ( result.converged ? "`True`" : "`False`" ) << std::endl; + + for ( size_t i = 0; i < all_leafs.size(); ++i ) { + const auto& node = *std::next( all_leafs.begin(), i ); + + auto it = std::find( subset.begin(), subset.end(), node ); + if ( it != subset.end() ) { + row_to_change.weights[ i ] = result.weights[ std::distance( subset.begin(), it ) ]; + } else { + row_to_change.weights[ i ] = std::nullopt; + } + } + + row_to_change.result.error_mean = result.error_mean; + row_to_change.result.converged = row_to_change.result.converged || result.converged; + } + } + } + + std::sort( table_copy.begin(), table_copy.end(), []( const TableRow& a, const TableRow& b ) { + if ( a.get_non_null_count() != b.get_non_null_count() ) { + return a.get_non_null_count() < b.get_non_null_count(); + } + + return a.result.error_mean < b.result.error_mean; + } ); + + std::erase_if( table_copy, [ & ]( const TableRow& row ) { + return !row.result.converged && row.get_non_null_count() != all_leafs.size() + 1; + } ); + + print_table( all_leafs, table_copy ); +} + void fuzzing::iid_node_dependence_props::dependencies_generation() { print_dependencies(); @@ -389,7 +503,8 @@ void fuzzing::iid_node_dependence_props::dependencies_generation() std::vector< std::set< node_direction > > subsets = get_subsets( all_leafs ); - std::vector< std::vector< std::optional< float > > > table; + std::vector< TableRow > table; + for ( const auto& subset : subsets ) { std::vector< std::vector< float > > sub_matrix = get_matrix( subset ); @@ -397,7 +512,7 @@ void fuzzing::iid_node_dependence_props::dependencies_generation() GradientDescentNew gd( sub_matrix, best_values ); auto result = gd.optimize(); - std::vector< float >& weights = result.weights; + std::vector< float > const& weights = result.weights; float dot_product = std::inner_product( weights.begin(), weights.end() - 1, weights.begin(), 0.0f ); std::vector< float > node_counts; @@ -405,20 +520,18 @@ void fuzzing::iid_node_dependence_props::dependencies_generation() node_counts.push_back( -weights.back() * ( *it ) / dot_product ); } - - std::vector< std::optional< float > > row; + std::vector< std::optional< float > > weights_with_nulls; for ( const auto& node : all_leafs ) { auto it = std::find( subset.begin(), subset.end(), node ); if ( it != subset.end() ) { - row.push_back( weights[ std::distance( subset.begin(), it ) ] ); + weights_with_nulls.push_back( weights[ std::distance( subset.begin(), it ) ] ); } else { - row.push_back( std::nullopt ); + weights_with_nulls.push_back( std::nullopt ); } } - row.push_back( weights.back() ); - row.push_back( result.error_mean ); - row.push_back( result.converged ); + weights_with_nulls.push_back( weights.back() ); + TableRow row( weights_with_nulls, result ); table.push_back( row ); @@ -427,6 +540,7 @@ void fuzzing::iid_node_dependence_props::dependencies_generation() } print_table( all_leafs, table ); + get_best_subset( table, subsets, all_leafs ); } std::map< location_id, fuzzing::path_decision > fuzzing::iid_node_dependence_props::generate_path() From dba09c73594f9b58da6fc2d5f3848b7e9f37b391 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Wed, 27 Nov 2024 22:09:16 +0100 Subject: [PATCH 057/144] feat: vector --- src/fuzzing/src/iid_node_dependencies.cpp | 101 +++++++++++++++++++++- 1 file changed, 99 insertions(+), 2 deletions(-) diff --git a/src/fuzzing/src/iid_node_dependencies.cpp b/src/fuzzing/src/iid_node_dependencies.cpp index 30ce3423..c56c5984 100644 --- a/src/fuzzing/src/iid_node_dependencies.cpp +++ b/src/fuzzing/src/iid_node_dependencies.cpp @@ -536,18 +536,115 @@ void fuzzing::iid_node_dependence_props::dependencies_generation() table.push_back( row ); print_subsets( subset, result, node_counts ); - // gd.print_input_matrix(); + gd.print_input_matrix(); } print_table( all_leafs, table ); get_best_subset( table, subsets, all_leafs ); } +void fuzzing::iid_node_dependence_props::vector_computation() +{ + std::set< node_direction > all_leafs; + for ( const auto& [ _, loop_bodies ] : dependencies_by_loops ) { + all_leafs.insert( loop_bodies.begin(), loop_bodies.end() ); + } + + std::vector< std::vector< float > > full_matrix = get_matrix( all_leafs ); + std::set< std::vector< float > > unique_rows; + std::vector< float > new_best_values; + std::vector< std::vector< float > > matrix; + + for ( size_t i = 0; i < full_matrix.size(); ++i ) { + std::vector< float > row = full_matrix[ i ]; + row.push_back( best_values[ i ] ); + + if ( unique_rows.insert( row ).second ) { + new_best_values.push_back( best_values[ i ] ); + matrix.push_back( full_matrix[ i ] ); + } + } + + std::set< DirectionVector > vectors; + + for ( size_t i = 0; i < matrix.size(); ++i ) { + for ( size_t j = 0; j < matrix.size(); ++j ) { + if ( i == j ) { + continue; + } + + std::vector< float > diff_vector; + std::transform( matrix[ i ].begin(), + matrix[ i ].end(), + matrix[ j ].begin(), + std::back_inserter( diff_vector ), + std::minus< float >() ); + + if ( std::any_of( diff_vector.begin(), diff_vector.end(), []( float val ) { return val < 0; } ) ) { + continue; + } + + float best_value_diff = new_best_values[ i ] - new_best_values[ j ]; + if ( best_value_diff == 0 ) { + continue; + } + + vectors.insert( { diff_vector, best_value_diff } ); + } + } + + auto is_approximately_equal = []( const std::vector< float >& a, const std::vector< float >& b ) { + const float epsilon = 1e-6; + + if ( a.size() != b.size() ) + return false; + for ( size_t i = 0; i < a.size(); ++i ) { + if ( std::fabs( a[ i ] - b[ i ] ) > epsilon ) + return false; + } + return true; + }; + + std::set< DirectionVector > vectors_with_hits; + for ( const auto& vec : vectors ) { + int compare_hits = 0; + + for ( const auto& row : matrix ) { + std::vector< float > new_row; + for ( size_t i = 0; i < row.size(); ++i ) { + new_row.push_back( row[ i ] + vec.vector[ i ] ); + } + + for ( const auto& existing_row : matrix ) { + if ( is_approximately_equal( new_row, existing_row ) ) { + compare_hits++; + } + } + } + + vectors_with_hits.insert( { vec.vector, vec.value, compare_hits } ); + } + + std::cout << "Vectors:" << std::endl; + for ( const auto& vec : vectors_with_hits ) { + std::cout << "( "; + auto delimeter = ""; + for ( const auto& val : vec.vector ) { + std::cout << delimeter << val; + delimeter = ", "; + } + std::cout << " ) -> " << vec.value << " (" << vec.compare_hits << ")" << std::endl; + } +} + + std::map< location_id, fuzzing::path_decision > fuzzing::iid_node_dependence_props::generate_path() { - dependencies_generation(); + // dependencies_generation(); + vector_computation(); return {}; + std::vector< float > weights = approximate_matrix(); std::map< fuzzing::node_direction, int > path; From 96a080879bcbaeb036f34746f53b69192f968b19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Tue, 3 Dec 2024 18:11:50 +0100 Subject: [PATCH 058/144] feat: vector computation --- .../include/fuzzing/iid_node_dependencies.hpp | 46 ++- src/fuzzing/src/iid_node_dependencies.cpp | 370 ++++++++++++------ 2 files changed, 296 insertions(+), 120 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp index 7337fab9..be6330be 100644 --- a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp +++ b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp @@ -17,6 +17,41 @@ using loop_to_bodies_t = std::unordered_map< location_id, std::unordered_set< location_id > >; using loading_loops_t = std::unordered_map< location_id, std::tuple< natural_32_bit, natural_32_bit > >; +struct DirectionVector { + DirectionVector( std::vector< float > vector, float value ) + : DirectionVector( std::move( vector ), value, 0 ) + {} + + DirectionVector( std::vector< float > vector, float value, int compare_hits ) + : vector( std::move( vector ) ) + , value( value ) + , compare_hits( compare_hits ) + {} + + std::vector< float > vector; + float value; + int compare_hits; + + bool operator<( const DirectionVector& other ) const + { + // if ( value != other.value ) { + // return value < other.value; + // } + + return get_vector_length() < other.get_vector_length(); + } + + bool operator==( const DirectionVector& other ) const + { + return std::tie( vector, value ) == std::tie( other.vector, other.value ); + } + + float get_vector_length() const + { + return std::sqrt( std::inner_product( vector.begin(), vector.end(), vector.begin(), 0.0f ) ); + } +}; + struct TableRow { TableRow( std::vector< std::optional< float > > weights, GradientDescentResult result ) : weights( std::move( weights ) ) @@ -141,7 +176,7 @@ struct iid_node_dependence_props { std::vector< std::vector< float > > matrix; std::vector< float > best_values; - std::unordered_map< location_id, std::set< node_direction > > dependencies_by_loops; + std::unordered_map< std::pair< location_id, bool >, std::set< node_direction > > dependencies_by_loops; std::unordered_map< location_id, std::set< node_direction > > dependencies_by_loading; coverage_value_props all_cov_value_props; @@ -151,6 +186,7 @@ struct iid_node_dependence_props { void recompute_matrix(); void add_equation( branching_node* path ); std::map< location_id, path_decision > generate_path(); + std::unordered_map< location_id::id_type, float > generate_probabilities(); void compute_dependencies_by_loading( const loop_to_bodies_t& loop_heads_to_bodies, branching_node* end_node ); @@ -162,12 +198,20 @@ struct iid_node_dependence_props { std::vector< std::set< node_direction > > const& subsets, std::set< node_direction > const& all_leafs ); void print_dependencies(); + std::vector< node_direction > get_all_leafs(); + std::map< location_id::id_type, std::pair< float, float > > + compute_counts_from_leaf_counts( std::vector< float > const& leaf_counts, + std::vector< node_direction > all_leafs ); + std::tuple< std::vector< std::vector< float > >, std::vector< float > > + get_unique_matrix_and_values( std::vector< std::vector< float > > const& full_matrix ); + std::set< DirectionVector > vector_computation(); void print_subsets( std::set< node_direction > const& subset, GradientDescentResult const& result, std::vector< float > const& node_counts ); void print_table( std::set< node_direction > const& all_leafs, std::vector< TableRow > const& table ); void compute_dependencies_by_loading( loading_loops_t& loading_loops, branching_node* end_node ); + std::vector< std::vector< float > > get_matrix( std::vector< node_direction > const& subset ) const; std::vector< std::vector< float > > get_matrix( std::set< node_direction > const& subset ) const; std::vector< std::set< node_direction > > get_subsets( std::set< node_direction > const& all_leafs ); }; diff --git a/src/fuzzing/src/iid_node_dependencies.cpp b/src/fuzzing/src/iid_node_dependencies.cpp index c56c5984..68615fac 100644 --- a/src/fuzzing/src/iid_node_dependencies.cpp +++ b/src/fuzzing/src/iid_node_dependencies.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include bool fuzzing::path_decision::get_next_direction() @@ -271,6 +272,12 @@ fuzzing::iid_node_dependence_props::get_subsets( std::set< node_direction > cons std::vector< std::vector< float > > fuzzing::iid_node_dependence_props::get_matrix( std::set< node_direction > const& subset ) const +{ + return get_matrix( std::vector< node_direction >( subset.begin(), subset.end() ) ); +} + +std::vector< std::vector< float > > +fuzzing::iid_node_dependence_props::get_matrix( std::vector< node_direction > const& subset ) const { std::vector< std::vector< float > > sub_matrix; for ( const auto& row : matrix ) { @@ -294,7 +301,7 @@ void fuzzing::iid_node_dependence_props::print_dependencies() std::cout << "## Dependencies by loops:" << std::endl; for ( const auto& [ loop, nodes ] : dependencies_by_loops ) { for ( const auto& body : nodes ) { - std::cout << "- " << "`(" << body << ") → " << loop.id << "`" << std::endl; + std::cout << "- " << "`(" << body << ") → " << loop.first.id << "`" << std::endl; } } @@ -543,20 +550,25 @@ void fuzzing::iid_node_dependence_props::dependencies_generation() get_best_subset( table, subsets, all_leafs ); } -void fuzzing::iid_node_dependence_props::vector_computation() +std::vector< fuzzing::node_direction > fuzzing::iid_node_dependence_props::get_all_leafs() { std::set< node_direction > all_leafs; for ( const auto& [ _, loop_bodies ] : dependencies_by_loops ) { all_leafs.insert( loop_bodies.begin(), loop_bodies.end() ); } - std::vector< std::vector< float > > full_matrix = get_matrix( all_leafs ); + return std::vector< node_direction >( all_leafs.begin(), all_leafs.end() ); +} + +std::tuple< std::vector< std::vector< float > >, std::vector< float > > +fuzzing::iid_node_dependence_props::get_unique_matrix_and_values( std::vector< std::vector< float > > const& full_matrix ) +{ std::set< std::vector< float > > unique_rows; std::vector< float > new_best_values; std::vector< std::vector< float > > matrix; for ( size_t i = 0; i < full_matrix.size(); ++i ) { - std::vector< float > row = full_matrix[ i ]; + auto row = full_matrix[ i ]; row.push_back( best_values[ i ] ); if ( unique_rows.insert( row ).second ) { @@ -565,171 +577,275 @@ void fuzzing::iid_node_dependence_props::vector_computation() } } - std::set< DirectionVector > vectors; + return { matrix, new_best_values }; +} + +std::set< DirectionVector > fuzzing::iid_node_dependence_props::vector_computation() +{ + std::vector< fuzzing::node_direction > all_leafs = get_all_leafs(); + std::vector< std::vector< float > > full_matrix = get_matrix( all_leafs ); + auto [ matrix, new_best_values ] = get_unique_matrix_and_values( full_matrix ); + + std::set< DirectionVector > vectors_with_hits; + + auto is_approximately_equal = []( const auto& a, const auto& b ) { + constexpr float epsilon = 1e-6; + return a.size() == b.size() && std::equal( a.begin(), a.end(), b.begin(), [ epsilon ]( float x, float y ) { + return std::fabs( x - y ) <= epsilon; + } ); + }; for ( size_t i = 0; i < matrix.size(); ++i ) { for ( size_t j = 0; j < matrix.size(); ++j ) { - if ( i == j ) { + if ( i == j ) continue; - } std::vector< float > diff_vector; std::transform( matrix[ i ].begin(), matrix[ i ].end(), matrix[ j ].begin(), std::back_inserter( diff_vector ), - std::minus< float >() ); + std::minus<>() ); - if ( std::any_of( diff_vector.begin(), diff_vector.end(), []( float val ) { return val < 0; } ) ) { + if ( std::any_of( diff_vector.begin(), diff_vector.end(), []( float val ) { return val < 0; } ) ) continue; - } float best_value_diff = new_best_values[ i ] - new_best_values[ j ]; - if ( best_value_diff == 0 ) { - continue; - } + if ( best_value_diff != 0 ) { + int compare_hits = 0; + + for ( const auto& row : matrix ) { + std::vector< float > new_row( row.size() ); + std::transform( row.begin(), row.end(), diff_vector.begin(), new_row.begin(), std::plus<>() ); + + if ( std::any_of( matrix.begin(), matrix.end(), [ & ]( const auto& existing_row ) { + return is_approximately_equal( new_row, existing_row ); + } ) ) { + compare_hits++; + } + } - vectors.insert( { diff_vector, best_value_diff } ); + vectors_with_hits.insert( { diff_vector, best_value_diff, compare_hits } ); + } } } - auto is_approximately_equal = []( const std::vector< float >& a, const std::vector< float >& b ) { - const float epsilon = 1e-6; + // std::cout << "Vectors:\n"; + // for ( const auto& vec : vectors_with_hits ) { + // std::cout << "( "; + // for ( size_t i = 0; i < vec.vector.size(); ++i ) { + // std::cout << ( i ? ", " : "" ) << vec.vector[ i ]; + // } + // std::cout << " ) -> " << vec.value << " (" << vec.compare_hits << ")\n"; + // } - if ( a.size() != b.size() ) - return false; - for ( size_t i = 0; i < a.size(); ++i ) { - if ( std::fabs( a[ i ] - b[ i ] ) > epsilon ) - return false; - } - return true; - }; + return vectors_with_hits; +} - std::set< DirectionVector > vectors_with_hits; +std::unordered_map< location_id::id_type, float > fuzzing::iid_node_dependence_props::generate_probabilities() +{ + // dependencies_generation(); + std::vector< node_direction > all_leafs = get_all_leafs(); + auto [ matrix, new_best_values ] = get_unique_matrix_and_values( get_matrix( all_leafs ) ); + std::set< DirectionVector > vectors = vector_computation(); + + float epsilon = 1e-6; + int column_count = matrix[ 0 ].size(); + + std::set< std::vector< float > > paths; + // std::cout << "Path counts:" << std::endl; for ( const auto& vec : vectors ) { - int compare_hits = 0; - for ( const auto& row : matrix ) { - std::vector< float > new_row; - for ( size_t i = 0; i < row.size(); ++i ) { - new_row.push_back( row[ i ] + vec.vector[ i ] ); + for ( int i = 0; i < matrix.size(); ++i ) { + std::vector< float > path_counts( column_count ); + + float curr_best_value = std::abs( new_best_values[ i ] ); + + float counts = curr_best_value / vec.value; + int counts_int = static_cast< int >( std::round( counts ) ); + if ( std::abs( counts - counts_int ) > epsilon ) { + continue; } - for ( const auto& existing_row : matrix ) { - if ( is_approximately_equal( new_row, existing_row ) ) { - compare_hits++; - } + for ( int j = 0; j < column_count; ++j ) { + path_counts[ j ] = matrix[ i ][ j ] + vec.vector[ j ] * counts_int; } - } - vectors_with_hits.insert( { vec.vector, vec.value, compare_hits } ); - } + if ( std::any_of( path_counts.begin(), path_counts.end(), []( float val ) { return val < 0; } ) ) { + continue; + } - std::cout << "Vectors:" << std::endl; - for ( const auto& vec : vectors_with_hits ) { - std::cout << "( "; - auto delimeter = ""; - for ( const auto& val : vec.vector ) { - std::cout << delimeter << val; - delimeter = ", "; + // std::cout << "( "; + // for ( size_t j = 0; j < column_count; ++j ) { + // std::cout << ( j ? ", " : "" ) << path_counts[ j ]; + // } + // std::cout << " ) " << std::endl; + + + paths.insert( path_counts ); } - std::cout << " ) -> " << vec.value << " (" << vec.compare_hits << ")" << std::endl; } -} + std::vector< std::vector< float > > sorted_paths( paths.begin(), paths.end() ); + std::sort( sorted_paths.begin(), + sorted_paths.end(), + []( const std::vector< float >& a, const std::vector< float >& b ) { + float a_length = std::inner_product( a.begin(), a.end(), a.begin(), 0.0f ); + float b_length = std::inner_product( b.begin(), b.end(), b.begin(), 0.0f ); + return a_length < b_length; + } ); -std::map< location_id, fuzzing::path_decision > fuzzing::iid_node_dependence_props::generate_path() -{ - // dependencies_generation(); - vector_computation(); - return {}; + // for ( const auto& path : sorted_paths ) { + // std::cout << "( "; + // for ( size_t i = 0; i < path.size(); ++i ) { + // std::cout << ( i ? ", " : "" ) << path[ i ]; + // } + // std::cout << " )" << std::endl; + // } + std::map< location_id::id_type, std::pair< float, float > > path_counts = compute_counts_from_leaf_counts( sorted_paths[ 0 ], all_leafs ); + for (const auto& [id, counts] : path_counts) { + std::cout << "Location ID: " << id << ", Left Count: " << counts.first << ", Right Count: " << counts.second << std::endl; + } - std::vector< float > weights = approximate_matrix(); + std::unordered_map< location_id::id_type, float > path_probabilities; + for ( int i = 0; i < path_counts.size(); i += 2 ) {} - std::map< fuzzing::node_direction, int > path; - int path_size = 0; - int possible_depth = get_possible_depth(); + // for ( int i = 0; i < all_leafs.size(); i += 2 ) { + // int sum = sorted_paths[ 0 ][ i ] + sorted_paths[ 0 ][ i + 1 ]; + // location_id::id_type id = std::next( all_leafs.begin(), i )->node_id.id; + // if ( sum > 0 ) + // path_probabilities[ id ] = sorted_paths[ 0 ][ i ] / sum; + // } + + // for (const auto& [id, count] : path_probabilities) { + // std::cout << "Location ID: " << id << ", Path Probability: " << count << std::endl; + // } - for ( const auto& [ dir, stats ] : all_cov_value_props.direction_statistics ) { - if ( stats.min == stats.max ) { - path[ dir ] = stats.min; - path_size += stats.min; + return path_probabilities; + return {}; +} + +std::map< location_id::id_type, std::pair< float, float > > +fuzzing::iid_node_dependence_props::compute_counts_from_leaf_counts( std::vector< float > const& leaf_counts, + std::vector< node_direction > all_leafs ) +{ + std::map< location_id::id_type, std::pair< float, float > > path_counts; + for ( int i = 0; i < all_leafs.size(); i += 2 ) { + float leaf_count = leaf_counts[ i ]; + node_direction leaf = all_leafs[ i ]; + + auto& [ left_count, right_count ] = path_counts[ leaf.node_id.id ]; + if ( leaf.branching_direction ) { + right_count = leaf_count; + } else { + left_count = leaf_count; } } - if ( false ) { - auto it = cov_values_to_props.begin(); - std::cout << "Path Depth" << it->second.path_depth << std::endl; - std::cout << "Closest value to 0: " << it->first << std::endl; - for ( const auto& [ direction, stats ] : it->second.direction_statistics ) { - std::cout << "Direction: " << direction << ", Min: " << stats.min << ", Max: " << stats.max - << ", Mean: " << stats.mean << std::endl; + for ( const auto& [ loop_head, dependent_bodies ] : dependencies_by_loops ) { + float loop_count = 0; + for ( const auto& body : dependent_bodies ) { + auto& [ left_count, right_count ] = path_counts[ body.node_id.id ]; + loop_count += right_count; + loop_count += left_count; + } + + location_id head_id = loop_head.first; + bool end_direction = loop_head.second; + + if ( end_direction ) { + path_counts[ head_id.id ] = { 1, loop_count }; + } else { + path_counts[ head_id.id ] = { loop_count, 1 }; } } - int computed_size = 0; - std::map< fuzzing::node_direction, int > computed_path; - for ( size_t i = 0; i < interesting_nodes.size(); ++i ) { - const auto& node = *std::next( interesting_nodes.begin(), i ); + return path_counts; +} - auto it = cov_values_to_props.begin(); - int x_1 = it->first; - int y_1 = it->second.direction_statistics.at( node ).mean; - ++it; +std::map< location_id, fuzzing::path_decision > fuzzing::iid_node_dependence_props::generate_path() +{ + // // dependencies_generation(); + // std::vector< node_direction > all_leafs = get_all_leafs(); + // auto [ matrix, new_best_values ] = get_unique_matrix_and_values( get_matrix( all_leafs ) ); + // std::set< DirectionVector > vectors = vector_computation(); - int x_2 = it->first; - int y_2 = it->second.direction_statistics.at( node ).mean; + // // auto min_value_it = std::min_element( best_values.begin(), best_values.end() ); + // // int min_index = std::distance( best_values.begin(), min_value_it ); + // // std::vector< float > min_row = matrix[ min_index ]; - int interpolated_y = linear_interpolation( x_1, y_1, x_2, y_2, 0 ); - int computed_count = static_cast< int >( interpolated_y * weights[ i ] ); - computed_count = interpolated_y; - computed_count = std::max( 0, computed_count ); - computed_size += computed_count; - computed_path[ node ] = computed_count; - } + // float epsilon = 1e-6; + // int column_count = matrix[ 0 ].size(); - float scale = static_cast< float >( possible_depth - path_size ) / computed_size; - for ( auto& [ node, count ] : computed_path ) { - // std::cout << "Count before scaling: " << count << std::endl; - count = static_cast< int >( count * scale ); - // std::cout << "Count after scaling: " << count << std::endl; - } + // std::set< std::vector< float > > paths; + // std::cout << "Path counts:" << std::endl; + // for ( const auto& vec : vectors ) { - path.insert( computed_path.begin(), computed_path.end() ); + // for ( int i = 0; i < matrix.size(); ++i ) { + // std::vector< float > path_counts( column_count ); - std::map< location_id, path_decision > decisions; - for ( int i = 1; i < path.size(); ++i ) { - auto first_p = std::next( path.begin(), i - 1 ); - auto second_p = std::next( path.begin(), i ); + // float curr_best_value = std::abs( new_best_values[ i ] ); - if ( first_p->first.node_id == second_p->first.node_id ) { - decisions[ first_p->first.node_id ] = { first_p->second, second_p->second }; - ++i; - } else { - if ( first_p->first.branching_direction ) { - decisions[ first_p->first.node_id ] = { 0, first_p->second }; - } else { - decisions[ first_p->first.node_id ] = { first_p->second, 0 }; - } - } - } + // float counts = curr_best_value / vec.value; + // int counts_int = static_cast< int >( std::round( counts ) ); + // if ( std::abs( counts - counts_int ) > epsilon ) { + // continue; + // } - if ( false ) { - for ( const auto& [ location, decision ] : decisions ) { - std::cout << location.id << decision << std::endl; - } - } + // for ( int j = 0; j < column_count; ++j ) { + // path_counts[ j ] = matrix[ i ][ j ] + vec.vector[ j ] * counts_int; + // } - if ( false ) { - for ( const auto& [ node, count ] : path ) { - std::cout << "Node ID: " << node.node_id.id << ", Direction: " << node.branching_direction - << ", Count: " << count << std::endl; - } - } + // if ( std::any_of( path_counts.begin(), path_counts.end(), []( float val ) { return val < 0; } ) ) { + // continue; + // } + + // // std::cout << "( "; + // // for ( size_t j = 0; j < column_count; ++j ) { + // // std::cout << ( j ? ", " : "" ) << path_counts[ j ]; + // // } + // // std::cout << " ) " << std::endl; + + + // paths.insert( path_counts ); + // } + // } - return decisions; + // std::vector< std::vector< float > > sorted_paths( paths.begin(), paths.end() ); + // std::sort( sorted_paths.begin(), + // sorted_paths.end(), + // []( const std::vector< float >& a, const std::vector< float >& b ) { + // float a_length = std::inner_product( a.begin(), a.end(), a.begin(), 0.0f ); + // float b_length = std::inner_product( b.begin(), b.end(), b.begin(), 0.0f ); + // return a_length < b_length; + // } ); + + // // for ( const auto& path : sorted_paths ) { + // // std::cout << "( "; + // // for ( size_t i = 0; i < path.size(); ++i ) { + // // std::cout << ( i ? ", " : "" ) << path[ i ]; + // // } + // // std::cout << " )" << std::endl; + // // } + + // std::unordered_map< location_id::id_type, float > path_counts; + // INVARIANT( all_leafs.size() % 2 == 0 ); + + // for ( int i = 0; i < all_leafs.size(); i += 2 ) { + // int sum = sorted_paths[ 0 ][ i ] + sorted_paths[ 0 ][ i + 1 ]; + // location_id::id_type id = std::next( all_leafs.begin(), i )->node_id.id; + // if ( sum > 0 ) + // path_counts[ id ] = sorted_paths[ 0 ][ i ] / sum; + // else + // path_counts[ id ] = 0.5; + // } + + // for ( const auto& [ id, count ] : path_counts ) { + // std::cout << "Location ID: " << id << ", Path Count: " << count << std::endl; + // } + return {}; } @@ -907,14 +1023,30 @@ void fuzzing::iid_dependencies::process_node_dependence( branching_node* node ) fuzzing::fuzzer::detect_loops_along_path_to_node( node, loop_heads_to_bodies, nullptr ); props.compute_dependencies_by_loading( loop_heads_to_bodies, node ); + std::map< location_id, bool > loop_heads_ending; + branching_node* current = node; + while ( current != nullptr ) { + branching_node* predecessor = current->predecessor; + if ( predecessor != nullptr && loop_heads_to_bodies.contains( predecessor->get_location_id() ) && + !loop_heads_ending.contains( predecessor->get_location_id() ) ) { + bool direction = predecessor->successor( true ).pointer->get_location_id() == + current->get_location_id(); + loop_heads_ending[ predecessor->get_location_id() ] = direction; + } + current = predecessor; + } + for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { + INVARIANT( loop_heads_ending.contains( loop_head ) ); + bool loop_head_direction = loop_heads_ending[ loop_head ]; + for ( const auto& body : loop_bodies ) { if ( props.interesting_nodes.contains( { body, true } ) ) { - props.dependencies_by_loops[ loop_head ].insert( { body, true } ); + props.dependencies_by_loops[ { loop_head, loop_head_direction } ].insert( { body, true } ); } if ( props.interesting_nodes.contains( { body, false } ) ) { - props.dependencies_by_loops[ loop_head ].insert( { body, false } ); + props.dependencies_by_loops[ { loop_head, loop_head_direction } ].insert( { body, false } ); } } } From 36677ab3572287fd1f48d274121d8f47890ddf9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 21 Dec 2024 20:09:47 +0100 Subject: [PATCH 059/144] feat: new file for analysis --- .../include/fuzzing/iid_vector_analysis.hpp | 57 ++++++++++++++++ src/fuzzing/src/iid_vector_analysis.cpp | 68 +++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 src/fuzzing/include/fuzzing/iid_vector_analysis.hpp create mode 100644 src/fuzzing/src/iid_vector_analysis.cpp diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp new file mode 100644 index 00000000..1d23a5f1 --- /dev/null +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -0,0 +1,57 @@ +#pragma once + +#include +#include +#include + +#include +#include + +struct node_direction; +using loop_ending_to_bodies = std::map< std::pair< location_id, bool >, std::set< node_direction > >; +using loop_endings = std::map< location_id, bool >; + +namespace fuzzing +{ +struct node_direction {}; +struct direction_vector {}; + + +struct equation { + std::vector< float > values; + float best_value; +}; + + +struct equation_matrix { + equation_matrix get_submatrix( std::set< node_direction > const& subset, bool unique ) const; + void add_equation( branching_node* end_node ); + +private: + std::vector< equation > matrix; +}; + + +struct iid_node_dependence_props { + std::unordered_map< location_id::id_type, float > generate_probabilities(); + void process_node( branching_node* end_node ); + +private: + loop_endings get_loop_heads_ending( branching_node* end_node ) const; + void compute_dependencies_by_loading( branching_node* end_node, const loop_endings& loop_heads_ending ); + void compute_dependencies_by_loops( branching_node* end_node, const loop_endings& loop_heads_ending ); + + equation_matrix matrix; + loop_ending_to_bodies dependencies_by_loops; + loop_ending_to_bodies dependencies_by_loading; +}; + +struct iid_dependencies { + void update_non_iid_nodes( const sensitivity_analysis& sensitivity ); + void process_node_dependence( branching_node* node ); + +private: + std::unordered_map< location_id, iid_node_dependence_props > id_to_equation_map; + std::set< location_id > non_iid_nodes; +}; +} // namespace fuzzing diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp new file mode 100644 index 00000000..68ffee71 --- /dev/null +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -0,0 +1,68 @@ +#include +#include +#include + +// ------------------------------------------------------------------------------------------------ +void fuzzing::iid_node_dependence_props::process_node( branching_node* end_node ) +{ + loop_endings loop_heads_ending = get_loop_heads_ending( end_node ); + compute_dependencies_by_loading( end_node, loop_heads_ending ); + compute_dependencies_by_loops( end_node, loop_heads_ending ); +} + +// ------------------------------------------------------------------------------------------------ +std::map< location_id, bool > fuzzing::iid_node_dependence_props::get_loop_heads_ending( branching_node* end_node ) const +{ + std::unordered_map< location_id, std::unordered_set< location_id > > loop_heads_to_bodies; + std::vector< fuzzer::loop_boundary_props > loops; + + fuzzing::fuzzer::detect_loops_along_path_to_node( end_node, loop_heads_to_bodies, &loops ); + + std::map< location_id, bool > loop_heads_ending; + + auto is_outside_loop = [ & ]( branching_node* successor, + location_id loop_head_id, + const std::unordered_set< location_id >& loop_bodies ) { + return successor != nullptr && successor->get_location_id() != loop_head_id && + !loop_bodies.contains( successor->get_location_id() ); + }; + + for ( const auto& loop : loops ) { + location_id loop_head_id = loop.exit->get_location_id(); + const auto& loop_bodies = loop_heads_to_bodies[ loop_head_id ]; + + branching_node* loop_end_node = loop.exit; + branching_node* left_successor = loop_end_node->successor( false ).pointer; + branching_node* right_successor = loop_end_node->successor( true ).pointer; + + if ( is_outside_loop( left_successor, loop_head_id, loop_bodies ) ) { + loop_heads_ending[ loop_end_node->get_location_id() ] = false; + } else if ( is_outside_loop( right_successor, loop_head_id, loop_bodies ) ) { + loop_heads_ending[ loop_end_node->get_location_id() ] = true; + } + } + + return loop_heads_ending; +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::iid_node_dependence_props::compute_dependencies_by_loading( branching_node* end_node, + const loop_endings& loop_heads_ending ) +{} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::iid_node_dependence_props::compute_dependencies_by_loops( branching_node* end_node, + const loop_endings& loop_heads_ending ) +{} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::iid_dependencies::update_non_iid_nodes( const sensitivity_analysis& sensitivity ) {} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::iid_dependencies::process_node_dependence( branching_node* node ) +{ + TMPROF_BLOCK(); + + if ( non_iid_nodes.contains( node->get_location_id() ) ) + return; +} From a218698806e49194a3a5dafa80aa1cc2cc66f083 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sun, 22 Dec 2024 10:36:09 +0100 Subject: [PATCH 060/144] feat: node processing --- .../include/fuzzing/iid_vector_analysis.hpp | 21 +++- src/fuzzing/src/iid_vector_analysis.cpp | 105 +++++++++++++++--- 2 files changed, 106 insertions(+), 20 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index 1d23a5f1..95de5fae 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -10,12 +10,18 @@ struct node_direction; using loop_ending_to_bodies = std::map< std::pair< location_id, bool >, std::set< node_direction > >; using loop_endings = std::map< location_id, bool >; +using loop_head_to_bodies_t = std::unordered_map< location_id, std::unordered_set< location_id > >; +using loop_head_to_loaded_bits_t = std::unordered_map< location_id, std::tuple< natural_32_bit, natural_32_bit > >; namespace fuzzing { -struct node_direction {}; struct direction_vector {}; +struct node_direction { + location_id node_id; + bool branching_direction; +}; + struct equation { std::vector< float > values; @@ -37,9 +43,14 @@ struct iid_node_dependence_props { void process_node( branching_node* end_node ); private: - loop_endings get_loop_heads_ending( branching_node* end_node ) const; - void compute_dependencies_by_loading( branching_node* end_node, const loop_endings& loop_heads_ending ); - void compute_dependencies_by_loops( branching_node* end_node, const loop_endings& loop_heads_ending ); + loop_endings get_loop_heads_ending( branching_node* end_node, + const loop_head_to_bodies_t& loop_heads_to_bodies, + const std::vector< fuzzer::loop_boundary_props >& loops ) const; + void compute_dependencies_by_loading( branching_node* end_node, + const loop_head_to_bodies_t& loop_heads_to_bodies, + const loop_endings& loop_heads_ending ); + void compute_dependencies_by_loops( const loop_head_to_bodies_t& loop_heads_to_bodies, + const loop_endings& loop_heads_ending ); equation_matrix matrix; loop_ending_to_bodies dependencies_by_loops; @@ -47,7 +58,7 @@ struct iid_node_dependence_props { }; struct iid_dependencies { - void update_non_iid_nodes( const sensitivity_analysis& sensitivity ); + void update_non_iid_nodes( sensitivity_analysis& sensitivity ); void process_node_dependence( branching_node* node ); private: diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 68ffee71..a55f75f3 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -3,21 +3,24 @@ #include // ------------------------------------------------------------------------------------------------ -void fuzzing::iid_node_dependence_props::process_node( branching_node* end_node ) +void fuzzing::iid_node_dependence_props::process_node( branching_node* end_node ) { - loop_endings loop_heads_ending = get_loop_heads_ending( end_node ); - compute_dependencies_by_loading( end_node, loop_heads_ending ); - compute_dependencies_by_loops( end_node, loop_heads_ending ); + loop_head_to_bodies_t loop_heads_to_bodies; + std::vector< fuzzer::loop_boundary_props > loops; + fuzzing::fuzzer::detect_loops_along_path_to_node( end_node, loop_heads_to_bodies, &loops ); + + loop_endings loop_heads_ending = get_loop_heads_ending( end_node, loop_heads_to_bodies, loops ); + + compute_dependencies_by_loading( end_node, loop_heads_to_bodies, loop_heads_ending ); + compute_dependencies_by_loops( loop_heads_to_bodies, loop_heads_ending ); } // ------------------------------------------------------------------------------------------------ -std::map< location_id, bool > fuzzing::iid_node_dependence_props::get_loop_heads_ending( branching_node* end_node ) const +std::map< location_id, bool > fuzzing::iid_node_dependence_props::get_loop_heads_ending( + branching_node* end_node, + const loop_head_to_bodies_t& loop_heads_to_bodies, + const std::vector< fuzzer::loop_boundary_props >& loops ) const { - std::unordered_map< location_id, std::unordered_set< location_id > > loop_heads_to_bodies; - std::vector< fuzzer::loop_boundary_props > loops; - - fuzzing::fuzzer::detect_loops_along_path_to_node( end_node, loop_heads_to_bodies, &loops ); - std::map< location_id, bool > loop_heads_ending; auto is_outside_loop = [ & ]( branching_node* successor, @@ -29,7 +32,7 @@ std::map< location_id, bool > fuzzing::iid_node_dependence_props::get_loop_heads for ( const auto& loop : loops ) { location_id loop_head_id = loop.exit->get_location_id(); - const auto& loop_bodies = loop_heads_to_bodies[ loop_head_id ]; + const auto& loop_bodies = loop_heads_to_bodies.at( loop_head_id ); branching_node* loop_end_node = loop.exit; branching_node* left_successor = loop_end_node->successor( false ).pointer; @@ -47,16 +50,85 @@ std::map< location_id, bool > fuzzing::iid_node_dependence_props::get_loop_heads // ------------------------------------------------------------------------------------------------ void fuzzing::iid_node_dependence_props::compute_dependencies_by_loading( branching_node* end_node, + const loop_head_to_bodies_t& loop_heads_to_bodies, const loop_endings& loop_heads_ending ) -{} +{ + loop_head_to_loaded_bits_t loading_loops; + for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { + loading_loops[ loop_head ] = { std::numeric_limits< natural_32_bit >::max(), + std::numeric_limits< natural_32_bit >::min() }; + } + + branching_node* node = end_node; + while ( node != nullptr ) { + for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { + if ( loop_head.id == node->get_location_id().id ) { + natural_32_bit bits_count = node->get_num_stdin_bits(); + + auto& [ min, max ] = loading_loops[ loop_head ]; + min = std::min( min, bits_count ); + max = std::max( max, bits_count ); + } + } + + node = node->predecessor; + } + + node = end_node; + + while ( node != nullptr ) { + auto node_id = node->get_location_id(); + + for ( const auto& bit_index : node->sensitive_stdin_bits ) { + for ( const auto& [ loop_head, values ] : loading_loops ) { + if ( !loop_heads_ending.contains( loop_head ) ) { + continue; + } + + bool loop_head_direction = loop_heads_ending.at( loop_head ); + + auto& [ min, max ] = values; + if ( bit_index >= min && bit_index <= max ) { + dependencies_by_loading[ { loop_head, loop_head_direction } ].emplace( node_id, true ); + dependencies_by_loading[ { loop_head, loop_head_direction } ].emplace( node_id, false ); + } + } + } + + node = node->predecessor; + } +} // ------------------------------------------------------------------------------------------------ -void fuzzing::iid_node_dependence_props::compute_dependencies_by_loops( branching_node* end_node, +void fuzzing::iid_node_dependence_props::compute_dependencies_by_loops( const loop_head_to_bodies_t& loop_heads_to_bodies, const loop_endings& loop_heads_ending ) -{} +{ + for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { + if ( !loop_heads_ending.contains( loop_head ) ) { + continue; + } + + bool loop_head_end_direction = loop_heads_ending.at( loop_head ); + + for ( const auto& body : loop_bodies ) { + dependencies_by_loops[ { loop_head, loop_head_end_direction } ].emplace( body, true ); + dependencies_by_loops[ { loop_head, loop_head_end_direction } ].emplace( body, false ); + } + } +} // ------------------------------------------------------------------------------------------------ -void fuzzing::iid_dependencies::update_non_iid_nodes( const sensitivity_analysis& sensitivity ) {} +void fuzzing::iid_dependencies::update_non_iid_nodes( sensitivity_analysis& sensitivity ) +{ + for ( branching_node* node : sensitivity.get_changed_nodes() ) { + if ( node->is_did_branching() ) { + auto location_id = node->get_location_id(); + if ( non_iid_nodes.insert( location_id ).second ) { + id_to_equation_map.erase( location_id ); + } + } + } +} // ------------------------------------------------------------------------------------------------ void fuzzing::iid_dependencies::process_node_dependence( branching_node* node ) @@ -65,4 +137,7 @@ void fuzzing::iid_dependencies::process_node_dependence( branching_node* node ) if ( non_iid_nodes.contains( node->get_location_id() ) ) return; + + iid_node_dependence_props& props = id_to_equation_map[ node->get_location_id() ]; + props.process_node( node ); } From 59bc263be4c94490caba7c0dcf8a49bddd502036 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sun, 22 Dec 2024 15:20:34 +0100 Subject: [PATCH 061/144] feat: loop processing --- .../include/fuzzing/iid_vector_analysis.hpp | 30 +++- src/fuzzing/src/iid_vector_analysis.cpp | 151 +++++++++++++++--- 2 files changed, 153 insertions(+), 28 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index 95de5fae..fed83f4f 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -1,17 +1,15 @@ #pragma once +#include #include #include #include +#include #include #include +#include -struct node_direction; -using loop_ending_to_bodies = std::map< std::pair< location_id, bool >, std::set< node_direction > >; -using loop_endings = std::map< location_id, bool >; -using loop_head_to_bodies_t = std::unordered_map< location_id, std::unordered_set< location_id > >; -using loop_head_to_loaded_bits_t = std::unordered_map< location_id, std::tuple< natural_32_bit, natural_32_bit > >; namespace fuzzing { @@ -20,8 +18,19 @@ struct direction_vector {}; struct node_direction { location_id node_id; bool branching_direction; + + auto operator<=>( node_direction const& other ) const; + bool operator==( node_direction const& other ) const; + friend std::ostream& operator<<( std::ostream& os, const node_direction& nav ) + { + return os << nav.node_id.id << " " << ( nav.branching_direction ? "right" : "left" ); + } }; +using loop_ending_to_bodies = std::map< std::pair< location_id, bool >, std::set< node_direction > >; +using loop_endings = std::map< location_id, bool >; +using loop_head_to_bodies_t = std::unordered_map< location_id, std::unordered_set< location_id > >; +using loop_head_to_loaded_bits_t = std::unordered_map< location_id, std::tuple< natural_32_bit, natural_32_bit > >; struct equation { std::vector< float > values; @@ -32,9 +41,11 @@ struct equation { struct equation_matrix { equation_matrix get_submatrix( std::set< node_direction > const& subset, bool unique ) const; void add_equation( branching_node* end_node ); + bool contains( node_direction const& node ) const; private: std::vector< equation > matrix; + std::set< node_direction > nodes; }; @@ -42,10 +53,11 @@ struct iid_node_dependence_props { std::unordered_map< location_id::id_type, float > generate_probabilities(); void process_node( branching_node* end_node ); + void print_dependencies(); + private: loop_endings get_loop_heads_ending( branching_node* end_node, - const loop_head_to_bodies_t& loop_heads_to_bodies, - const std::vector< fuzzer::loop_boundary_props >& loops ) const; + loop_head_to_bodies_t& loop_heads_to_bodies ); void compute_dependencies_by_loading( branching_node* end_node, const loop_head_to_bodies_t& loop_heads_to_bodies, const loop_endings& loop_heads_ending ); @@ -60,9 +72,13 @@ struct iid_node_dependence_props { struct iid_dependencies { void update_non_iid_nodes( sensitivity_analysis& sensitivity ); void process_node_dependence( branching_node* node ); + iid_node_dependence_props& get_props( location_id id ); + std::vector< location_id > get_iid_nodes(); private: std::unordered_map< location_id, iid_node_dependence_props > id_to_equation_map; std::set< location_id > non_iid_nodes; }; + +std::vector< node_direction > get_path( branching_node* node ); } // namespace fuzzing diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index a55f75f3..57aeb4e9 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -1,26 +1,86 @@ +#include #include #include +#include +#include +#include #include +// ------------------------------------------------------------------------------------------------ +auto fuzzing::node_direction::operator<=>( node_direction const& other ) const +{ + if ( auto const cmp = node_id.id <=> other.node_id.id; cmp != 0 ) + return cmp; + + return branching_direction <=> other.branching_direction; +} + +// ------------------------------------------------------------------------------------------------ +bool fuzzing::node_direction::operator==( node_direction const& other ) const +{ + return node_id.id == other.node_id.id && branching_direction == other.branching_direction; +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::equation_matrix::add_equation( branching_node* end_node ) +{ + std::vector< node_direction > path = get_path( end_node ); + for ( const node_direction& nav : path ) { + nodes.insert( nav ); + } +} + +// ------------------------------------------------------------------------------------------------ +bool fuzzing::equation_matrix::contains( node_direction const& node ) const { return nodes.contains( node ); } + + +// ------------------------------------------------------------------------------------------------ +std::unordered_map< location_id::id_type, float > fuzzing::iid_node_dependence_props::generate_probabilities() +{ + print_dependencies(); + + return std::unordered_map< location_id::id_type, float >(); +} + // ------------------------------------------------------------------------------------------------ void fuzzing::iid_node_dependence_props::process_node( branching_node* end_node ) { loop_head_to_bodies_t loop_heads_to_bodies; - std::vector< fuzzer::loop_boundary_props > loops; - fuzzing::fuzzer::detect_loops_along_path_to_node( end_node, loop_heads_to_bodies, &loops ); - - loop_endings loop_heads_ending = get_loop_heads_ending( end_node, loop_heads_to_bodies, loops ); + loop_endings loop_heads_ending = get_loop_heads_ending( end_node, loop_heads_to_bodies ); compute_dependencies_by_loading( end_node, loop_heads_to_bodies, loop_heads_ending ); compute_dependencies_by_loops( loop_heads_to_bodies, loop_heads_ending ); + + matrix.add_equation( end_node ); } // ------------------------------------------------------------------------------------------------ -std::map< location_id, bool > fuzzing::iid_node_dependence_props::get_loop_heads_ending( - branching_node* end_node, - const loop_head_to_bodies_t& loop_heads_to_bodies, - const std::vector< fuzzer::loop_boundary_props >& loops ) const +void fuzzing::iid_node_dependence_props::print_dependencies() { + std::cout << "# Dependencies:" << std::endl; + std::cout << "## Dependencies by loops:" << std::endl; + for ( const auto& [ loop, nodes ] : dependencies_by_loops ) { + for ( const auto& body : nodes ) { + std::cout << "- " << "`(" << body << ") → " << loop.first.id << "`" << std::endl; + } + } + + std::cout << "## Dependencies by loading:" << std::endl; + for ( const auto& [ loading, nodes ] : dependencies_by_loading ) { + for ( const auto& body : nodes ) { + std::cout << "- " << "`(" << body << ") → " << loading.first.id << "`" << std::endl; + } + } +} + +// ------------------------------------------------------------------------------------------------ +std::map< location_id, bool > +fuzzing::iid_node_dependence_props::get_loop_heads_ending( branching_node* end_node, + loop_head_to_bodies_t& loop_heads_to_bodies ) +{ + std::vector< fuzzer::loop_boundary_props > loops; + fuzzing::fuzzer::detect_loops_along_path_to_node( end_node, loop_heads_to_bodies, &loops ); + std::map< location_id, bool > loop_heads_ending; auto is_outside_loop = [ & ]( branching_node* successor, @@ -79,18 +139,28 @@ void fuzzing::iid_node_dependence_props::compute_dependencies_by_loading( branch while ( node != nullptr ) { auto node_id = node->get_location_id(); - for ( const auto& bit_index : node->sensitive_stdin_bits ) { - for ( const auto& [ loop_head, values ] : loading_loops ) { - if ( !loop_heads_ending.contains( loop_head ) ) { - continue; - } + for ( const auto& [ loop_head, values ] : loading_loops ) { + if ( !loop_heads_ending.contains( loop_head ) ) { + continue; + } + + bool loop_head_direction = loop_heads_ending.at( loop_head ); + + const auto& [ min, max ] = values; + + auto it = std::find_if( node->sensitive_stdin_bits.begin(), + node->sensitive_stdin_bits.end(), + [ & ]( natural_32_bit bit_index ) { + return bit_index >= min && bit_index <= max; + } ); - bool loop_head_direction = loop_heads_ending.at( loop_head ); + if ( it != node->sensitive_stdin_bits.end() ) { + for ( bool direction : { true, false } ) { + node_direction node_id_direction = { node_id, direction }; - auto& [ min, max ] = values; - if ( bit_index >= min && bit_index <= max ) { - dependencies_by_loading[ { loop_head, loop_head_direction } ].emplace( node_id, true ); - dependencies_by_loading[ { loop_head, loop_head_direction } ].emplace( node_id, false ); + if ( matrix.contains( node_id_direction ) ) { + dependencies_by_loading[ { loop_head, loop_head_direction } ].insert( node_id_direction ); + } } } } @@ -111,14 +181,18 @@ void fuzzing::iid_node_dependence_props::compute_dependencies_by_loops( const lo bool loop_head_end_direction = loop_heads_ending.at( loop_head ); for ( const auto& body : loop_bodies ) { - dependencies_by_loops[ { loop_head, loop_head_end_direction } ].emplace( body, true ); - dependencies_by_loops[ { loop_head, loop_head_end_direction } ].emplace( body, false ); + for ( bool direction : { true, false } ) { + node_direction node_id_direction = { body, direction }; + + if ( matrix.contains( node_id_direction ) ) + dependencies_by_loops[ { loop_head, loop_head_end_direction } ].insert( node_id_direction ); + } } } } // ------------------------------------------------------------------------------------------------ -void fuzzing::iid_dependencies::update_non_iid_nodes( sensitivity_analysis& sensitivity ) +void fuzzing::iid_dependencies::update_non_iid_nodes( sensitivity_analysis& sensitivity ) { for ( branching_node* node : sensitivity.get_changed_nodes() ) { if ( node->is_did_branching() ) { @@ -141,3 +215,38 @@ void fuzzing::iid_dependencies::process_node_dependence( branching_node* node ) iid_node_dependence_props& props = id_to_equation_map[ node->get_location_id() ]; props.process_node( node ); } + +// ------------------------------------------------------------------------------------------------ +fuzzing::iid_node_dependence_props& fuzzing::iid_dependencies::get_props( location_id id ) +{ + return id_to_equation_map.at( id ); +} + +// ------------------------------------------------------------------------------------------------ +std::vector< location_id > fuzzing::iid_dependencies::get_iid_nodes() +{ + std::vector< location_id > result; + for ( const auto& [ key, _ ] : id_to_equation_map ) { + result.push_back( key ); + } + + return result; +} + +// ------------------------------------------------------------------------------------------------ +std::vector< fuzzing::node_direction > fuzzing::get_path( branching_node* node ) +{ + std::vector< node_direction > result; + + branching_node* current = node; + while ( current != nullptr ) { + branching_node* predecessor = current->predecessor; + if ( predecessor != nullptr ) { + node_direction nav = { predecessor->get_location_id(), predecessor->successor_direction( current ) }; + result.push_back( nav ); + } + current = predecessor; + } + + return result; +} \ No newline at end of file From abf0bad8aa0b231e74ce35b6ec080762fa201a53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sun, 22 Dec 2024 16:36:58 +0100 Subject: [PATCH 062/144] feat: matrix methods --- .../include/fuzzing/iid_vector_analysis.hpp | 19 +++- src/fuzzing/src/iid_vector_analysis.cpp | 100 +++++++++++++++++- 2 files changed, 114 insertions(+), 5 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index fed83f4f..a08aa200 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -33,18 +33,33 @@ using loop_head_to_bodies_t = std::unordered_map< location_id, std::unordered_se using loop_head_to_loaded_bits_t = std::unordered_map< location_id, std::tuple< natural_32_bit, natural_32_bit > >; struct equation { - std::vector< float > values; - float best_value; + equation( std::vector< int > values, double best_value ) + : values( std::move( values ) ) + , best_value( best_value ) + {} + + std::vector< int > values; + double best_value; + + bool operator==( const equation& other ) const + { + return values == other.values && best_value == other.best_value; + } }; struct equation_matrix { equation_matrix get_submatrix( std::set< node_direction > const& subset, bool unique ) const; + void process_node( branching_node* end_node ); void add_equation( branching_node* end_node ); bool contains( node_direction const& node ) const; + void print_matrix(); private: + void recompute_matrix(); + std::vector< equation > matrix; + std::vector< branching_node* > all_paths; std::set< node_direction > nodes; }; diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 57aeb4e9..84bd9bbf 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -22,22 +22,116 @@ bool fuzzing::node_direction::operator==( node_direction const& other ) const } // ------------------------------------------------------------------------------------------------ -void fuzzing::equation_matrix::add_equation( branching_node* end_node ) +fuzzing::equation_matrix fuzzing::equation_matrix::get_submatrix( std::set< node_direction > const& subset, bool unique ) const { + equation_matrix result; + result.nodes = subset; + + for ( int i = 0; i < matrix.size(); ++i ) { + const equation& row = matrix[ i ]; + + std::vector< int > new_row_values; + for ( const node_direction& nav : subset ) { + auto it = std::find( nodes.begin(), nodes.end(), nav ); + if ( it != nodes.end() ) { + new_row_values.push_back( row.values[ std::distance( nodes.begin(), it ) ] ); + } + } + + equation new_row = { new_row_values, row.best_value }; + + if ( unique ) { + if ( std::find( result.matrix.begin(), result.matrix.end(), new_row ) == result.matrix.end() ) { + result.matrix.push_back( new_row ); + result.all_paths.push_back( all_paths[ i ] ); + } + } else { + result.matrix.push_back( new_row ); + result.all_paths.push_back( all_paths[ i ] ); + } + } + + return result; +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::equation_matrix::process_node( branching_node* end_node ) +{ + all_paths.push_back( end_node ); + std::vector< node_direction > path = get_path( end_node ); + bool new_node = false; for ( const node_direction& nav : path ) { - nodes.insert( nav ); + auto [ it, inserted ] = nodes.insert( nav ); + new_node |= inserted; + } + + if ( new_node ) { + recompute_matrix(); + } else { + add_equation( end_node ); } } +// ------------------------------------------------------------------------------------------------ +void fuzzing::equation_matrix::add_equation( branching_node* end_node ) +{ + TMPROF_BLOCK(); + + std::map< node_direction, int > directions_in_path; + for ( const node_direction& navigation : nodes ) { + directions_in_path[ navigation ] = 0; + } + + std::vector< node_direction > path_nodes = get_path( end_node ); + + for ( const node_direction& nav : path_nodes ) { + if ( nodes.contains( nav ) ) { + directions_in_path[ nav ]++; + } + } + + std::vector< int > values_in_path; + for ( const auto& [ direction, count ] : directions_in_path ) { + values_in_path.push_back( count ); + } + + equation row = { values_in_path, end_node->best_coverage_value }; + matrix.push_back( row ); +} + // ------------------------------------------------------------------------------------------------ bool fuzzing::equation_matrix::contains( node_direction const& node ) const { return nodes.contains( node ); } +// ------------------------------------------------------------------------------------------------ +void fuzzing::equation_matrix::print_matrix() +{ + std::cout << "# Matrix:" << std::endl; + for ( size_t i = 0; i < matrix.size(); ++i ) { + for ( size_t j = 0; j < matrix[ i ].values.size(); ++j ) { + std::cout << ( j ? " " : "" ) << matrix[ i ].values[ j ]; + } + std::cout << " -> | " << matrix[ i ].best_value << std::endl; + } +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::equation_matrix::recompute_matrix() +{ + TMPROF_BLOCK(); + + matrix.clear(); + + for ( branching_node* path : all_paths ) { + add_equation( path ); + } +} // ------------------------------------------------------------------------------------------------ std::unordered_map< location_id::id_type, float > fuzzing::iid_node_dependence_props::generate_probabilities() { print_dependencies(); + matrix.print_matrix(); return std::unordered_map< location_id::id_type, float >(); } @@ -51,7 +145,7 @@ void fuzzing::iid_node_dependence_props::process_node( branching_node* end_node compute_dependencies_by_loading( end_node, loop_heads_to_bodies, loop_heads_ending ); compute_dependencies_by_loops( loop_heads_to_bodies, loop_heads_ending ); - matrix.add_equation( end_node ); + matrix.process_node( end_node ); } // ------------------------------------------------------------------------------------------------ From a38fa5b8911a85ed3c7b8ac3e07ecbae23f1a390 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sun, 22 Dec 2024 23:28:21 +0100 Subject: [PATCH 063/144] feat: vector computation --- .../include/fuzzing/iid_vector_analysis.hpp | 43 ++--- src/fuzzing/src/iid_vector_analysis.cpp | 157 ++++++++++++++++-- 2 files changed, 170 insertions(+), 30 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index a08aa200..ccfeb63e 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -13,14 +13,29 @@ namespace fuzzing { -struct direction_vector {}; +struct equation { + equation( std::vector< int > values, double best_value ) + : values( std::move( values ) ) + , best_value( best_value ) + {} + + std::vector< int > values; + double best_value; + + equation operator+( const equation& other ) const; + equation operator-( const equation& other ) const; + auto operator<=>( const equation& other ) const = default; + bool operator==( const equation& other ) const = default; + + bool is_any_negative() const; +}; struct node_direction { location_id node_id; bool branching_direction; auto operator<=>( node_direction const& other ) const; - bool operator==( node_direction const& other ) const; + bool operator==( node_direction const& other ) const = default; friend std::ostream& operator<<( std::ostream& os, const node_direction& nav ) { return os << nav.node_id.id << " " << ( nav.branching_direction ? "right" : "left" ); @@ -32,29 +47,16 @@ using loop_endings = std::map< location_id, bool >; using loop_head_to_bodies_t = std::unordered_map< location_id, std::unordered_set< location_id > >; using loop_head_to_loaded_bits_t = std::unordered_map< location_id, std::tuple< natural_32_bit, natural_32_bit > >; -struct equation { - equation( std::vector< int > values, double best_value ) - : values( std::move( values ) ) - , best_value( best_value ) - {} - - std::vector< int > values; - double best_value; - - bool operator==( const equation& other ) const - { - return values == other.values && best_value == other.best_value; - } -}; - - struct equation_matrix { equation_matrix get_submatrix( std::set< node_direction > const& subset, bool unique ) const; void process_node( branching_node* end_node ); void add_equation( branching_node* end_node ); bool contains( node_direction const& node ) const; + std::pair< std::size_t, std::size_t > get_dimensions() const; + std::map< equation, int > compute_vectors(); void print_matrix(); + private: void recompute_matrix(); @@ -71,8 +73,9 @@ struct iid_node_dependence_props { void print_dependencies(); private: - loop_endings get_loop_heads_ending( branching_node* end_node, - loop_head_to_bodies_t& loop_heads_to_bodies ); + equation get_best_vector( const std::map< equation, int >& vectors_with_hits ); + std::set< node_direction > get_leaf_subsets(); + loop_endings get_loop_heads_ending( branching_node* end_node, loop_head_to_bodies_t& loop_heads_to_bodies ); void compute_dependencies_by_loading( branching_node* end_node, const loop_head_to_bodies_t& loop_heads_to_bodies, const loop_endings& loop_heads_ending ); diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 84bd9bbf..2248b175 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -6,23 +6,49 @@ #include #include +// equation // ------------------------------------------------------------------------------------------------ -auto fuzzing::node_direction::operator<=>( node_direction const& other ) const +fuzzing::equation fuzzing::equation::operator+( const equation& other ) const { - if ( auto const cmp = node_id.id <=> other.node_id.id; cmp != 0 ) - return cmp; + std::vector< int > new_values; + for ( int i = 0; i < values.size(); ++i ) { + new_values.push_back( values[ i ] + other.values[ i ] ); + } - return branching_direction <=> other.branching_direction; + return { new_values, best_value + other.best_value }; } // ------------------------------------------------------------------------------------------------ -bool fuzzing::node_direction::operator==( node_direction const& other ) const +fuzzing::equation fuzzing::equation::operator-( const equation& other ) const { - return node_id.id == other.node_id.id && branching_direction == other.branching_direction; + std::vector< int > new_values; + for ( int i = 0; i < values.size(); ++i ) { + new_values.push_back( values[ i ] - other.values[ i ] ); + } + + return { new_values, best_value - other.best_value }; +} + +// ------------------------------------------------------------------------------------------------ +bool fuzzing::equation::is_any_negative() const +{ + return std::any_of( values.begin(), values.end(), []( int val ) { return val < 0; } ); +} + +// node_direction +// ------------------------------------------------------------------------------------------------ +auto fuzzing::node_direction::operator<=>( node_direction const& other ) const +{ + if ( auto const cmp = node_id.id <=> other.node_id.id; cmp != 0 ) + return cmp; + + return branching_direction <=> other.branching_direction; } +// equation_matrix // ------------------------------------------------------------------------------------------------ -fuzzing::equation_matrix fuzzing::equation_matrix::get_submatrix( std::set< node_direction > const& subset, bool unique ) const +fuzzing::equation_matrix fuzzing::equation_matrix::get_submatrix( std::set< node_direction > const& subset, + bool unique ) const { equation_matrix result; result.nodes = subset; @@ -74,7 +100,7 @@ void fuzzing::equation_matrix::process_node( branching_node* end_node ) } // ------------------------------------------------------------------------------------------------ -void fuzzing::equation_matrix::add_equation( branching_node* end_node ) +void fuzzing::equation_matrix::add_equation( branching_node* end_node ) { TMPROF_BLOCK(); @@ -103,6 +129,45 @@ void fuzzing::equation_matrix::add_equation( branching_node* end_node ) // ------------------------------------------------------------------------------------------------ bool fuzzing::equation_matrix::contains( node_direction const& node ) const { return nodes.contains( node ); } +// ------------------------------------------------------------------------------------------------ +std::pair< std::size_t, std::size_t > fuzzing::equation_matrix::get_dimensions() const +{ + return { matrix.size(), nodes.size() }; +} + +// ------------------------------------------------------------------------------------------------ +std::map< fuzzing::equation, int > fuzzing::equation_matrix::compute_vectors() +{ + std::pair< std::size_t, std::size_t > dimensions = get_dimensions(); + std::map< equation, int > vectors_with_hits; + + for ( int i = 0; i < dimensions.first; ++i ) { + for ( int j = 0; j < dimensions.first; ++j ) { + if ( i == j ) + continue; + + equation difference = matrix[ i ] - matrix[ j ]; + + if ( difference.is_any_negative() || difference.best_value == 0 ) + continue; + + vectors_with_hits[ difference ] = 0; + } + } + + for ( const auto& [ vector, hits ] : vectors_with_hits ) { + for ( const auto& row : matrix ) { + equation new_possible_equation = row + vector; + + if ( std::find( matrix.begin(), matrix.end(), new_possible_equation ) != matrix.end() ) { + vectors_with_hits[ vector ]++; + } + } + } + + return vectors_with_hits; +} + // ------------------------------------------------------------------------------------------------ void fuzzing::equation_matrix::print_matrix() { @@ -116,7 +181,7 @@ void fuzzing::equation_matrix::print_matrix() } // ------------------------------------------------------------------------------------------------ -void fuzzing::equation_matrix::recompute_matrix() +void fuzzing::equation_matrix::recompute_matrix() { TMPROF_BLOCK(); @@ -127,11 +192,32 @@ void fuzzing::equation_matrix::recompute_matrix() } } +// iid_node_dependence_props // ------------------------------------------------------------------------------------------------ std::unordered_map< location_id::id_type, float > fuzzing::iid_node_dependence_props::generate_probabilities() { print_dependencies(); - matrix.print_matrix(); + // matrix.print_matrix(); + + std::set< node_direction > all_leafs = get_leaf_subsets(); + equation_matrix submatrix = matrix.get_submatrix( all_leafs, true ); + + std::map< equation, int > vectors = submatrix.compute_vectors(); + + for ( const auto& [ vec, hits ] : vectors ) { + std::cout << "Vector: "; + for ( int i = 0; i < vec.values.size(); ++i ) { + std::cout << ( i ? ", " : "" ) << vec.values[ i ]; + } + std::cout << " -> " << vec.best_value << " (" << hits << ")" << std::endl; + } + + equation best_vector = get_best_vector( vectors ); + std::cout << "Best vector: "; + for ( int i = 0; i < best_vector.values.size(); ++i ) { + std::cout << ( i ? ", " : "" ) << best_vector.values[ i ]; + } + std::cout << " -> " << best_vector.best_value << std::endl; return std::unordered_map< location_id::id_type, float >(); } @@ -167,6 +253,55 @@ void fuzzing::iid_node_dependence_props::print_dependencies() } } +// ------------------------------------------------------------------------------------------------ +fuzzing::equation fuzzing::iid_node_dependence_props::get_best_vector( const std::map< equation, int >& vectors_with_hits ) +{ + auto max_it = std::max_element(vectors_with_hits.begin(), vectors_with_hits.end(), + [](const auto& a, const auto& b) { + return a.second < b.second; + }); + + return max_it->first; + + if ( vectors_with_hits.empty() ) { + throw std::invalid_argument( "Input map is empty." ); + } + + std::vector< equation > equations; + std::vector< double > probabilities; + + int total_hits = std::accumulate( vectors_with_hits.begin(), + vectors_with_hits.end(), + 0, + []( int sum, const auto& pair ) { return sum + pair.second; } ); + + if ( total_hits == 0 ) { + throw std::invalid_argument( "Total hits is zero." ); + } + + for ( const auto& [ eq, hits ] : vectors_with_hits ) { + equations.push_back( eq ); + probabilities.push_back( static_cast< double >( hits ) / total_hits ); + } + + std::random_device rd; + std::mt19937 gen( rd() ); + std::discrete_distribution<> dist( probabilities.begin(), probabilities.end() ); + + return equations[ dist( gen ) ]; +} + +// ------------------------------------------------------------------------------------------------ +std::set< fuzzing::node_direction > fuzzing::iid_node_dependence_props::get_leaf_subsets() +{ + std::set< node_direction > all_leafs; + for ( const auto& [ _, loop_bodies ] : dependencies_by_loops ) { + all_leafs.insert( loop_bodies.begin(), loop_bodies.end() ); + } + + return all_leafs; +} + // ------------------------------------------------------------------------------------------------ std::map< location_id, bool > fuzzing::iid_node_dependence_props::get_loop_heads_ending( branching_node* end_node, @@ -285,6 +420,7 @@ void fuzzing::iid_node_dependence_props::compute_dependencies_by_loops( const lo } } +// iid_dependencies // ------------------------------------------------------------------------------------------------ void fuzzing::iid_dependencies::update_non_iid_nodes( sensitivity_analysis& sensitivity ) { @@ -327,6 +463,7 @@ std::vector< location_id > fuzzing::iid_dependencies::get_iid_nodes() return result; } +// non member functions // ------------------------------------------------------------------------------------------------ std::vector< fuzzing::node_direction > fuzzing::get_path( branching_node* node ) { From a67f518d25df6fe4b4a2660476179edc87f29b4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Mon, 23 Dec 2024 12:31:52 +0100 Subject: [PATCH 064/144] feat: enhance equation operations and add new path computation --- .../include/fuzzing/iid_vector_analysis.hpp | 16 ++- src/fuzzing/src/iid_vector_analysis.cpp | 106 ++++++++++++++---- 2 files changed, 99 insertions(+), 23 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index ccfeb63e..0e532b3a 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -24,10 +25,20 @@ struct equation { equation operator+( const equation& other ) const; equation operator-( const equation& other ) const; + equation operator*( int scalar ) const; auto operator<=>( const equation& other ) const = default; bool operator==( const equation& other ) const = default; + int get_vector_length() const; bool is_any_negative() const; + + friend std::ostream& operator<<( std::ostream& os, const equation& eq ) + { + for ( int i = 0; i < eq.values.size(); ++i ) { + os << ( i ? " " : "" ) << eq.values[ i ]; + } + return os << " -> " << eq.best_value; + } }; struct node_direction { @@ -54,6 +65,8 @@ struct equation_matrix { bool contains( node_direction const& node ) const; std::pair< std::size_t, std::size_t > get_dimensions() const; std::map< equation, int > compute_vectors(); + std::vector< equation >& get_matrix(); + std::optional< equation > get_new_path_from_vector( const equation& vector ); void print_matrix(); @@ -73,7 +86,8 @@ struct iid_node_dependence_props { void print_dependencies(); private: - equation get_best_vector( const std::map< equation, int >& vectors_with_hits ); + equation get_best_vector( const std::map< equation, int >& vectors_with_hits, bool use_random ); + equation get_random_vector( const std::map< equation, int >& vectors_with_hits ); std::set< node_direction > get_leaf_subsets(); loop_endings get_loop_heads_ending( branching_node* end_node, loop_head_to_bodies_t& loop_heads_to_bodies ); void compute_dependencies_by_loading( branching_node* end_node, diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 2248b175..08ca2b14 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -4,12 +4,17 @@ #include #include #include +#include +#include #include + // equation // ------------------------------------------------------------------------------------------------ fuzzing::equation fuzzing::equation::operator+( const equation& other ) const { + INVARIANT( values.size() == other.values.size() ); + std::vector< int > new_values; for ( int i = 0; i < values.size(); ++i ) { new_values.push_back( values[ i ] + other.values[ i ] ); @@ -21,6 +26,8 @@ fuzzing::equation fuzzing::equation::operator+( const equation& other ) const // ------------------------------------------------------------------------------------------------ fuzzing::equation fuzzing::equation::operator-( const equation& other ) const { + INVARIANT( values.size() == other.values.size() ); + std::vector< int > new_values; for ( int i = 0; i < values.size(); ++i ) { new_values.push_back( values[ i ] - other.values[ i ] ); @@ -29,6 +36,23 @@ fuzzing::equation fuzzing::equation::operator-( const equation& other ) const return { new_values, best_value - other.best_value }; } +// ------------------------------------------------------------------------------------------------ +fuzzing::equation fuzzing::equation::operator*( int scalar ) const +{ + std::vector< int > new_values; + for ( int i = 0; i < values.size(); ++i ) { + new_values.push_back( values[ i ] * scalar ); + } + + return { new_values, best_value * scalar }; +} + +// ------------------------------------------------------------------------------------------------ +int fuzzing::equation::get_vector_length() const +{ + return std::sqrt( std::inner_product( values.begin(), values.end(), values.begin(), 0 ) ); +} + // ------------------------------------------------------------------------------------------------ bool fuzzing::equation::is_any_negative() const { @@ -168,6 +192,9 @@ std::map< fuzzing::equation, int > fuzzing::equation_matrix::compute_vectors() return vectors_with_hits; } +// ------------------------------------------------------------------------------------------------ +std::vector< fuzzing::equation >& fuzzing::equation_matrix::get_matrix() { return matrix; } + // ------------------------------------------------------------------------------------------------ void fuzzing::equation_matrix::print_matrix() { @@ -180,6 +207,40 @@ void fuzzing::equation_matrix::print_matrix() } } +// ------------------------------------------------------------------------------------------------ +std::optional< fuzzing::equation > fuzzing::equation_matrix::get_new_path_from_vector( const equation& vector ) +{ + INVARIANT( vector.values.size() == nodes.size() ); + + std::vector< equation > paths; + std::pair< std::size_t, std::size_t > dimensions = get_dimensions(); + + for ( const auto& row : matrix ) { + double counts = std::abs( row.best_value ) / vector.best_value; + + if ( std::abs( counts - std::round( counts ) ) > 1e-6 ) { + continue; + } + + equation new_path = vector * static_cast< int >( std::round( counts ) ) + row; + if ( new_path.is_any_negative() ) { + continue; + } + + paths.push_back( new_path ); + } + + if ( paths.empty() ) { + return std::nullopt; + } + + auto min_it = std::min_element( paths.begin(), paths.end(), []( const equation& a, const equation& b ) { + return a.get_vector_length() < b.get_vector_length(); + } ); + + return *min_it; +} + // ------------------------------------------------------------------------------------------------ void fuzzing::equation_matrix::recompute_matrix() { @@ -196,28 +257,21 @@ void fuzzing::equation_matrix::recompute_matrix() // ------------------------------------------------------------------------------------------------ std::unordered_map< location_id::id_type, float > fuzzing::iid_node_dependence_props::generate_probabilities() { - print_dependencies(); + // print_dependencies(); // matrix.print_matrix(); std::set< node_direction > all_leafs = get_leaf_subsets(); equation_matrix submatrix = matrix.get_submatrix( all_leafs, true ); - std::map< equation, int > vectors = submatrix.compute_vectors(); - for ( const auto& [ vec, hits ] : vectors ) { - std::cout << "Vector: "; - for ( int i = 0; i < vec.values.size(); ++i ) { - std::cout << ( i ? ", " : "" ) << vec.values[ i ]; - } - std::cout << " -> " << vec.best_value << " (" << hits << ")" << std::endl; - } + equation best_vector = get_best_vector( vectors, false ); - equation best_vector = get_best_vector( vectors ); - std::cout << "Best vector: "; - for ( int i = 0; i < best_vector.values.size(); ++i ) { - std::cout << ( i ? ", " : "" ) << best_vector.values[ i ]; + std::optional< equation > new_path = submatrix.get_new_path_from_vector( best_vector ); + + if ( !new_path.has_value() ) { + return std::unordered_map< location_id::id_type, float >(); } - std::cout << " -> " << best_vector.best_value << std::endl; + return std::unordered_map< location_id::id_type, float >(); } @@ -254,19 +308,27 @@ void fuzzing::iid_node_dependence_props::print_dependencies() } // ------------------------------------------------------------------------------------------------ -fuzzing::equation fuzzing::iid_node_dependence_props::get_best_vector( const std::map< equation, int >& vectors_with_hits ) +fuzzing::equation fuzzing::iid_node_dependence_props::get_best_vector( const std::map< equation, int >& vectors_with_hits, + bool use_random ) { - auto max_it = std::max_element(vectors_with_hits.begin(), vectors_with_hits.end(), - [](const auto& a, const auto& b) { - return a.second < b.second; - }); - - return max_it->first; - if ( vectors_with_hits.empty() ) { throw std::invalid_argument( "Input map is empty." ); } + if ( use_random ) { + return get_random_vector( vectors_with_hits ); + } + + auto max_it = std::max_element( vectors_with_hits.begin(), + vectors_with_hits.end(), + []( const auto& a, const auto& b ) { return a.second < b.second; } ); + + return max_it->first; +} + +// ------------------------------------------------------------------------------------------------ +fuzzing::equation fuzzing::iid_node_dependence_props::get_random_vector( const std::map< equation, int >& vectors_with_hits ) +{ std::vector< equation > equations; std::vector< double > probabilities; From 73c6163d91c1c58a744665105de129abb0321f8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Mon, 23 Dec 2024 14:45:56 +0100 Subject: [PATCH 065/144] feat: add node_counts structure and compute_path_counts method for path analysis --- .../include/fuzzing/iid_vector_analysis.hpp | 7 ++ src/fuzzing/src/iid_vector_analysis.cpp | 71 ++++++++++++++++++- 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index 0e532b3a..e0560f3f 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -14,6 +14,11 @@ namespace fuzzing { +struct node_counts { + int left_count; + int right_count; +}; + struct equation { equation( std::vector< int > values, double best_value ) : values( std::move( values ) ) @@ -57,6 +62,7 @@ using loop_ending_to_bodies = std::map< std::pair< location_id, bool >, std::set using loop_endings = std::map< location_id, bool >; using loop_head_to_bodies_t = std::unordered_map< location_id, std::unordered_set< location_id > >; using loop_head_to_loaded_bits_t = std::unordered_map< location_id, std::tuple< natural_32_bit, natural_32_bit > >; +using nodes_to_counts = std::map< location_id::id_type, node_counts >; struct equation_matrix { equation_matrix get_submatrix( std::set< node_direction > const& subset, bool unique ) const; @@ -86,6 +92,7 @@ struct iid_node_dependence_props { void print_dependencies(); private: + nodes_to_counts compute_path_counts( const equation& path, std::set< node_direction > const& all_leafs ); equation get_best_vector( const std::map< equation, int >& vectors_with_hits, bool use_random ); equation get_random_vector( const std::map< equation, int >& vectors_with_hits ); std::set< node_direction > get_leaf_subsets(); diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 08ca2b14..014c2f96 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -272,8 +272,22 @@ std::unordered_map< location_id::id_type, float > fuzzing::iid_node_dependence_p return std::unordered_map< location_id::id_type, float >(); } + nodes_to_counts path_counts = compute_path_counts( new_path.value(), all_leafs ); - return std::unordered_map< location_id::id_type, float >(); + { + std::cout << "# Path counts:" << std::endl; + for ( const auto& [ node_id, counts ] : path_counts ) { + std::cout << "- " << node_id << " → " << counts.left_count << ", " << counts.right_count << std::endl; + } + } + + std::unordered_map< location_id::id_type, float_32_bit > probabilities; + for ( const auto& [ id, counts ] : path_counts ) { + float_32_bit total_count = counts.left_count + counts.right_count; + probabilities[ id ] = float_32_bit(counts.left_count) / total_count; + } + + return probabilities; } // ------------------------------------------------------------------------------------------------ @@ -307,6 +321,61 @@ void fuzzing::iid_node_dependence_props::print_dependencies() } } +// ------------------------------------------------------------------------------------------------ +fuzzing::nodes_to_counts +fuzzing::iid_node_dependence_props::compute_path_counts( const equation& path, + std::set< node_direction > const& all_leafs ) +{ + nodes_to_counts path_counts; + + std::vector< node_direction > leafs = std::vector< node_direction >( all_leafs.begin(), all_leafs.end() ); + + INVARIANT( leafs.size() == path.values.size() ); + + for ( int i = 0; i < leafs.size(); ++i ) { + auto& [ left_count, right_count ] = path_counts[ leafs[ i ].node_id.id ]; + if ( leafs[ i ].branching_direction ) { + right_count = path.values[ i ]; + } else { + left_count = path.values[ i ]; + } + } + + for ( const auto& [ loop_head, dependent_bodies ] : dependencies_by_loops ) { + int loop_count = 0; + for ( const auto& body : dependent_bodies ) { + auto& [ left_count, right_count ] = path_counts[ body.node_id.id ]; + loop_count = std::max( loop_count, left_count + right_count ); + } + + if ( loop_head.second ) { + path_counts[ loop_head.first.id ] = { loop_count, 1 }; + } else { + path_counts[ loop_head.first.id ] = { 1, loop_count }; + } + } + + for ( const auto& [ loading_head, loading_bodies ] : dependencies_by_loading ) { + int loop_count = 0; + for ( const auto& body : loading_bodies ) { + if ( !path_counts.contains( body.node_id.id ) ) { + continue; + } + + auto& [ left_count, right_count ] = path_counts[ body.node_id.id ]; + loop_count = std::max( loop_count, left_count + right_count ); + } + + if ( loading_head.second ) { + path_counts[ loading_head.first.id ] = { loop_count, 1 }; + } else { + path_counts[ loading_head.first.id ] = { 1, loop_count }; + } + } + + return path_counts; +} + // ------------------------------------------------------------------------------------------------ fuzzing::equation fuzzing::iid_node_dependence_props::get_best_vector( const std::map< equation, int >& vectors_with_hits, bool use_random ) From 4cb1d5c3d498ea4c43b9a23a69514c16a0bec7e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Mon, 23 Dec 2024 21:07:53 +0100 Subject: [PATCH 066/144] feat: improve path count handling --- src/fuzzing/src/iid_vector_analysis.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 014c2f96..f892168d 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -199,6 +199,10 @@ std::vector< fuzzing::equation >& fuzzing::equation_matrix::get_matrix() { retur void fuzzing::equation_matrix::print_matrix() { std::cout << "# Matrix:" << std::endl; + for ( const node_direction& nav : nodes ) { + std::cout << nav << " "; + } + std::cout << std::endl; for ( size_t i = 0; i < matrix.size(); ++i ) { for ( size_t j = 0; j < matrix[ i ].values.size(); ++j ) { std::cout << ( j ? " " : "" ) << matrix[ i ].values[ j ]; @@ -261,8 +265,12 @@ std::unordered_map< location_id::id_type, float > fuzzing::iid_node_dependence_p // matrix.print_matrix(); std::set< node_direction > all_leafs = get_leaf_subsets(); + equation_matrix submatrix = matrix.get_submatrix( all_leafs, true ); std::map< equation, int > vectors = submatrix.compute_vectors(); + if ( vectors.empty() ) { + return std::unordered_map< location_id::id_type, float >(); + } equation best_vector = get_best_vector( vectors, false ); @@ -274,13 +282,6 @@ std::unordered_map< location_id::id_type, float > fuzzing::iid_node_dependence_p nodes_to_counts path_counts = compute_path_counts( new_path.value(), all_leafs ); - { - std::cout << "# Path counts:" << std::endl; - for ( const auto& [ node_id, counts ] : path_counts ) { - std::cout << "- " << node_id << " → " << counts.left_count << ", " << counts.right_count << std::endl; - } - } - std::unordered_map< location_id::id_type, float_32_bit > probabilities; for ( const auto& [ id, counts ] : path_counts ) { float_32_bit total_count = counts.left_count + counts.right_count; @@ -296,10 +297,11 @@ void fuzzing::iid_node_dependence_props::process_node( branching_node* end_node loop_head_to_bodies_t loop_heads_to_bodies; loop_endings loop_heads_ending = get_loop_heads_ending( end_node, loop_heads_to_bodies ); + matrix.process_node( end_node ); + compute_dependencies_by_loading( end_node, loop_heads_to_bodies, loop_heads_ending ); compute_dependencies_by_loops( loop_heads_to_bodies, loop_heads_ending ); - matrix.process_node( end_node ); } // ------------------------------------------------------------------------------------------------ From c395c30c7ca57c0f735cbfcca14f1a4338232fe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Fri, 27 Dec 2024 14:21:29 +0100 Subject: [PATCH 067/144] feat: remove probabilities with both zeros --- src/fuzzing/src/iid_vector_analysis.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index f892168d..e60bf19d 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -273,6 +273,13 @@ std::unordered_map< location_id::id_type, float > fuzzing::iid_node_dependence_p } equation best_vector = get_best_vector( vectors, false ); + { + // std::cout << "Best vector:" << std::endl; + // for ( int i = 0; i < best_vector.values.size(); ++i ) { + // std::cout << best_vector.values[ i ] << " "; + // } + // std::cout << " -> | " << best_vector.best_value << std::endl; + } std::optional< equation > new_path = submatrix.get_new_path_from_vector( best_vector ); @@ -281,9 +288,19 @@ std::unordered_map< location_id::id_type, float > fuzzing::iid_node_dependence_p } nodes_to_counts path_counts = compute_path_counts( new_path.value(), all_leafs ); + { + // std::cout << "Path counts:" << std::endl; + // for ( const auto& [ id, counts ] : path_counts ) { + // std::cout << "ID: " << id << " Left: " << counts.left_count << " Right: " << counts.right_count << std::endl; + // } + } std::unordered_map< location_id::id_type, float_32_bit > probabilities; for ( const auto& [ id, counts ] : path_counts ) { + if ( counts.left_count == 0 && counts.right_count == 0 ) { + continue; + } + float_32_bit total_count = counts.left_count + counts.right_count; probabilities[ id ] = float_32_bit(counts.left_count) / total_count; } From c604118b7f9de99bb0fe68d076ba05b56367b123 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Fri, 27 Dec 2024 14:22:39 +0100 Subject: [PATCH 068/144] feat: add new IID testing benchmarks and update existing benchmark configurations --- benchmarks/benman.py | 4 +- benchmarks/iid_testing/combination_needed.c | 37 ++++++++++++ .../iid_testing/combination_needed.json | 43 ++++++++++++++ .../iid_testing/decrement_in_condition.c | 40 +++++++++++++ .../iid_testing/decrement_in_condition.json | 45 ++++++++++++++ benchmarks/iid_testing/decrement_in_else.c | 40 +++++++++++++ benchmarks/iid_testing/decrement_in_else.json | 41 +++++++++++++ benchmarks/iid_testing/decrement_on_b.c | 37 ++++++++++++ benchmarks/iid_testing/decrement_on_b.json | 43 ++++++++++++++ benchmarks/iid_testing/double_loop.c | 48 +++++++++++++++ benchmarks/iid_testing/double_loop.json | 49 +++++++++++++++ benchmarks/iid_testing/iid.c | 35 +++++++++++ benchmarks/iid_testing/iid.json | 41 +++++++++++++ benchmarks/iid_testing/iid_multiple.c | 53 +++++++++++++++++ benchmarks/iid_testing/iid_multiple.json | 59 +++++++++++++++++++ benchmarks/iid_testing/iid_twice.c | 48 +++++++++++++++ benchmarks/iid_testing/iid_twice.json | 47 +++++++++++++++ benchmarks/iid_testing/more_iid_condition.c | 42 +++++++++++++ .../iid_testing/more_iid_condition.json | 45 ++++++++++++++ .../iid_testing/more_iid_condition_nested.c | 42 +++++++++++++ .../more_iid_condition_nested.json | 45 ++++++++++++++ .../iid_testing/multiplication_in_iid.c | 35 +++++++++++ .../iid_testing/multiplication_in_iid.json | 41 +++++++++++++ benchmarks/iid_testing/same_loop_dependent.c | 49 +++++++++++++++ .../iid_testing/same_loop_dependent.json | 49 +++++++++++++++ .../iid_testing/same_loop_dependent_2.c | 49 +++++++++++++++ .../iid_testing/same_loop_dependent_2.json | 49 +++++++++++++++ .../iid_testing/same_loop_independent.c | 50 ++++++++++++++++ .../iid_testing/same_loop_independent.json | 49 +++++++++++++++ 29 files changed, 1253 insertions(+), 2 deletions(-) create mode 100644 benchmarks/iid_testing/combination_needed.c create mode 100644 benchmarks/iid_testing/combination_needed.json create mode 100644 benchmarks/iid_testing/decrement_in_condition.c create mode 100644 benchmarks/iid_testing/decrement_in_condition.json create mode 100644 benchmarks/iid_testing/decrement_in_else.c create mode 100644 benchmarks/iid_testing/decrement_in_else.json create mode 100644 benchmarks/iid_testing/decrement_on_b.c create mode 100644 benchmarks/iid_testing/decrement_on_b.json create mode 100644 benchmarks/iid_testing/double_loop.c create mode 100644 benchmarks/iid_testing/double_loop.json create mode 100644 benchmarks/iid_testing/iid.c create mode 100644 benchmarks/iid_testing/iid.json create mode 100644 benchmarks/iid_testing/iid_multiple.c create mode 100644 benchmarks/iid_testing/iid_multiple.json create mode 100644 benchmarks/iid_testing/iid_twice.c create mode 100644 benchmarks/iid_testing/iid_twice.json create mode 100644 benchmarks/iid_testing/more_iid_condition.c create mode 100644 benchmarks/iid_testing/more_iid_condition.json create mode 100644 benchmarks/iid_testing/more_iid_condition_nested.c create mode 100644 benchmarks/iid_testing/more_iid_condition_nested.json create mode 100644 benchmarks/iid_testing/multiplication_in_iid.c create mode 100644 benchmarks/iid_testing/multiplication_in_iid.json create mode 100644 benchmarks/iid_testing/same_loop_dependent.c create mode 100644 benchmarks/iid_testing/same_loop_dependent.json create mode 100644 benchmarks/iid_testing/same_loop_dependent_2.c create mode 100644 benchmarks/iid_testing/same_loop_dependent_2.json create mode 100644 benchmarks/iid_testing/same_loop_independent.c create mode 100644 benchmarks/iid_testing/same_loop_independent.json diff --git a/benchmarks/benman.py b/benchmarks/benman.py index 3da6a36b..a78f08df 100755 --- a/benchmarks/benman.py +++ b/benchmarks/benman.py @@ -163,7 +163,7 @@ def _ok_stats_message(self, config : dict, outcomes : dict) -> str: except Exception as e: return "Unknown executions count" percentage = 100.0 * num_execution / max_num_execution - return "#" + ("%.2f" % (percentage - 100)) + "%" if max_num_execution >= 100 and percentage < 90 else "" + return "#" + ("%.2f" % (percentage - 100)) + "%" if max_num_execution >= 100 else "" def _fail_stats_message(self, config : dict, outcomes : dict) -> str: expected_termination_type = config["results"]["termination_type"] @@ -328,7 +328,7 @@ def search_for_benchmarks(folder : str) -> list: pass return benchmarks - kinds = ["fast", "medium", "slow", "pending"] + kinds = ["fast", "iid_testing", "medium", "slow", "pending"] benchmarks = [] if name == "all": for kind in kinds: diff --git a/benchmarks/iid_testing/combination_needed.c b/benchmarks/iid_testing/combination_needed.c new file mode 100644 index 00000000..d63fb0b8 --- /dev/null +++ b/benchmarks/iid_testing/combination_needed.c @@ -0,0 +1,37 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + k += 2; + if ( s[ i ] == 'B' ) // ID: 7 + --k; + ++i; + } + if ( k == 5 ) // ID: 8 + return 1; + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/combination_needed.json b/benchmarks/iid_testing/combination_needed.json new file mode 100644 index 00000000..543832a6 --- /dev/null +++ b/benchmarks/iid_testing/combination_needed.json @@ -0,0 +1,43 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 10000, + "num_covered_branchings": 8, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0 + ], + "num_generated_tests": 8, + "num_crashes": 0, + "num_boundary_violations": 0 + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/decrement_in_condition.c b/benchmarks/iid_testing/decrement_in_condition.c new file mode 100644 index 00000000..982004eb --- /dev/null +++ b/benchmarks/iid_testing/decrement_in_condition.c @@ -0,0 +1,40 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + if ( s[ i ] % 2 == 0 ) // ID: 7 + { + if ( s[ i ] == 'B' ) // ID: 8 + --k; + } + ++i; + } + if ( k == 10 ) // ID: 9 + return 1; + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/decrement_in_condition.json b/benchmarks/iid_testing/decrement_in_condition.json new file mode 100644 index 00000000..28b98ee6 --- /dev/null +++ b/benchmarks/iid_testing/decrement_in_condition.json @@ -0,0 +1,45 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 7110, + "num_covered_branchings": 9, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0 + ], + "num_generated_tests": 9, + "num_crashes": 0, + "num_boundary_violations": 0 + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/decrement_in_else.c b/benchmarks/iid_testing/decrement_in_else.c new file mode 100644 index 00000000..d1a570b8 --- /dev/null +++ b/benchmarks/iid_testing/decrement_in_else.c @@ -0,0 +1,40 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + { + ++k; + } else { + --k; + } + + ++i; + } + if ( k == 5 ) // ID: 7 + return 1; + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/decrement_in_else.json b/benchmarks/iid_testing/decrement_in_else.json new file mode 100644 index 00000000..bce701d9 --- /dev/null +++ b/benchmarks/iid_testing/decrement_in_else.json @@ -0,0 +1,41 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 1868, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "num_generated_tests": 7, + "num_crashes": 0, + "num_boundary_violations": 0 + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/decrement_on_b.c b/benchmarks/iid_testing/decrement_on_b.c new file mode 100644 index 00000000..4fe1d4a0 --- /dev/null +++ b/benchmarks/iid_testing/decrement_on_b.c @@ -0,0 +1,37 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + if ( s[ i ] == 'B' ) // ID: 7 + --k; + ++i; + } + if ( k == 10 ) // ID: 8 + return 1; + return 0; + } +} diff --git a/benchmarks/iid_testing/decrement_on_b.json b/benchmarks/iid_testing/decrement_on_b.json new file mode 100644 index 00000000..fdd0c48e --- /dev/null +++ b/benchmarks/iid_testing/decrement_on_b.json @@ -0,0 +1,43 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 7000, + "num_covered_branchings": 8, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0 + ], + "num_generated_tests": 8, + "num_crashes": 0, + "num_boundary_violations": 0 + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/double_loop.c b/benchmarks/iid_testing/double_loop.c new file mode 100644 index 00000000..e98f1023 --- /dev/null +++ b/benchmarks/iid_testing/double_loop.c @@ -0,0 +1,48 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + if ( s[ i ] == 'B' ) // ID: 7 + --k; + ++i; + } + i = 0; + int l = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 8 + break; + if ( s[ i ] == 'X' ) // ID: 9 + ++l; + if ( s[ i ] == 'Y' ) // ID: 10 + --l; + ++i; + } + if ( k == 10 ) // ID: 11 + return 1; + return 0; + } +} diff --git a/benchmarks/iid_testing/double_loop.json b/benchmarks/iid_testing/double_loop.json new file mode 100644 index 00000000..c4755714 --- /dev/null +++ b/benchmarks/iid_testing/double_loop.json @@ -0,0 +1,49 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 9000, + "num_covered_branchings": 11, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0, + 11, + 0 + ], + "num_generated_tests": 10, + "num_crashes": 0, + "num_boundary_violations": 0 + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/iid.c b/benchmarks/iid_testing/iid.c new file mode 100644 index 00000000..aa117a8c --- /dev/null +++ b/benchmarks/iid_testing/iid.c @@ -0,0 +1,35 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + ++i; + } + if ( k == 10 ) // ID: 7 + return 1; + return 0; + } +} diff --git a/benchmarks/iid_testing/iid.json b/benchmarks/iid_testing/iid.json new file mode 100644 index 00000000..8e3e2423 --- /dev/null +++ b/benchmarks/iid_testing/iid.json @@ -0,0 +1,41 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 6044, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "num_generated_tests": 7, + "num_crashes": 0, + "num_boundary_violations": 0 + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/iid_multiple.c b/benchmarks/iid_testing/iid_multiple.c new file mode 100644 index 00000000..bcca4ed8 --- /dev/null +++ b/benchmarks/iid_testing/iid_multiple.c @@ -0,0 +1,53 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + if ( s[ i ] == 'B' ) // ID: 7 + --k; + if ( s[ i ] == 'C' ) // ID: 8 + ++k; + if ( s[ i ] == 'D' ) // ID: 9 + --k; + if ( s[ i ] == 'E' ) // ID: 10 + ++k; + if ( s[ i ] == 'F' ) // ID: 11 + --k; + if ( s[ i ] == 'G' ) // ID: 12 + ++k; + if ( s[ i ] == 'H' ) // ID: 13 + --k; + if ( s[ i ] == 'I' ) // ID: 14 + ++k; + if ( s[ i ] == 'J' ) // ID: 15 + --k; + ++i; + } + if ( k == 10 ) // ID: 16 + return 1; + return 0; + } +} diff --git a/benchmarks/iid_testing/iid_multiple.json b/benchmarks/iid_testing/iid_multiple.json new file mode 100644 index 00000000..e3b1afba --- /dev/null +++ b/benchmarks/iid_testing/iid_multiple.json @@ -0,0 +1,59 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 6480, + "num_covered_branchings": 16, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0, + 11, + 0, + 12, + 0, + 13, + 0, + 14, + 0, + 15, + 0, + 16, + 0 + ], + "num_generated_tests": 16, + "num_crashes": 0, + "num_boundary_violations": 0 + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/iid_twice.c b/benchmarks/iid_testing/iid_twice.c new file mode 100644 index 00000000..3eecf0dc --- /dev/null +++ b/benchmarks/iid_testing/iid_twice.c @@ -0,0 +1,48 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + ++i; + } + if ( k == 10 ) // ID: 7 + return 1; + + i = 0; + int l = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 8 + break; + if ( s[ i ] == 'B' ) // ID: 9 + ++l; + ++i; + } + if ( l == 10 ) // ID: 10 + return 1; + + return 0; + } +} diff --git a/benchmarks/iid_testing/iid_twice.json b/benchmarks/iid_testing/iid_twice.json new file mode 100644 index 00000000..c8ab3391 --- /dev/null +++ b/benchmarks/iid_testing/iid_twice.json @@ -0,0 +1,47 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 1000, + "num_covered_branchings": 10, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0 + ], + "num_generated_tests": 10, + "num_crashes": 0, + "num_boundary_violations": 0 + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/more_iid_condition.c b/benchmarks/iid_testing/more_iid_condition.c new file mode 100644 index 00000000..b8e15dc7 --- /dev/null +++ b/benchmarks/iid_testing/more_iid_condition.c @@ -0,0 +1,42 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + ++i; + } + if ( k == 10 ) // ID: 7 + return 1; + + if ( k == 5 ) // ID: 8 + return 1; + + if ( k % 2 == 0 ) // ID: 9 + return 1; + + return 0; + } +} diff --git a/benchmarks/iid_testing/more_iid_condition.json b/benchmarks/iid_testing/more_iid_condition.json new file mode 100644 index 00000000..6e9de3a8 --- /dev/null +++ b/benchmarks/iid_testing/more_iid_condition.json @@ -0,0 +1,45 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 10000, + "num_covered_branchings": 9, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0 + ], + "num_generated_tests": 9, + "num_crashes": 0, + "num_boundary_violations": 0 + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/more_iid_condition_nested.c b/benchmarks/iid_testing/more_iid_condition_nested.c new file mode 100644 index 00000000..2d0dcd1b --- /dev/null +++ b/benchmarks/iid_testing/more_iid_condition_nested.c @@ -0,0 +1,42 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + ++i; + } + + if ( k == 10 ) // ID: 7 + { + if ( k > 13 ) // ID: 8 + { + if ( k % 2 == 0 ) // ID: 9 + return 1; + } + } + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/more_iid_condition_nested.json b/benchmarks/iid_testing/more_iid_condition_nested.json new file mode 100644 index 00000000..6e9de3a8 --- /dev/null +++ b/benchmarks/iid_testing/more_iid_condition_nested.json @@ -0,0 +1,45 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 10000, + "num_covered_branchings": 9, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0 + ], + "num_generated_tests": 9, + "num_crashes": 0, + "num_boundary_violations": 0 + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/multiplication_in_iid.c b/benchmarks/iid_testing/multiplication_in_iid.c new file mode 100644 index 00000000..2f9fdccf --- /dev/null +++ b/benchmarks/iid_testing/multiplication_in_iid.c @@ -0,0 +1,35 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + ++i; + } + if ( 4 * k == 20 ) // ID: 7 + return 1; + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/multiplication_in_iid.json b/benchmarks/iid_testing/multiplication_in_iid.json new file mode 100644 index 00000000..b2952a07 --- /dev/null +++ b/benchmarks/iid_testing/multiplication_in_iid.json @@ -0,0 +1,41 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 10000, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "num_generated_tests": 7, + "num_crashes": 0, + "num_boundary_violations": 0 + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/same_loop_dependent.c b/benchmarks/iid_testing/same_loop_dependent.c new file mode 100644 index 00000000..1aeb4025 --- /dev/null +++ b/benchmarks/iid_testing/same_loop_dependent.c @@ -0,0 +1,49 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + if ( s[ i ] == 'B' ) // ID: 7 + --k; + ++i; + } + + i = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 8 + break; + if ( s[ i ] == 'A' ) // ID: 9 + ++k; + if ( s[ i ] == 'B' ) // ID: 10 + --k; + ++i; + } + + if ( k == 10 ) // ID: 11 + return 1; + return 0; + } +} diff --git a/benchmarks/iid_testing/same_loop_dependent.json b/benchmarks/iid_testing/same_loop_dependent.json new file mode 100644 index 00000000..67c22e54 --- /dev/null +++ b/benchmarks/iid_testing/same_loop_dependent.json @@ -0,0 +1,49 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 10000, + "num_covered_branchings": 11, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0, + 11, + 0 + ], + "num_generated_tests": 8, + "num_crashes": 0, + "num_boundary_violations": 0 + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/same_loop_dependent_2.c b/benchmarks/iid_testing/same_loop_dependent_2.c new file mode 100644 index 00000000..8ec81152 --- /dev/null +++ b/benchmarks/iid_testing/same_loop_dependent_2.c @@ -0,0 +1,49 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + if ( s[ i ] == 'B' ) // ID: 7 + --k; + ++i; + } + + i = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 8 + break; + if ( s[ i ] == 'C' ) // ID: 9 + ++k; + if ( s[ i ] == 'D' ) // ID: 10 + --k; + ++i; + } + + if ( k == 10 ) // ID: 11 + return 1; + return 0; + } +} diff --git a/benchmarks/iid_testing/same_loop_dependent_2.json b/benchmarks/iid_testing/same_loop_dependent_2.json new file mode 100644 index 00000000..67c22e54 --- /dev/null +++ b/benchmarks/iid_testing/same_loop_dependent_2.json @@ -0,0 +1,49 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 10000, + "num_covered_branchings": 11, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0, + 11, + 0 + ], + "num_generated_tests": 8, + "num_crashes": 0, + "num_boundary_violations": 0 + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/same_loop_independent.c b/benchmarks/iid_testing/same_loop_independent.c new file mode 100644 index 00000000..33c40aff --- /dev/null +++ b/benchmarks/iid_testing/same_loop_independent.c @@ -0,0 +1,50 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + if ( s[ i ] == 'B' ) // ID: 7 + --k; + ++i; + } + + i = 0; + int l = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 8 + break; + if ( s[ i ] == 'X' ) // ID: 9 + ++l; + if ( s[ i ] == 'Y' ) // ID: 10 + --l; + ++i; + } + + if ( k == 10 ) // ID: 11 + return 1; + return 0; + } +} diff --git a/benchmarks/iid_testing/same_loop_independent.json b/benchmarks/iid_testing/same_loop_independent.json new file mode 100644 index 00000000..879fc4b2 --- /dev/null +++ b/benchmarks/iid_testing/same_loop_independent.json @@ -0,0 +1,49 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 10000, + "num_covered_branchings": 11, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0, + 11, + 0 + ], + "num_generated_tests": 10, + "num_crashes": 0, + "num_boundary_violations": 0 + } +} \ No newline at end of file From 3fd36aa2c763e9351a943f6d24f86b42d6710094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Wed, 8 Jan 2025 19:13:23 +0100 Subject: [PATCH 069/144] feat: add new IID testing benchmarks and update existing condition checks --- benchmarks/iid_testing/input_cycle.c | 47 +++++++++++++++++++ benchmarks/iid_testing/input_cycle.json | 45 ++++++++++++++++++ benchmarks/iid_testing/more_iid_condition.c | 6 +-- .../iid_testing/more_iid_condition_nested.c | 7 +-- 4 files changed, 99 insertions(+), 6 deletions(-) create mode 100644 benchmarks/iid_testing/input_cycle.c create mode 100644 benchmarks/iid_testing/input_cycle.json diff --git a/benchmarks/iid_testing/input_cycle.c b/benchmarks/iid_testing/input_cycle.c new file mode 100644 index 00000000..3763aa5a --- /dev/null +++ b/benchmarks/iid_testing/input_cycle.c @@ -0,0 +1,47 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + short loop_count; + loop_count = __VERIFIER_nondet_short(); + + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + if ( loop_count > 20 ) // ID: 5 + return -1; + + int i = 0, k = 0; + + for ( short index = 0; index < loop_count; ++index ) { // ID: 6 + i = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 7 + break; + if ( s[ i ] == 'A' ) // ID: 8 + ++k; + ++i; + } + } + + if ( k == 10 ) // ID: 9 + return 1; + + return 0; + } +} diff --git a/benchmarks/iid_testing/input_cycle.json b/benchmarks/iid_testing/input_cycle.json new file mode 100644 index 00000000..6e9de3a8 --- /dev/null +++ b/benchmarks/iid_testing/input_cycle.json @@ -0,0 +1,45 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 10000, + "num_covered_branchings": 9, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0 + ], + "num_generated_tests": 9, + "num_crashes": 0, + "num_boundary_violations": 0 + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/more_iid_condition.c b/benchmarks/iid_testing/more_iid_condition.c index b8e15dc7..2fe177aa 100644 --- a/benchmarks/iid_testing/more_iid_condition.c +++ b/benchmarks/iid_testing/more_iid_condition.c @@ -28,13 +28,13 @@ int main() ++k; ++i; } - if ( k == 10 ) // ID: 7 + if ( k == 8 ) // ID: 7 return 1; - if ( k == 5 ) // ID: 8 + if ( k == 10 ) // ID: 8 return 1; - if ( k % 2 == 0 ) // ID: 9 + if ( k == 11 ) // ID: 9 return 1; return 0; diff --git a/benchmarks/iid_testing/more_iid_condition_nested.c b/benchmarks/iid_testing/more_iid_condition_nested.c index 2d0dcd1b..cc420c35 100644 --- a/benchmarks/iid_testing/more_iid_condition_nested.c +++ b/benchmarks/iid_testing/more_iid_condition_nested.c @@ -29,11 +29,12 @@ int main() ++i; } - if ( k == 10 ) // ID: 7 + // if ( k % 2 == 0 ) // ID: 7 + if ( k >= 6 ) // ID: 7 { - if ( k > 13 ) // ID: 8 + if ( k > 8 ) // ID: 8 { - if ( k % 2 == 0 ) // ID: 9 + if ( k == 11 ) // ID: 9 return 1; } } From ce547c08eb9c2316961e25f9cd8fd49f06d2dca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Wed, 8 Jan 2025 19:39:17 +0100 Subject: [PATCH 070/144] feat: add new benchmark for input cycle and update execution counts --- benchmarks/iid_testing/input_cycle.json | 2 +- benchmarks/iid_testing/input_cycle_dec_b.c | 48 +++++++++++++++++++ benchmarks/iid_testing/input_cycle_dec_b.json | 47 ++++++++++++++++++ 3 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 benchmarks/iid_testing/input_cycle_dec_b.c create mode 100644 benchmarks/iid_testing/input_cycle_dec_b.json diff --git a/benchmarks/iid_testing/input_cycle.json b/benchmarks/iid_testing/input_cycle.json index 6e9de3a8..11ccc79e 100644 --- a/benchmarks/iid_testing/input_cycle.json +++ b/benchmarks/iid_testing/input_cycle.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 10000, + "num_executions": 1556, "num_covered_branchings": 9, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/input_cycle_dec_b.c b/benchmarks/iid_testing/input_cycle_dec_b.c new file mode 100644 index 00000000..1ca3bd91 --- /dev/null +++ b/benchmarks/iid_testing/input_cycle_dec_b.c @@ -0,0 +1,48 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + short loop_count; + loop_count = __VERIFIER_nondet_short(); + if ( loop_count > 50 ) // ID: 5 + return -1; + + int i = 0, k = 0; + + for ( short index = 0; index < loop_count; ++index ) { // ID: 6 + i = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 7 + break; + if ( s[ i ] == 'A' ) // ID: 8 + ++k; + if ( s[ i ] == 'B' ) // ID: 9 + --k; + ++i; + } + } + + if ( k == 10 ) // ID: 10 + return 1; + + return 0; + } +} diff --git a/benchmarks/iid_testing/input_cycle_dec_b.json b/benchmarks/iid_testing/input_cycle_dec_b.json new file mode 100644 index 00000000..a1511e0a --- /dev/null +++ b/benchmarks/iid_testing/input_cycle_dec_b.json @@ -0,0 +1,47 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 1923, + "num_covered_branchings": 10, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0 + ], + "num_generated_tests": 10, + "num_crashes": 0, + "num_boundary_violations": 0 + } +} \ No newline at end of file From 67151e94b2a6dac746091e58c8f57efd1ee7ae9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Wed, 8 Jan 2025 22:00:16 +0100 Subject: [PATCH 071/144] feat: update loop thresholds and execution counts in IID testing benchmarks --- benchmarks/iid_testing/input_cycle_dec_b.c | 2 +- benchmarks/iid_testing/input_cycle_dec_b.json | 2 +- benchmarks/iid_testing/same_loop_dependent_2.c | 2 +- benchmarks/iid_testing/same_loop_dependent_2.json | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/benchmarks/iid_testing/input_cycle_dec_b.c b/benchmarks/iid_testing/input_cycle_dec_b.c index 1ca3bd91..ef0f1e72 100644 --- a/benchmarks/iid_testing/input_cycle_dec_b.c +++ b/benchmarks/iid_testing/input_cycle_dec_b.c @@ -22,7 +22,7 @@ int main() { short loop_count; loop_count = __VERIFIER_nondet_short(); - if ( loop_count > 50 ) // ID: 5 + if ( loop_count > 20 ) // ID: 5 return -1; int i = 0, k = 0; diff --git a/benchmarks/iid_testing/input_cycle_dec_b.json b/benchmarks/iid_testing/input_cycle_dec_b.json index a1511e0a..edaeb82f 100644 --- a/benchmarks/iid_testing/input_cycle_dec_b.json +++ b/benchmarks/iid_testing/input_cycle_dec_b.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 1923, + "num_executions": 2007, "num_covered_branchings": 10, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/same_loop_dependent_2.c b/benchmarks/iid_testing/same_loop_dependent_2.c index 8ec81152..4bd33d7f 100644 --- a/benchmarks/iid_testing/same_loop_dependent_2.c +++ b/benchmarks/iid_testing/same_loop_dependent_2.c @@ -42,7 +42,7 @@ int main() ++i; } - if ( k == 10 ) // ID: 11 + if ( k == 7 ) // ID: 11 return 1; return 0; } diff --git a/benchmarks/iid_testing/same_loop_dependent_2.json b/benchmarks/iid_testing/same_loop_dependent_2.json index 67c22e54..5efb5bdd 100644 --- a/benchmarks/iid_testing/same_loop_dependent_2.json +++ b/benchmarks/iid_testing/same_loop_dependent_2.json @@ -1,6 +1,6 @@ { "args": { - "max_executions": 10000, + "max_executions": 15000, "max_seconds": 300, "max_trace_length": 10000, "max_stack_size": 25, @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 10000, + "num_executions": 7739, "num_covered_branchings": 11, "covered_branchings": [ 1, @@ -42,7 +42,7 @@ 11, 0 ], - "num_generated_tests": 8, + "num_generated_tests": 10, "num_crashes": 0, "num_boundary_violations": 0 } From bd00c7f09a1616a761c109650980a10201f1c1cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Fri, 10 Jan 2025 15:19:30 +0100 Subject: [PATCH 072/144] feat: update execution counts and add original execution values in IID testing benchmarks --- benchmarks/iid_testing/combination_needed.json | 3 ++- benchmarks/iid_testing/decrement_in_condition.json | 3 ++- benchmarks/iid_testing/decrement_in_else.json | 1 + benchmarks/iid_testing/decrement_on_b.json | 3 ++- benchmarks/iid_testing/double_loop.json | 3 ++- benchmarks/iid_testing/iid.json | 3 ++- benchmarks/iid_testing/iid_multiple.json | 3 ++- benchmarks/iid_testing/iid_twice.json | 3 ++- benchmarks/iid_testing/input_cycle.json | 1 + benchmarks/iid_testing/input_cycle_dec_b.json | 1 + benchmarks/iid_testing/more_iid_condition.json | 3 ++- benchmarks/iid_testing/multiplication_in_iid.json | 3 ++- benchmarks/iid_testing/same_loop_dependent.json | 3 ++- benchmarks/iid_testing/same_loop_independent.json | 3 ++- 14 files changed, 25 insertions(+), 11 deletions(-) diff --git a/benchmarks/iid_testing/combination_needed.json b/benchmarks/iid_testing/combination_needed.json index 543832a6..f35f0e9d 100644 --- a/benchmarks/iid_testing/combination_needed.json +++ b/benchmarks/iid_testing/combination_needed.json @@ -13,10 +13,11 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, + "num_executions_original": 10000, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 10000, + "num_executions": 9100, "num_covered_branchings": 8, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/decrement_in_condition.json b/benchmarks/iid_testing/decrement_in_condition.json index 28b98ee6..9866f6c3 100644 --- a/benchmarks/iid_testing/decrement_in_condition.json +++ b/benchmarks/iid_testing/decrement_in_condition.json @@ -13,10 +13,11 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, + "num_executions_original": 7110, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 7110, + "num_executions": 3981, "num_covered_branchings": 9, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/decrement_in_else.json b/benchmarks/iid_testing/decrement_in_else.json index bce701d9..1bc13e3e 100644 --- a/benchmarks/iid_testing/decrement_in_else.json +++ b/benchmarks/iid_testing/decrement_in_else.json @@ -13,6 +13,7 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, + "num_executions_original": 1868, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", diff --git a/benchmarks/iid_testing/decrement_on_b.json b/benchmarks/iid_testing/decrement_on_b.json index fdd0c48e..e5ce7d8b 100644 --- a/benchmarks/iid_testing/decrement_on_b.json +++ b/benchmarks/iid_testing/decrement_on_b.json @@ -13,10 +13,11 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, + "num_executions_original": 7000, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 7000, + "num_executions": 3710, "num_covered_branchings": 8, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/double_loop.json b/benchmarks/iid_testing/double_loop.json index c4755714..2dc5d8a4 100644 --- a/benchmarks/iid_testing/double_loop.json +++ b/benchmarks/iid_testing/double_loop.json @@ -13,10 +13,11 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, + "num_executions_original": 9000, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 9000, + "num_executions": 3600, "num_covered_branchings": 11, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/iid.json b/benchmarks/iid_testing/iid.json index 8e3e2423..eb0cc95d 100644 --- a/benchmarks/iid_testing/iid.json +++ b/benchmarks/iid_testing/iid.json @@ -13,10 +13,11 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, + "num_executions_original": 6044, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 6044, + "num_executions": 3807, "num_covered_branchings": 7, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/iid_multiple.json b/benchmarks/iid_testing/iid_multiple.json index e3b1afba..8cc8ddfe 100644 --- a/benchmarks/iid_testing/iid_multiple.json +++ b/benchmarks/iid_testing/iid_multiple.json @@ -13,10 +13,11 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, + "num_executions_original": 6480, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 6480, + "num_executions": 3693, "num_covered_branchings": 16, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/iid_twice.json b/benchmarks/iid_testing/iid_twice.json index c8ab3391..a5afd7c5 100644 --- a/benchmarks/iid_testing/iid_twice.json +++ b/benchmarks/iid_testing/iid_twice.json @@ -13,10 +13,11 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, + "num_executions_original": 1000, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 1000, + "num_executions": 10000, "num_covered_branchings": 10, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/input_cycle.json b/benchmarks/iid_testing/input_cycle.json index 11ccc79e..313fc712 100644 --- a/benchmarks/iid_testing/input_cycle.json +++ b/benchmarks/iid_testing/input_cycle.json @@ -13,6 +13,7 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, + "num_executions_original": 2007, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", diff --git a/benchmarks/iid_testing/input_cycle_dec_b.json b/benchmarks/iid_testing/input_cycle_dec_b.json index edaeb82f..454ff514 100644 --- a/benchmarks/iid_testing/input_cycle_dec_b.json +++ b/benchmarks/iid_testing/input_cycle_dec_b.json @@ -13,6 +13,7 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, + "num_executions_original": 2007, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", diff --git a/benchmarks/iid_testing/more_iid_condition.json b/benchmarks/iid_testing/more_iid_condition.json index 6e9de3a8..79c37bc3 100644 --- a/benchmarks/iid_testing/more_iid_condition.json +++ b/benchmarks/iid_testing/more_iid_condition.json @@ -13,10 +13,11 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, + "num_executions_original": 10000, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 10000, + "num_executions": 4000, "num_covered_branchings": 9, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/multiplication_in_iid.json b/benchmarks/iid_testing/multiplication_in_iid.json index b2952a07..8302a11b 100644 --- a/benchmarks/iid_testing/multiplication_in_iid.json +++ b/benchmarks/iid_testing/multiplication_in_iid.json @@ -13,10 +13,11 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, + "num_executions_original": 10000, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 10000, + "num_executions": 2000, "num_covered_branchings": 7, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/same_loop_dependent.json b/benchmarks/iid_testing/same_loop_dependent.json index 67c22e54..82000af4 100644 --- a/benchmarks/iid_testing/same_loop_dependent.json +++ b/benchmarks/iid_testing/same_loop_dependent.json @@ -13,10 +13,11 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, + "num_executions_original": 10000, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 10000, + "num_executions": 1900, "num_covered_branchings": 11, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/same_loop_independent.json b/benchmarks/iid_testing/same_loop_independent.json index 879fc4b2..8b93505a 100644 --- a/benchmarks/iid_testing/same_loop_independent.json +++ b/benchmarks/iid_testing/same_loop_independent.json @@ -13,10 +13,11 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, + "num_executions_original": 10000, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 10000, + "num_executions": 3600, "num_covered_branchings": 11, "covered_branchings": [ 1, From 9337689f38ee2a19beb7557d312bc35c364ea8aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Fri, 10 Jan 2025 19:47:27 +0100 Subject: [PATCH 073/144] feat: add new IID testing benchmarks for non-IID conditions --- .../iid_testing/lots_of_non_iid_conditions.c | 55 ++++++++++++++++++ .../lots_of_non_iid_conditions.json | 56 +++++++++++++++++++ 2 files changed, 111 insertions(+) create mode 100644 benchmarks/iid_testing/lots_of_non_iid_conditions.c create mode 100644 benchmarks/iid_testing/lots_of_non_iid_conditions.json diff --git a/benchmarks/iid_testing/lots_of_non_iid_conditions.c b/benchmarks/iid_testing/lots_of_non_iid_conditions.c new file mode 100644 index 00000000..093a68a0 --- /dev/null +++ b/benchmarks/iid_testing/lots_of_non_iid_conditions.c @@ -0,0 +1,55 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + if ( s[ 0 ] == 'A' ) // ID: 5 + return 1; + + if ( s[ 1 ] == 'A' ) // ID: 6 + return 1; + + if ( s[ 2 ] == 'A' ) // ID: 7 + return 1; + + if ( s[ 3 ] == 'A' ) // ID: 8 + return 1; + + if ( s[ 4 ] == 'A' ) // ID: 9 + return 1; + + if ( s[ 5 ] == 'A' ) // ID: 10 + return 1; + + if ( s[ 6 ] == 'A' ) // ID: 11 + return 1; + + if ( s[ 7 ] == 'A' ) // ID: 12 + return 1; + + if ( s[ 8 ] == 'A' ) // ID: 13 + return 1; + + if ( s[ 9 ] == 'A' ) // ID: 14 + return 1; + + return 0; + } +} diff --git a/benchmarks/iid_testing/lots_of_non_iid_conditions.json b/benchmarks/iid_testing/lots_of_non_iid_conditions.json new file mode 100644 index 00000000..2ec248ec --- /dev/null +++ b/benchmarks/iid_testing/lots_of_non_iid_conditions.json @@ -0,0 +1,56 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "num_executions_original": 410, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 410, + "num_covered_branchings": 14, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0, + 11, + 0, + 12, + 0, + 13, + 0, + 14, + 0 + ], + "num_generated_tests": 14, + "num_crashes": 0, + "num_boundary_violations": 0 + } +} \ No newline at end of file From 7775c1213a3c2ceb147ec2ddce0a8aa813bd5fc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Fri, 10 Jan 2025 20:17:28 +0100 Subject: [PATCH 074/144] feat: add new IID testing benchmark for condition checks with execution results --- benchmarks/iid_testing/iid_in_condition.c | 43 ++++++++++++++++++ benchmarks/iid_testing/iid_in_condition.json | 46 ++++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 benchmarks/iid_testing/iid_in_condition.c create mode 100644 benchmarks/iid_testing/iid_in_condition.json diff --git a/benchmarks/iid_testing/iid_in_condition.c b/benchmarks/iid_testing/iid_in_condition.c new file mode 100644 index 00000000..991c55a2 --- /dev/null +++ b/benchmarks/iid_testing/iid_in_condition.c @@ -0,0 +1,43 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + + if ( s[ 0 ] > 20 ) // ID: 5 + { + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 6 + break; + if ( s[ i ] == 'A' ) // ID: 7 + ++k; + if ( s[ i ] == 'B' ) // ID: 8 + --k; + ++i; + } + } + + if ( k == 10 ) // ID: 9 + return 1; + + return 0; + } +} diff --git a/benchmarks/iid_testing/iid_in_condition.json b/benchmarks/iid_testing/iid_in_condition.json new file mode 100644 index 00000000..7cb98f87 --- /dev/null +++ b/benchmarks/iid_testing/iid_in_condition.json @@ -0,0 +1,46 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "num_executions_original": 6810, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 3850, + "num_covered_branchings": 9, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0 + ], + "num_generated_tests": 8, + "num_crashes": 0, + "num_boundary_violations": 0 + } +} \ No newline at end of file From b46197e58068d83b68536332e964c98aa4ea7fba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Fri, 10 Jan 2025 23:34:09 +0100 Subject: [PATCH 075/144] feat: implement path_node_props and possible_path structures with direction handling --- .../include/fuzzing/iid_vector_analysis.hpp | 48 +++++- src/fuzzing/src/iid_vector_analysis.cpp | 149 ++++++++++++++++-- 2 files changed, 184 insertions(+), 13 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index e0560f3f..a63c0a22 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -19,6 +19,48 @@ struct node_counts { int right_count; }; + +struct path_node_props { + path_node_props( node_counts computed_counts, bool is_loop_head, bool loop_head_direction ) + : computed_counts( computed_counts ) + , taken_counts( { 0, 0 } ) + , is_loop_head( is_loop_head ) + , loop_head_direction( loop_head_direction ) + {} + + bool get_desired_direction() const; + bool can_go_direction( bool direction ) const; + void go_direction( bool direction ); + bool can_take_next_direction() const; + + float_32_bit get_false_direction_probability() const; + +private: + node_counts computed_counts; + node_counts taken_counts; + bool is_loop_head; + bool loop_head_direction; + + bool get_preferred_direction_loop_head() const; +}; + + +struct possible_path { + possible_path( std::map< location_id::id_type, path_node_props > path ) + : path( std::move( path ) ) + {} + + possible_path() = default; + + bool contains( location_id::id_type id ) const; + std::map< location_id::id_type, path_node_props > get_path() const; + path_node_props& get_props( location_id::id_type id ) { return path.at( id ); } + +private: + std::map< location_id::id_type, path_node_props > path; +}; + + struct equation { equation( std::vector< int > values, double best_value ) : values( std::move( values ) ) @@ -58,7 +100,8 @@ struct node_direction { } }; -using loop_ending_to_bodies = std::map< std::pair< location_id, bool >, std::set< node_direction > >; +using loop_head_end_direction = bool; +using loop_ending_to_bodies = std::map< std::pair< location_id, loop_head_end_direction >, std::set< node_direction > >; using loop_endings = std::map< location_id, bool >; using loop_head_to_bodies_t = std::unordered_map< location_id, std::unordered_set< location_id > >; using loop_head_to_loaded_bits_t = std::unordered_map< location_id, std::tuple< natural_32_bit, natural_32_bit > >; @@ -86,7 +129,7 @@ struct equation_matrix { struct iid_node_dependence_props { - std::unordered_map< location_id::id_type, float > generate_probabilities(); + possible_path generate_probabilities(); void process_node( branching_node* end_node ); void print_dependencies(); @@ -111,6 +154,7 @@ struct iid_node_dependence_props { struct iid_dependencies { void update_non_iid_nodes( sensitivity_analysis& sensitivity ); void process_node_dependence( branching_node* node ); + void remove_node_dependence( location_id id ); iid_node_dependence_props& get_props( location_id id ); std::vector< location_id > get_iid_nodes(); diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index e60bf19d..f945eb77 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -8,6 +8,98 @@ #include #include +// path_node_props +// ------------------------------------------------------------------------------------------------ +bool fuzzing::path_node_props::get_desired_direction() const +{ + INVARIANT( computed_counts.left_count + computed_counts.right_count > 0 ); + + if ( is_loop_head ) { + return get_preferred_direction_loop_head(); + } + + if ( taken_counts.left_count < computed_counts.left_count ) { + return false; + } + + if ( taken_counts.right_count < computed_counts.right_count ) { + return true; + } + + // TODO: Should this be here? + float total_count = computed_counts.left_count + computed_counts.right_count; + float left_probability = static_cast< float >( computed_counts.left_count ) / total_count; + float random_value = static_cast< float >( rand() ) / RAND_MAX; + + if ( random_value < left_probability ) { + return false; + } else { + return true; + } +} + +// ------------------------------------------------------------------------------------------------ +bool fuzzing::path_node_props::can_go_direction( bool direction ) const +{ + if ( direction ) { + return taken_counts.right_count < computed_counts.right_count; + } else { + return taken_counts.left_count < computed_counts.left_count; + } +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::path_node_props::go_direction( bool direction ) +{ + if ( direction ) { + taken_counts.right_count; + } else { + taken_counts.left_count; + } + + // TODO: Maybe taken count should be reset after the loop is finished, in case this loop is visited again + // later in the code. +} + +// ------------------------------------------------------------------------------------------------ +bool fuzzing::path_node_props::can_take_next_direction() const +{ + return taken_counts.left_count < computed_counts.left_count || + taken_counts.right_count < computed_counts.right_count; +} + +// ------------------------------------------------------------------------------------------------ +float_32_bit fuzzing::path_node_props::get_false_direction_probability() const +{ + INVARIANT( computed_counts.left_count + computed_counts.right_count > 0 ); + + return float_32_bit( computed_counts.left_count ) / + ( computed_counts.left_count + computed_counts.right_count ); +} + +// ------------------------------------------------------------------------------------------------ +bool fuzzing::path_node_props::get_preferred_direction_loop_head() const +{ + auto is_depleted = []( int computed, int taken ) { return computed == taken; }; + + if ( !loop_head_direction ) { + INVARIANT( computed_counts.left_count == 1 ); + return !is_depleted( computed_counts.right_count, taken_counts.right_count ); + } else { + INVARIANT( computed_counts.right_count == 1 ); + return is_depleted( computed_counts.left_count, taken_counts.left_count ); + } +} + +// possible_path +// ------------------------------------------------------------------------------------------------ +bool fuzzing::possible_path::contains( location_id::id_type id ) const { return path.contains( id ); } + +// ------------------------------------------------------------------------------------------------ +std::map< location_id::id_type, fuzzing::path_node_props > fuzzing::possible_path::get_path() const +{ + return path; +} // equation // ------------------------------------------------------------------------------------------------ @@ -259,17 +351,16 @@ void fuzzing::equation_matrix::recompute_matrix() // iid_node_dependence_props // ------------------------------------------------------------------------------------------------ -std::unordered_map< location_id::id_type, float > fuzzing::iid_node_dependence_props::generate_probabilities() +fuzzing::possible_path fuzzing::iid_node_dependence_props::generate_probabilities() { // print_dependencies(); // matrix.print_matrix(); std::set< node_direction > all_leafs = get_leaf_subsets(); - equation_matrix submatrix = matrix.get_submatrix( all_leafs, true ); std::map< equation, int > vectors = submatrix.compute_vectors(); if ( vectors.empty() ) { - return std::unordered_map< location_id::id_type, float >(); + return {}; } equation best_vector = get_best_vector( vectors, false ); @@ -284,28 +375,54 @@ std::unordered_map< location_id::id_type, float > fuzzing::iid_node_dependence_p std::optional< equation > new_path = submatrix.get_new_path_from_vector( best_vector ); if ( !new_path.has_value() ) { - return std::unordered_map< location_id::id_type, float >(); + return {}; } nodes_to_counts path_counts = compute_path_counts( new_path.value(), all_leafs ); { // std::cout << "Path counts:" << std::endl; // for ( const auto& [ id, counts ] : path_counts ) { - // std::cout << "ID: " << id << " Left: " << counts.left_count << " Right: " << counts.right_count << std::endl; + // std::cout << "ID: " << id << " Left: " << counts.left_count << " Right: " << counts.right_count + // << std::endl; // } } - std::unordered_map< location_id::id_type, float_32_bit > probabilities; + std::map< location_id::id_type, path_node_props > path; for ( const auto& [ id, counts ] : path_counts ) { if ( counts.left_count == 0 && counts.right_count == 0 ) { continue; } - - float_32_bit total_count = counts.left_count + counts.right_count; - probabilities[ id ] = float_32_bit(counts.left_count) / total_count; + + bool loop_head_direction = false; + bool is_loop_head = false; + + for ( bool direction : { false, true } ) { + for ( const auto& map : { dependencies_by_loops, dependencies_by_loading } ) { + if ( map.contains( { id, direction } ) ) { + is_loop_head = true; + loop_head_direction = direction; + break; + } + } + } + + path[ id ] = { counts, is_loop_head, loop_head_direction }; } - return probabilities; + possible_path best_path( path ); + return best_path; + + // std::unordered_map< location_id::id_type, float_32_bit > probabilities; + // for ( const auto& [ id, counts ] : path_counts ) { + // if ( counts.left_count == 0 && counts.right_count == 0 ) { + // continue; + // } + + // float_32_bit total_count = counts.left_count + counts.right_count; + // probabilities[ id ] = float_32_bit( counts.left_count ) / total_count; + // } + + // return probabilities; } // ------------------------------------------------------------------------------------------------ @@ -318,7 +435,6 @@ void fuzzing::iid_node_dependence_props::process_node( branching_node* end_node compute_dependencies_by_loading( end_node, loop_heads_to_bodies, loop_heads_ending ); compute_dependencies_by_loops( loop_heads_to_bodies, loop_heads_ending ); - } // ------------------------------------------------------------------------------------------------ @@ -374,6 +490,7 @@ fuzzing::iid_node_dependence_props::compute_path_counts( const equation& path, } } + for ( const auto& [ loading_head, loading_bodies ] : dependencies_by_loading ) { int loop_count = 0; for ( const auto& body : loading_bodies ) { @@ -596,6 +713,15 @@ void fuzzing::iid_dependencies::process_node_dependence( branching_node* node ) props.process_node( node ); } +// ------------------------------------------------------------------------------------------------ +void fuzzing::iid_dependencies::remove_node_dependence( location_id id ) +{ + if ( id_to_equation_map.contains( id ) ) { + id_to_equation_map.erase( id ); + non_iid_nodes.insert( id ); + } +} + // ------------------------------------------------------------------------------------------------ fuzzing::iid_node_dependence_props& fuzzing::iid_dependencies::get_props( location_id id ) { @@ -610,6 +736,7 @@ std::vector< location_id > fuzzing::iid_dependencies::get_iid_nodes() result.push_back( key ); } + std::sort( result.begin(), result.end() ); return result; } From c26cc77964e7e75772638669f04fdc7d6fa67152 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 11 Jan 2025 16:22:54 +0100 Subject: [PATCH 076/144] feat: add ostream operators for path_node_props and possible_path structures --- .../include/fuzzing/iid_vector_analysis.hpp | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index a63c0a22..3a81b624 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -35,6 +35,16 @@ struct path_node_props { float_32_bit get_false_direction_probability() const; + friend std::ostream& operator<<( std::ostream& os, const path_node_props& eq ) + { + os << "L: (" << eq.computed_counts.left_count << " | " << eq.taken_counts.left_count << ") "; + os << "R: (" << eq.computed_counts.right_count << " | " << eq.taken_counts.right_count << ") "; + if ( eq.is_loop_head ) { + os << "Loop head: " << ( eq.loop_head_direction ? "R" : "L" ); + } + return os; + } + private: node_counts computed_counts; node_counts taken_counts; @@ -56,6 +66,16 @@ struct possible_path { std::map< location_id::id_type, path_node_props > get_path() const; path_node_props& get_props( location_id::id_type id ) { return path.at( id ); } + friend std::ostream& operator<<( std::ostream& os, const possible_path& eq ) + { + for ( const auto& [ id, props ] : eq.path ) { + os << id << ":" << std::endl; + os << props << std::endl; + } + + return os; + } + private: std::map< location_id::id_type, path_node_props > path; }; @@ -101,11 +121,12 @@ struct node_direction { }; using loop_head_end_direction = bool; -using loop_ending_to_bodies = std::map< std::pair< location_id, loop_head_end_direction >, std::set< node_direction > >; +using loop_ending_to_bodies = + std::map< std::pair< location_id, loop_head_end_direction >, std::set< node_direction > >; using loop_endings = std::map< location_id, bool >; using loop_head_to_bodies_t = std::unordered_map< location_id, std::unordered_set< location_id > >; using loop_head_to_loaded_bits_t = std::unordered_map< location_id, std::tuple< natural_32_bit, natural_32_bit > >; -using nodes_to_counts = std::map< location_id::id_type, node_counts >; +using nodes_to_counts = std::map< location_id, node_counts >; struct equation_matrix { equation_matrix get_submatrix( std::set< node_direction > const& subset, bool unique ) const; @@ -116,6 +137,7 @@ struct equation_matrix { std::map< equation, int > compute_vectors(); std::vector< equation >& get_matrix(); std::optional< equation > get_new_path_from_vector( const equation& vector ); + int get_desired_vector_direction() const; void print_matrix(); @@ -136,7 +158,9 @@ struct iid_node_dependence_props { private: nodes_to_counts compute_path_counts( const equation& path, std::set< node_direction > const& all_leafs ); - equation get_best_vector( const std::map< equation, int >& vectors_with_hits, bool use_random ); + equation get_best_vector( const std::map< equation, int >& vectors_with_hits, + bool use_random, + int desired_direction ); equation get_random_vector( const std::map< equation, int >& vectors_with_hits ); std::set< node_direction > get_leaf_subsets(); loop_endings get_loop_heads_ending( branching_node* end_node, loop_head_to_bodies_t& loop_heads_to_bodies ); @@ -145,6 +169,7 @@ struct iid_node_dependence_props { const loop_endings& loop_heads_ending ); void compute_dependencies_by_loops( const loop_head_to_bodies_t& loop_heads_to_bodies, const loop_endings& loop_heads_ending ); + possible_path generate_path_from_node_counts( const nodes_to_counts& path_counts ); equation_matrix matrix; loop_ending_to_bodies dependencies_by_loops; From 81510da37b0017960d10a4778b9fb330b4e381aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 11 Jan 2025 16:23:04 +0100 Subject: [PATCH 077/144] feat: enhance direction handling in path_node_props and equation_matrix, add print_matrix method --- src/fuzzing/src/iid_vector_analysis.cpp | 178 +++++++++++++----------- 1 file changed, 98 insertions(+), 80 deletions(-) diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index f945eb77..4d1584e1 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -29,7 +29,7 @@ bool fuzzing::path_node_props::get_desired_direction() const // TODO: Should this be here? float total_count = computed_counts.left_count + computed_counts.right_count; float left_probability = static_cast< float >( computed_counts.left_count ) / total_count; - float random_value = static_cast< float >( rand() ) / RAND_MAX; + float random_value = static_cast< float >( rand() ) / static_cast< float >( RAND_MAX ); if ( random_value < left_probability ) { return false; @@ -39,8 +39,8 @@ bool fuzzing::path_node_props::get_desired_direction() const } // ------------------------------------------------------------------------------------------------ -bool fuzzing::path_node_props::can_go_direction( bool direction ) const -{ +bool fuzzing::path_node_props::can_go_direction( bool direction ) const +{ if ( direction ) { return taken_counts.right_count < computed_counts.right_count; } else { @@ -52,9 +52,9 @@ bool fuzzing::path_node_props::can_go_direction( bool direction ) const void fuzzing::path_node_props::go_direction( bool direction ) { if ( direction ) { - taken_counts.right_count; + taken_counts.right_count++; } else { - taken_counts.left_count; + taken_counts.left_count++; } // TODO: Maybe taken count should be reset after the loop is finished, in case this loop is visited again @@ -288,18 +288,17 @@ std::map< fuzzing::equation, int > fuzzing::equation_matrix::compute_vectors() std::vector< fuzzing::equation >& fuzzing::equation_matrix::get_matrix() { return matrix; } // ------------------------------------------------------------------------------------------------ -void fuzzing::equation_matrix::print_matrix() +int fuzzing::equation_matrix::get_desired_vector_direction() const { - std::cout << "# Matrix:" << std::endl; - for ( const node_direction& nav : nodes ) { - std::cout << nav << " "; - } - std::cout << std::endl; - for ( size_t i = 0; i < matrix.size(); ++i ) { - for ( size_t j = 0; j < matrix[ i ].values.size(); ++j ) { - std::cout << ( j ? " " : "" ) << matrix[ i ].values[ j ]; - } - std::cout << " -> | " << matrix[ i ].best_value << std::endl; + auto is_positive = []( const equation& eq ) { return eq.best_value > 0; }; + auto is_negative = []( const equation& eq ) { return eq.best_value < 0; }; + + if ( std::all_of( matrix.begin(), matrix.end(), is_positive ) ) { + return -1; + } else if ( std::all_of( matrix.begin(), matrix.end(), is_negative ) ) { + return 1; + } else { + return 0; } } @@ -337,6 +336,22 @@ std::optional< fuzzing::equation > fuzzing::equation_matrix::get_new_path_from_v return *min_it; } +// ------------------------------------------------------------------------------------------------ +void fuzzing::equation_matrix::print_matrix() +{ + std::cout << "# Matrix:" << std::endl; + for ( const node_direction& nav : nodes ) { + std::cout << nav << " "; + } + std::cout << std::endl; + for ( size_t i = 0; i < matrix.size(); ++i ) { + for ( size_t j = 0; j < matrix[ i ].values.size(); ++j ) { + std::cout << ( j ? " " : "" ) << matrix[ i ].values[ j ]; + } + std::cout << " -> | " << matrix[ i ].best_value << std::endl; + } +} + // ------------------------------------------------------------------------------------------------ void fuzzing::equation_matrix::recompute_matrix() { @@ -363,66 +378,18 @@ fuzzing::possible_path fuzzing::iid_node_dependence_props::generate_probabilitie return {}; } - equation best_vector = get_best_vector( vectors, false ); - { - // std::cout << "Best vector:" << std::endl; - // for ( int i = 0; i < best_vector.values.size(); ++i ) { - // std::cout << best_vector.values[ i ] << " "; - // } - // std::cout << " -> | " << best_vector.best_value << std::endl; - } + int desired_vector_direction = submatrix.get_desired_vector_direction(); + equation best_vector = get_best_vector( vectors, false, desired_vector_direction ); std::optional< equation > new_path = submatrix.get_new_path_from_vector( best_vector ); - if ( !new_path.has_value() ) { return {}; } nodes_to_counts path_counts = compute_path_counts( new_path.value(), all_leafs ); - { - // std::cout << "Path counts:" << std::endl; - // for ( const auto& [ id, counts ] : path_counts ) { - // std::cout << "ID: " << id << " Left: " << counts.left_count << " Right: " << counts.right_count - // << std::endl; - // } - } + possible_path path = generate_path_from_node_counts( path_counts ); - std::map< location_id::id_type, path_node_props > path; - for ( const auto& [ id, counts ] : path_counts ) { - if ( counts.left_count == 0 && counts.right_count == 0 ) { - continue; - } - - bool loop_head_direction = false; - bool is_loop_head = false; - - for ( bool direction : { false, true } ) { - for ( const auto& map : { dependencies_by_loops, dependencies_by_loading } ) { - if ( map.contains( { id, direction } ) ) { - is_loop_head = true; - loop_head_direction = direction; - break; - } - } - } - - path[ id ] = { counts, is_loop_head, loop_head_direction }; - } - - possible_path best_path( path ); - return best_path; - - // std::unordered_map< location_id::id_type, float_32_bit > probabilities; - // for ( const auto& [ id, counts ] : path_counts ) { - // if ( counts.left_count == 0 && counts.right_count == 0 ) { - // continue; - // } - - // float_32_bit total_count = counts.left_count + counts.right_count; - // probabilities[ id ] = float_32_bit( counts.left_count ) / total_count; - // } - - // return probabilities; + return path; } // ------------------------------------------------------------------------------------------------ @@ -456,6 +423,7 @@ void fuzzing::iid_node_dependence_props::print_dependencies() } } + // ------------------------------------------------------------------------------------------------ fuzzing::nodes_to_counts fuzzing::iid_node_dependence_props::compute_path_counts( const equation& path, @@ -468,7 +436,7 @@ fuzzing::iid_node_dependence_props::compute_path_counts( const equation& path, INVARIANT( leafs.size() == path.values.size() ); for ( int i = 0; i < leafs.size(); ++i ) { - auto& [ left_count, right_count ] = path_counts[ leafs[ i ].node_id.id ]; + auto& [ left_count, right_count ] = path_counts[ leafs[ i ].node_id ]; if ( leafs[ i ].branching_direction ) { right_count = path.values[ i ]; } else { @@ -479,14 +447,14 @@ fuzzing::iid_node_dependence_props::compute_path_counts( const equation& path, for ( const auto& [ loop_head, dependent_bodies ] : dependencies_by_loops ) { int loop_count = 0; for ( const auto& body : dependent_bodies ) { - auto& [ left_count, right_count ] = path_counts[ body.node_id.id ]; + auto& [ left_count, right_count ] = path_counts[ body.node_id ]; loop_count = std::max( loop_count, left_count + right_count ); } if ( loop_head.second ) { - path_counts[ loop_head.first.id ] = { loop_count, 1 }; + path_counts[ loop_head.first ] = { loop_count, 1 }; } else { - path_counts[ loop_head.first.id ] = { 1, loop_count }; + path_counts[ loop_head.first ] = { 1, loop_count }; } } @@ -494,18 +462,18 @@ fuzzing::iid_node_dependence_props::compute_path_counts( const equation& path, for ( const auto& [ loading_head, loading_bodies ] : dependencies_by_loading ) { int loop_count = 0; for ( const auto& body : loading_bodies ) { - if ( !path_counts.contains( body.node_id.id ) ) { + if ( !path_counts.contains( body.node_id ) ) { continue; } - auto& [ left_count, right_count ] = path_counts[ body.node_id.id ]; + auto& [ left_count, right_count ] = path_counts[ body.node_id ]; loop_count = std::max( loop_count, left_count + right_count ); } if ( loading_head.second ) { - path_counts[ loading_head.first.id ] = { loop_count, 1 }; + path_counts[ loading_head.first ] = { loop_count, 1 }; } else { - path_counts[ loading_head.first.id ] = { 1, loop_count }; + path_counts[ loading_head.first ] = { 1, loop_count }; } } @@ -514,18 +482,38 @@ fuzzing::iid_node_dependence_props::compute_path_counts( const equation& path, // ------------------------------------------------------------------------------------------------ fuzzing::equation fuzzing::iid_node_dependence_props::get_best_vector( const std::map< equation, int >& vectors_with_hits, - bool use_random ) + bool use_random, + int desired_direction ) { if ( vectors_with_hits.empty() ) { throw std::invalid_argument( "Input map is empty." ); } + std::map< equation, int > filtered_vectors_with_hits; + if ( desired_direction < 0 ) { + std::copy_if( vectors_with_hits.begin(), + vectors_with_hits.end(), + std::inserter( filtered_vectors_with_hits, filtered_vectors_with_hits.end() ), + []( const auto& pair ) { return pair.first.best_value < 0; } ); + } else if ( desired_direction > 0 ) { + std::copy_if( vectors_with_hits.begin(), + vectors_with_hits.end(), + std::inserter( filtered_vectors_with_hits, filtered_vectors_with_hits.end() ), + []( const auto& pair ) { return pair.first.best_value > 0; } ); + } else { + filtered_vectors_with_hits = vectors_with_hits; + } + + if ( filtered_vectors_with_hits.empty() ) { + throw std::invalid_argument( "No vectors match the desired direction." ); + } + if ( use_random ) { - return get_random_vector( vectors_with_hits ); + return get_random_vector( filtered_vectors_with_hits ); } - auto max_it = std::max_element( vectors_with_hits.begin(), - vectors_with_hits.end(), + auto max_it = std::max_element( filtered_vectors_with_hits.begin(), + filtered_vectors_with_hits.end(), []( const auto& a, const auto& b ) { return a.second < b.second; } ); return max_it->first; @@ -687,6 +675,36 @@ void fuzzing::iid_node_dependence_props::compute_dependencies_by_loops( const lo } } +// ------------------------------------------------------------------------------------------------ +fuzzing::possible_path +fuzzing::iid_node_dependence_props::generate_path_from_node_counts( const nodes_to_counts& path_counts ) +{ + std::map< location_id::id_type, path_node_props > path; + for ( const auto& [ id, counts ] : path_counts ) { + if ( counts.left_count == 0 && counts.right_count == 0 ) { + continue; + } + + bool loop_head_direction = false; + bool is_loop_head = false; + + for ( bool direction : { false, true } ) { + for ( const auto& map : { dependencies_by_loops, dependencies_by_loading } ) { + if ( map.contains( std::make_pair( id, direction ) ) ) { + is_loop_head = true; + loop_head_direction = direction; + break; + } + } + } + + path_node_props props = { counts, is_loop_head, loop_head_direction }; + path.emplace( id.id, props ); + } + + return possible_path( path ); +} + // iid_dependencies // ------------------------------------------------------------------------------------------------ void fuzzing::iid_dependencies::update_non_iid_nodes( sensitivity_analysis& sensitivity ) From ed2b79637cd1f3187d0069e01c884b24149ec680 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 11 Jan 2025 20:31:43 +0100 Subject: [PATCH 078/144] feat: enhance iid_node_dependence_props with const correctness and add get_dependencies_without_bodies_in_loop method --- .../include/fuzzing/iid_vector_analysis.hpp | 3 +- src/fuzzing/src/iid_vector_analysis.cpp | 109 ++++++++++++++++-- 2 files changed, 99 insertions(+), 13 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index 3a81b624..30d3639e 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -154,9 +154,10 @@ struct iid_node_dependence_props { possible_path generate_probabilities(); void process_node( branching_node* end_node ); - void print_dependencies(); + void print_dependencies() const; private: + loop_ending_to_bodies get_dependencies_without_bodies_in_loop() const; nodes_to_counts compute_path_counts( const equation& path, std::set< node_direction > const& all_leafs ); equation get_best_vector( const std::map< equation, int >& vectors_with_hits, bool use_random, diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 4d1584e1..c917b15b 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -352,6 +352,7 @@ void fuzzing::equation_matrix::print_matrix() } } + // ------------------------------------------------------------------------------------------------ void fuzzing::equation_matrix::recompute_matrix() { @@ -372,6 +373,11 @@ fuzzing::possible_path fuzzing::iid_node_dependence_props::generate_probabilitie // matrix.print_matrix(); std::set< node_direction > all_leafs = get_leaf_subsets(); + // std::cout << "All leafs:" << std::endl; + // for ( const auto& leaf : all_leafs ) { + // std::cout << "Node ID: " << leaf.node_id.id + // << ", Direction: " << ( leaf.branching_direction ? "Right" : "Left" ) << std::endl; + // } equation_matrix submatrix = matrix.get_submatrix( all_leafs, true ); std::map< equation, int > vectors = submatrix.compute_vectors(); if ( vectors.empty() ) { @@ -387,8 +393,14 @@ fuzzing::possible_path fuzzing::iid_node_dependence_props::generate_probabilitie } nodes_to_counts path_counts = compute_path_counts( new_path.value(), all_leafs ); + for ( const auto& [ node_id, counts ] : path_counts ) { + std::cout << "Node ID: " << node_id.id << ", Left Count: " << counts.left_count + << ", Right Count: " << counts.right_count << std::endl; + } possible_path path = generate_path_from_node_counts( path_counts ); + // std::cout << path << std::endl; + return path; } @@ -405,7 +417,7 @@ void fuzzing::iid_node_dependence_props::process_node( branching_node* end_node } // ------------------------------------------------------------------------------------------------ -void fuzzing::iid_node_dependence_props::print_dependencies() +void fuzzing::iid_node_dependence_props::print_dependencies() const { std::cout << "# Dependencies:" << std::endl; std::cout << "## Dependencies by loops:" << std::endl; @@ -423,6 +435,48 @@ void fuzzing::iid_node_dependence_props::print_dependencies() } } +// ------------------------------------------------------------------------------------------------ +fuzzing::loop_ending_to_bodies fuzzing::iid_node_dependence_props::get_dependencies_without_bodies_in_loop() const +{ + loop_ending_to_bodies result; + int result_size = dependencies_by_loops.size(); + std::set< location_id > loop_heads; + std::set< location_id > already_added; + + for ( const auto& [ loop_head, dependent_bodies ] : dependencies_by_loops ) { + loop_heads.insert( loop_head.first ); + } + + while ( result.size() < result_size ) { + for ( const auto& [ loop_head, dependent_bodies ] : dependencies_by_loops ) { + if ( result.contains( loop_head ) ) { + continue; + } + + bool can_add = true; + for ( const auto& body : dependent_bodies ) { + if ( loop_heads.contains( body.node_id ) && !result.contains( { body.node_id, true } ) && + !result.contains( { body.node_id, false } ) ) { + can_add = false; + break; + } + } + + if ( !can_add ) { + continue; + } + + for ( const auto& body : dependent_bodies ) { + auto it = already_added.insert( body.node_id ); + if ( it.second ) { + result[ loop_head ].insert( body ); + } + } + } + } + + return result; +} // ------------------------------------------------------------------------------------------------ fuzzing::nodes_to_counts @@ -444,21 +498,46 @@ fuzzing::iid_node_dependence_props::compute_path_counts( const equation& path, } } - for ( const auto& [ loop_head, dependent_bodies ] : dependencies_by_loops ) { - int loop_count = 0; - for ( const auto& body : dependent_bodies ) { - auto& [ left_count, right_count ] = path_counts[ body.node_id ]; - loop_count = std::max( loop_count, left_count + right_count ); - } + loop_ending_to_bodies dependencies = get_dependencies_without_bodies_in_loop(); + bool changed = true; - if ( loop_head.second ) { - path_counts[ loop_head.first ] = { loop_count, 1 }; - } else { - path_counts[ loop_head.first ] = { 1, loop_count }; + while ( changed ) { + changed = false; + + for ( const auto& [ loop_head, dependent_bodies ] : dependencies ) { + if ( path_counts.contains( loop_head.first ) ) { + continue; + } + + bool can_add = true; + for ( const auto& body : dependent_bodies ) { + if ( !path_counts.contains( body.node_id ) ) { + can_add = false; + break; + } + } + + if ( !can_add ) { + continue; + } + + changed = true; + int loop_count = 0; + for ( const auto& body : dependent_bodies ) { + auto& [ left_count, right_count ] = path_counts[ body.node_id ]; + loop_count = std::max( loop_count, left_count + right_count ); + } + + if ( loop_head.second ) { + path_counts[ loop_head.first ] = { loop_count, 1 }; + } else { + path_counts[ loop_head.first ] = { 1, loop_count }; + } } } + for ( const auto& [ loading_head, loading_bodies ] : dependencies_by_loading ) { int loop_count = 0; for ( const auto& body : loading_bodies ) { @@ -551,7 +630,13 @@ std::set< fuzzing::node_direction > fuzzing::iid_node_dependence_props::get_leaf { std::set< node_direction > all_leafs; for ( const auto& [ _, loop_bodies ] : dependencies_by_loops ) { - all_leafs.insert( loop_bodies.begin(), loop_bodies.end() ); + for ( const auto& body : loop_bodies ) { + location_id body_id = body.node_id; + if ( !dependencies_by_loops.contains( { body_id, false } ) && + !dependencies_by_loops.contains( { body_id, true } ) ) { + all_leafs.insert( body ); + } + } } return all_leafs; From 5e6997cbb9ec6fb72fd5dd1c409e5b8c73ec5f3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sun, 19 Jan 2025 15:37:50 +0100 Subject: [PATCH 079/144] feat: add iid_vector_analysis and update fuzzer to include new analysis methods --- benchmarks/iid_testing/iid_twice.json | 4 +- benchmarks/iid_testing/sum_of_lopp.c | 36 ++ src/fuzzing/CMakeLists.txt | 4 +- src/fuzzing/include/fuzzing/fuzzer.hpp | 22 +- .../include/fuzzing/iid_vector_analysis.hpp | 21 +- src/fuzzing/src/fuzzer.cpp | 185 ++++++--- src/fuzzing/src/iid_vector_analysis.cpp | 369 ++++++++++++------ 7 files changed, 469 insertions(+), 172 deletions(-) create mode 100644 benchmarks/iid_testing/sum_of_lopp.c diff --git a/benchmarks/iid_testing/iid_twice.json b/benchmarks/iid_testing/iid_twice.json index a5afd7c5..e0908c22 100644 --- a/benchmarks/iid_testing/iid_twice.json +++ b/benchmarks/iid_testing/iid_twice.json @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 10000, + "num_executions": 8000, "num_covered_branchings": 10, "covered_branchings": [ 1, @@ -41,7 +41,7 @@ 10, 0 ], - "num_generated_tests": 10, + "num_generated_tests": 9, "num_crashes": 0, "num_boundary_violations": 0 } diff --git a/benchmarks/iid_testing/sum_of_lopp.c b/benchmarks/iid_testing/sum_of_lopp.c new file mode 100644 index 00000000..92c683c7 --- /dev/null +++ b/benchmarks/iid_testing/sum_of_lopp.c @@ -0,0 +1,36 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + k += s[ i ]; + ++i; + } + + if ( k == 500 ) // ID: 6 + return 1; + + return 0; + } +} diff --git a/src/fuzzing/CMakeLists.txt b/src/fuzzing/CMakeLists.txt index 00b2778c..5702a5e6 100644 --- a/src/fuzzing/CMakeLists.txt +++ b/src/fuzzing/CMakeLists.txt @@ -48,7 +48,9 @@ add_library(${THIS_TARGET_NAME} ./include/fuzzing/execution_record.hpp ./include/fuzzing/execution_record_writer.hpp ./src/execution_record_writer.cpp - ) + + ./include/fuzzing/iid_vector_analysis.hpp + ./src/iid_vector_analysis.cpp set_target_properties(${THIS_TARGET_NAME} PROPERTIES DEBUG_OUTPUT_NAME "${THIS_TARGET_NAME}_${CMAKE_SYSTEM_NAME}_Debug" diff --git a/src/fuzzing/include/fuzzing/fuzzer.hpp b/src/fuzzing/include/fuzzing/fuzzer.hpp index 48ba2f78..0c6a312e 100644 --- a/src/fuzzing/include/fuzzing/fuzzer.hpp +++ b/src/fuzzing/include/fuzzing/fuzzer.hpp @@ -4,7 +4,7 @@ # include # include # include -# include +# include # include # include # include @@ -197,13 +197,15 @@ struct fuzzer final using histogram_of_false_direction_probabilities = std::unordered_map; using probability_generators_for_locations = std::unordered_map >; +public: struct loop_boundary_props { branching_node* entry; branching_node* exit; // branching_node* successor; }; - + +private: struct iid_pivot_props { std::vector loop_boundaries; @@ -252,6 +254,11 @@ struct fuzzer final histogram_of_false_direction_probabilities& histogram ); + static branching_node* select_start_node_for_monte_carlo_search_with_vector( + possible_path const& path, + std::vector const& loop_boundaries, + branching_node* fallback_node + ); static branching_node* select_start_node_for_monte_carlo_search( std::vector const& loop_boundaries, random_generator_for_natural_32_bit& random_generator, @@ -272,7 +279,8 @@ struct fuzzer final branching_node* root, histogram_of_false_direction_probabilities const& histogram, probability_generators_for_locations const& generators, - probability_generator_random_uniform& location_miss_generator + probability_generator_random_uniform& location_miss_generator, + possible_path& path ); static std::pair monte_carlo_backward_search( branching_node* const start_node, @@ -287,6 +295,13 @@ struct fuzzer final probability_generators_for_locations const& generators, probability_generator_random_uniform& location_miss_generator ); + static branching_node* monte_carlo_step_with_path( + branching_node* const pivot, + histogram_of_false_direction_probabilities const& histogram, + probability_generators_for_locations const& generators, + probability_generator_random_uniform& location_miss_generator, + possible_path& path + ); void generate_next_input(vecb& stdin_bits); execution_record::execution_flags process_execution_results(); @@ -295,7 +310,6 @@ struct fuzzer final void collect_iid_pivots_from_sensitivity_results(); void select_next_state(); branching_node* select_iid_coverage_target(); - branching_node* select_iid_coverage_target_from_dependencies(); void remove_leaf_branching_node(branching_node* node); bool apply_coverage_failures_with_hope(); diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index 30d3639e..f4fda606 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -93,11 +93,13 @@ struct equation { equation operator+( const equation& other ) const; equation operator-( const equation& other ) const; equation operator*( int scalar ) const; + equation operator/( const equation& other ) const; auto operator<=>( const equation& other ) const = default; bool operator==( const equation& other ) const = default; int get_vector_length() const; bool is_any_negative() const; + bool same_values() const; friend std::ostream& operator<<( std::ostream& os, const equation& eq ) { @@ -120,9 +122,13 @@ struct node_direction { } }; -using loop_head_end_direction = bool; -using loop_ending_to_bodies = - std::map< std::pair< location_id, loop_head_end_direction >, std::set< node_direction > >; +struct loop_dependencies_props { + bool end_direction; + std::set< node_direction > bodies; + std::vector< node_counts > previous_counts; +}; + +using loop_ending_to_bodies = std::map< location_id, loop_dependencies_props >; using loop_endings = std::map< location_id, bool >; using loop_head_to_bodies_t = std::unordered_map< location_id, std::unordered_set< location_id > >; using loop_head_to_loaded_bits_t = std::unordered_map< location_id, std::tuple< natural_32_bit, natural_32_bit > >; @@ -138,6 +144,7 @@ struct equation_matrix { std::vector< equation >& get_matrix(); std::optional< equation > get_new_path_from_vector( const equation& vector ); int get_desired_vector_direction() const; + float get_biggest_branching_value() const; void print_matrix(); @@ -157,11 +164,14 @@ struct iid_node_dependence_props { void print_dependencies() const; private: - loop_ending_to_bodies get_dependencies_without_bodies_in_loop() const; + int compute_path_counts_for_nested_loops( std::map< location_id, int >& counts, int minimum_count ); nodes_to_counts compute_path_counts( const equation& path, std::set< node_direction > const& all_leafs ); equation get_best_vector( const std::map< equation, int >& vectors_with_hits, bool use_random, - int desired_direction ); + int desired_direction, + float biggest_branching_value ); + std::map< equation, int > get_linear_dependent_vector( const std::map< equation, int >& vectors_with_hits, + equation& best_vector ); equation get_random_vector( const std::map< equation, int >& vectors_with_hits ); std::set< node_direction > get_leaf_subsets(); loop_endings get_loop_heads_ending( branching_node* end_node, loop_head_to_bodies_t& loop_heads_to_bodies ); @@ -171,6 +181,7 @@ struct iid_node_dependence_props { void compute_dependencies_by_loops( const loop_head_to_bodies_t& loop_heads_to_bodies, const loop_endings& loop_heads_ending ); possible_path generate_path_from_node_counts( const nodes_to_counts& path_counts ); + std::set< location_id > get_loop_heads( bool include_loading_loops = true ); equation_matrix matrix; loop_ending_to_bodies dependencies_by_loops; diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index 7042c48c..5f2351b8 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -617,6 +617,26 @@ void fuzzer::compute_histogram_of_false_direction_probabilities( } } +branching_node* fuzzer::select_start_node_for_monte_carlo_search_with_vector( + const possible_path& path, + std::vector< branching_node* > const& loop_boundaries, + branching_node* fallback_node ) +{ + std::vector< branching_node* > sorted_loop_boundaries = loop_boundaries; + std::sort( sorted_loop_boundaries.begin(), + sorted_loop_boundaries.end(), + []( branching_node* left, branching_node* right ) { + return left->get_location_id().id < right->get_location_id().id; + } ); + + for ( const auto& node : sorted_loop_boundaries ) { + if ( path.contains( node->get_location_id().id ) && !node->is_closed() ) { + return node; + } + } + + return fallback_node; +} branching_node* fuzzer::select_start_node_for_monte_carlo_search( std::vector const& loop_boundaries, @@ -695,7 +715,8 @@ branching_node* fuzzer::monte_carlo_search( branching_node* const start_node, histogram_of_false_direction_probabilities const& histogram, probability_generators_for_locations const& generators, - probability_generator_random_uniform& location_miss_generator + probability_generator_random_uniform& location_miss_generator, + possible_path& path ) { TMPROF_BLOCK(); @@ -705,7 +726,8 @@ branching_node* fuzzer::monte_carlo_search( branching_node* pivot = start_node; while (true) { - branching_node* const successor{ monte_carlo_step(pivot, histogram, generators, location_miss_generator) }; + // branching_node* const successor{ monte_carlo_step(pivot, histogram, generators, location_miss_generator) }; + branching_node* const successor{ monte_carlo_step_with_path(pivot, histogram, generators, location_miss_generator, path) }; if (successor == nullptr) break; pivot = successor; @@ -748,6 +770,64 @@ std::pair fuzzer::monte_carlo_backward_search( return { pivot->predecessor, !pivot->predecessor->successor_direction(pivot) }; } +branching_node* +fuzzing::fuzzer::monte_carlo_step_with_path( branching_node* const pivot, + histogram_of_false_direction_probabilities const& histogram, + probability_generators_for_locations const& generators, + probability_generator_random_uniform& location_miss_generator, + possible_path& path ) +{ + INVARIANT( pivot != nullptr && !pivot->is_closed() ); + + branching_node* successor = nullptr; + + branching_node* left = pivot->successor( false ).pointer; + branching_node* right = pivot->successor( true ).pointer; + + bool const can_go_left = left != nullptr && !left->is_closed(); + bool const can_go_right = right != nullptr && !right->is_closed(); + + bool desired_direction; + { + float_32_bit false_direction_probability; + { + auto const it = histogram.find( pivot->get_location_id().id ); + false_direction_probability = it != histogram.end() ? it->second : 0.5f; + } + float_32_bit probability; + { + auto const it = generators.find( pivot->get_location_id().id ); + probability = it != generators.end() ? it->second->next() : location_miss_generator.next(); + } + desired_direction = probability <= false_direction_probability ? false : true; + } + + location_id::id_type pivot_id = pivot->get_location_id().id; + if ( path.contains( pivot_id ) ) { + path_node_props& props = path.get_props( pivot_id ); + if ( props.can_take_next_direction() ) { + desired_direction = props.get_desired_direction(); + } + } + + bool const can_go_desired_direction = ( desired_direction == false && can_go_left ) || + ( desired_direction == true && can_go_right ); + + if ( path.contains( pivot_id ) && can_go_desired_direction ) { + path_node_props& props = path.get_props( pivot_id ); + if ( props.can_take_next_direction() ) { + props.go_direction( desired_direction ); + } + } + + if ( can_go_desired_direction ) + successor = desired_direction == false ? left : right; + else if ( !pivot->is_open_branching() ) + successor = can_go_left ? left : right; + + // std::cout << path << std::endl; + return successor; +} branching_node* fuzzer::monte_carlo_step( branching_node* const pivot, @@ -1297,7 +1377,10 @@ void fuzzer::do_cleanup() for (auto it = iid_pivots.begin(); it != iid_pivots.end(); ) if (covered_branchings.contains(it->first)) + { + iid_dependences.remove_node_dependence(it->first); it = iid_pivots.erase(it); + } else ++it; @@ -1554,6 +1637,31 @@ branching_node* fuzzer::select_iid_coverage_target() histogram_of_hit_counts_per_direction::hit_counts_map hit_counts; it_pivot->second.histogram_ptr->merge(hit_counts); + + std::vector< location_id > iid_locations = iid_dependences.get_iid_nodes(); + { + // std::cout << "IID Locations:" << std::endl; + // for (const auto& loc : iid_locations) { + // std::cout << loc.id << std::endl; + // } + } + + possible_path path; + + if ( !iid_locations.empty() ) { + iid_node_dependence_props& node_props = iid_dependences.get_props( iid_locations[0] ); + // std::cout << "IID Location: " << iid_locations[0].id << std::endl; + path = node_props.generate_probabilities(); + + for ( const auto& path_props : path.get_path() ) { + auto it = histogram.find( path_props.first ); + if ( it != histogram.end() ) { + it->second = path_props.second.get_false_direction_probability(); + // std::cout << "IID Location: " << path_props.first << " Probability: " << it->second << std::endl; + } + } + } + probability_generators_for_locations generators; auto const random_uniform_generator = compute_probability_generators_for_locations( histogram, @@ -1568,22 +1676,22 @@ branching_node* fuzzer::select_iid_coverage_target() if (false) // original code: if (get_random_natural_32_bit_in_range(1, 100, generator_for_iid_approach_selection) <= 50) // Currently diabled, because it performs worse for some yet unknown reason. { - auto const node_and_direction = monte_carlo_backward_search( - it_pivot->first, - entry_branching, - histogram, - generators, - *random_uniform_generator - ); - branching_node* const successor = node_and_direction.first->successor(node_and_direction.second).pointer; - if (successor != nullptr) - winner = monte_carlo_search(successor, histogram, generators, *random_uniform_generator); - else if (!node_and_direction.first->is_open_branching()) - winner = monte_carlo_search(node_and_direction.first, histogram, generators, *random_uniform_generator); - else - winner = node_and_direction.first; - - recorder().on_strategy_turn_monte_carlo_backward(winner); + // auto const node_and_direction = monte_carlo_backward_search( + // it_pivot->first, + // entry_branching, + // histogram, + // generators, + // *random_uniform_generator + // ); + // branching_node* const successor = node_and_direction.first->successor(node_and_direction.second).pointer; + // if (successor != nullptr) + // winner = monte_carlo_search(successor, histogram, generators, *random_uniform_generator); + // else if (!node_and_direction.first->is_open_branching()) + // winner = monte_carlo_search(node_and_direction.first, histogram, generators, *random_uniform_generator); + // else + // winner = node_and_direction.first; + + // recorder().on_strategy_turn_monte_carlo_backward(winner); } else { @@ -1593,13 +1701,17 @@ branching_node* fuzzer::select_iid_coverage_target() 0.75f, entry_branching ); - - branching_node* possible_start = select_iid_coverage_target_from_dependencies(); - if (possible_start != nullptr) - start_node = possible_start; + + if ( !path.get_path().empty() ) { + start_node = select_start_node_for_monte_carlo_search_with_vector( + path, + it_pivot->second.loop_boundaries, + entry_branching + ); + } recorder().on_node_chosen(start_node, fuzzing::progress_recorder::START_MONTE_CARLO); - winner = monte_carlo_search(start_node, histogram, generators, *random_uniform_generator); + winner = monte_carlo_search(start_node, histogram, generators, *random_uniform_generator, path); ++statistics.strategy_monte_carlo; recorder().on_strategy_turn_monte_carlo(winner); @@ -1610,33 +1722,6 @@ branching_node* fuzzer::select_iid_coverage_target() return winner; } -branching_node* fuzzer::select_iid_coverage_target_from_dependencies() -{ - if (iid_dependences.id_to_equation_map.empty()) { - return nullptr; - } - - instrumentation::location_id loc_id = iid_dependences.id_to_equation_map.begin()->first; - - - iid_node_dependence_props& props = iid_dependences.id_to_equation_map.at(loc_id); - std::map< location_id, fuzzing::path_decision > path = props.generate_path(); - - branching_node* node = entry_branching; - while ( true ) { - bool direction = path[node->get_location_id()].get_next_direction(); - branching_node* next_node = node->successor(direction).pointer; - if (next_node == nullptr) { - break; - } - recorder().on_node_chosen(next_node, fuzzing::progress_recorder::DEPENDENCY_STEP); - node = next_node; - } - recorder().on_node_chosen(node, fuzzing::progress_recorder::DEPENDENCY_END); - - return nullptr; - // return node; -} void fuzzer::remove_leaf_branching_node(branching_node* node) { diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index c917b15b..42b795f5 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -139,6 +140,24 @@ fuzzing::equation fuzzing::equation::operator*( int scalar ) const return { new_values, best_value * scalar }; } +// ------------------------------------------------------------------------------------------------ +fuzzing::equation fuzzing::equation::operator/( const equation& other ) const +{ + INVARIANT( values.size() == other.values.size() ); + + std::vector< int > new_values; + for ( int i = 0; i < values.size(); ++i ) { + if ( other.values[ i ] == 0 ) { + new_values.push_back( 0 ); + } else { + new_values.push_back( values[ i ] / other.values[ i ] ); + } + } + + + return { new_values, best_value / other.best_value }; +} + // ------------------------------------------------------------------------------------------------ int fuzzing::equation::get_vector_length() const { @@ -151,6 +170,18 @@ bool fuzzing::equation::is_any_negative() const return std::any_of( values.begin(), values.end(), []( int val ) { return val < 0; } ); } +// ------------------------------------------------------------------------------------------------ +bool fuzzing::equation::same_values() const +{ + for ( int i = 0; i < values.size(); ++i ) { + if ( values[ i ] != best_value ) { + return false; + } + } + + return true; +} + // node_direction // ------------------------------------------------------------------------------------------------ auto fuzzing::node_direction::operator<=>( node_direction const& other ) const @@ -302,6 +333,20 @@ int fuzzing::equation_matrix::get_desired_vector_direction() const } } +// ------------------------------------------------------------------------------------------------ +float fuzzing::equation_matrix::get_biggest_branching_value() const +{ + float biggest_value = 0.0f; + + for ( const equation& row : matrix ) { + if ( std::abs( row.best_value ) > biggest_value ) { + biggest_value = std::abs( row.best_value ); + } + } + + return biggest_value; +} + // ------------------------------------------------------------------------------------------------ std::optional< fuzzing::equation > fuzzing::equation_matrix::get_new_path_from_vector( const equation& vector ) { @@ -309,20 +354,31 @@ std::optional< fuzzing::equation > fuzzing::equation_matrix::get_new_path_from_v std::vector< equation > paths; std::pair< std::size_t, std::size_t > dimensions = get_dimensions(); + bool get_precise = true; + + auto add_if_positive = [ &paths ]( const equation& new_path ) { + if ( !new_path.is_any_negative() ) { + paths.push_back( new_path ); + } + }; for ( const auto& row : matrix ) { double counts = std::abs( row.best_value ) / vector.best_value; + int rounded_counts = static_cast< int >( std::round( counts ) ); - if ( std::abs( counts - std::round( counts ) ) > 1e-6 ) { - continue; - } + if ( get_precise ) { + if ( std::abs( counts - double( rounded_counts ) ) > 1e-6 ) { + continue; + } - equation new_path = vector * static_cast< int >( std::round( counts ) ) + row; - if ( new_path.is_any_negative() ) { - continue; + equation new_path = vector * rounded_counts + row; + add_if_positive( new_path ); + } else { + for ( int vector_increment = 1; vector_increment <= rounded_counts; ++vector_increment ) { + equation new_path = ( vector * vector_increment ) + row; + add_if_positive( new_path ); + } } - - paths.push_back( new_path ); } if ( paths.empty() ) { @@ -330,7 +386,10 @@ std::optional< fuzzing::equation > fuzzing::equation_matrix::get_new_path_from_v } auto min_it = std::min_element( paths.begin(), paths.end(), []( const equation& a, const equation& b ) { - return a.get_vector_length() < b.get_vector_length(); + if ( std::abs( a.best_value ) == std::abs( b.best_value ) ) { + return a.get_vector_length() < b.get_vector_length(); + } + return std::abs( a.best_value ) < std::abs( b.best_value ); } ); return *min_it; @@ -373,34 +432,49 @@ fuzzing::possible_path fuzzing::iid_node_dependence_props::generate_probabilitie // matrix.print_matrix(); std::set< node_direction > all_leafs = get_leaf_subsets(); - // std::cout << "All leafs:" << std::endl; - // for ( const auto& leaf : all_leafs ) { - // std::cout << "Node ID: " << leaf.node_id.id - // << ", Direction: " << ( leaf.branching_direction ? "Right" : "Left" ) << std::endl; - // } equation_matrix submatrix = matrix.get_submatrix( all_leafs, true ); std::map< equation, int > vectors = submatrix.compute_vectors(); if ( vectors.empty() ) { + // std::cout << "No vectors found." << std::endl; return {}; } int desired_vector_direction = submatrix.get_desired_vector_direction(); - equation best_vector = get_best_vector( vectors, false, desired_vector_direction ); + float biggest_branching_value = submatrix.get_biggest_branching_value(); + equation best_vector = get_best_vector( vectors, false, desired_vector_direction, biggest_branching_value ); std::optional< equation > new_path = submatrix.get_new_path_from_vector( best_vector ); if ( !new_path.has_value() ) { + // std::cout << "Desired vector direction: " << desired_vector_direction << std::endl; + // std::cout << "Biggest branching value: " << biggest_branching_value << std::endl; + // std::cout << "Vectors:" << std::endl; + // for ( const auto& [ vector, hits ] : vectors ) { + // std::cout << "Vector: "; + // for ( const auto& value : vector.values ) { + // std::cout << value << " "; + // } + // std::cout << "-> " << vector.best_value << " "; + // std::cout << "Hits: " << hits << std::endl; + // } + // std::cout << "Best vector: "; + // for ( const auto& value : best_vector.values ) { + // std::cout << value << " "; + // } + // std::cout << "-> " << best_vector.best_value << " "; + // std::cout << "Hits: " << vectors[ best_vector ] << std::endl; return {}; } nodes_to_counts path_counts = compute_path_counts( new_path.value(), all_leafs ); - for ( const auto& [ node_id, counts ] : path_counts ) { - std::cout << "Node ID: " << node_id.id << ", Left Count: " << counts.left_count - << ", Right Count: " << counts.right_count << std::endl; - } possible_path path = generate_path_from_node_counts( path_counts ); - // std::cout << path << std::endl; - + // std::cout << "Best vector: "; + // for ( const auto& value : best_vector.values ) { + // std::cout << value << " "; + // } + // std::cout << "-> " << best_vector.best_value << " "; + // std::cout << "Hits: " << vectors[ best_vector ] << std::endl; + // std::cout << "Generated path:" << std::endl << path << std::endl; return path; } @@ -421,61 +495,51 @@ void fuzzing::iid_node_dependence_props::print_dependencies() const { std::cout << "# Dependencies:" << std::endl; std::cout << "## Dependencies by loops:" << std::endl; - for ( const auto& [ loop, nodes ] : dependencies_by_loops ) { - for ( const auto& body : nodes ) { - std::cout << "- " << "`(" << body << ") → " << loop.first.id << "`" << std::endl; + for ( const auto& [ loop_head, props ] : dependencies_by_loops ) { + for ( const auto& body : props.bodies ) { + std::cout << "- " << "`(" << body << ") → " << loop_head.id << "`" << std::endl; } } std::cout << "## Dependencies by loading:" << std::endl; - for ( const auto& [ loading, nodes ] : dependencies_by_loading ) { - for ( const auto& body : nodes ) { - std::cout << "- " << "`(" << body << ") → " << loading.first.id << "`" << std::endl; + for ( const auto& [ loop_head, props ] : dependencies_by_loading ) { + for ( const auto& body : props.bodies ) { + std::cout << "- " << "`(" << body << ") → " << loop_head.id << "`" << std::endl; } } } // ------------------------------------------------------------------------------------------------ -fuzzing::loop_ending_to_bodies fuzzing::iid_node_dependence_props::get_dependencies_without_bodies_in_loop() const +int fuzzing::iid_node_dependence_props::compute_path_counts_for_nested_loops( std::map< location_id, int >& counts, int minimum_count ) { - loop_ending_to_bodies result; - int result_size = dependencies_by_loops.size(); - std::set< location_id > loop_heads; - std::set< location_id > already_added; + int previous_count = 1; - for ( const auto& [ loop_head, dependent_bodies ] : dependencies_by_loops ) { - loop_heads.insert( loop_head.first ); - } - - while ( result.size() < result_size ) { - for ( const auto& [ loop_head, dependent_bodies ] : dependencies_by_loops ) { - if ( result.contains( loop_head ) ) { - continue; - } + int max_count = std::max_element( counts.begin(), counts.end(), []( const auto& a, const auto& b ) { + return a.second < b.second; + } )->second; - bool can_add = true; - for ( const auto& body : dependent_bodies ) { - if ( loop_heads.contains( body.node_id ) && !result.contains( { body.node_id, true } ) && - !result.contains( { body.node_id, false } ) ) { - can_add = false; - break; - } - } + int highest_count = 1; + for ( int i = minimum_count; i <= max_count; ++i ) { - if ( !can_add ) { - continue; + bool is_good = true; + for ( const auto& [ node_id, count ] : counts ) { + if ( count % i != 0 ) { + is_good = false; + break; } + } - for ( const auto& body : dependent_bodies ) { - auto it = already_added.insert( body.node_id ); - if ( it.second ) { - result[ loop_head ].insert( body ); - } - } + if ( is_good ) { + highest_count = i; + break; } } - return result; + for ( auto& [ node_id, count ] : counts ) { + count /= highest_count; + } + + return highest_count; } // ------------------------------------------------------------------------------------------------ @@ -486,6 +550,7 @@ fuzzing::iid_node_dependence_props::compute_path_counts( const equation& path, nodes_to_counts path_counts; std::vector< node_direction > leafs = std::vector< node_direction >( all_leafs.begin(), all_leafs.end() ); + std::set< location_id > loop_heads = get_loop_heads( false ); INVARIANT( leafs.size() == path.values.size() ); @@ -498,61 +563,86 @@ fuzzing::iid_node_dependence_props::compute_path_counts( const equation& path, } } - loop_ending_to_bodies dependencies = get_dependencies_without_bodies_in_loop(); - bool changed = true; + // First + for ( int i = 0; i < dependencies_by_loops.size(); ++i ) { + for ( const auto& [ loop_head, props ] : dependencies_by_loops ) { + int loop_count = 0; + for ( const auto& body : props.bodies ) { + auto& [ left_count, right_count ] = path_counts[ body.node_id ]; - while ( changed ) { - changed = false; + if ( loop_heads.contains( body.node_id ) ) { + loop_count = std::max( loop_count, 1 ); + } else { + loop_count = std::max( loop_count, left_count + right_count ); + } + } - for ( const auto& [ loop_head, dependent_bodies ] : dependencies ) { - if ( path_counts.contains( loop_head.first ) ) { - continue; + if ( props.end_direction ) { + path_counts[ loop_head ] = { loop_count, 1 }; + } else { + path_counts[ loop_head ] = { 1, loop_count }; } + } + } - bool can_add = true; - for ( const auto& body : dependent_bodies ) { - if ( !path_counts.contains( body.node_id ) ) { - can_add = false; - break; - } + // Second + for ( const auto& [ loop_head, props ] : std::ranges::views::reverse( dependencies_by_loops ) ) { + int non_loop_child_max_count = 1; + std::map< location_id, int > counts; + + for ( const auto& body : props.bodies ) { + auto& [ left_count, right_count ] = path_counts[ body.node_id ]; + + if ( !loop_heads.contains( body.node_id ) ) { + non_loop_child_max_count = std::max( non_loop_child_max_count, left_count + right_count ); + } else { + counts[ body.node_id ] = std::max( left_count, right_count ); } + } - if ( !can_add ) { - continue; + int new_count = compute_path_counts_for_nested_loops( counts, non_loop_child_max_count ); + if ( props.end_direction ) { + path_counts[ loop_head ] = { new_count, 1 }; + } else { + path_counts[ loop_head ] = { 1, new_count }; + } + + for ( const auto& [ node_id, count ] : counts ) { + const loop_dependencies_props& o_props = dependencies_by_loops.at( node_id ); + if ( o_props.end_direction ) { + path_counts[ node_id ] = { count, 1 }; + } else { + path_counts[ node_id ] = { 1, count }; } + } + } - changed = true; + for ( int i = 0; i < dependencies_by_loading.size(); ++i ) { + for ( const auto& [ loading_head, props ] : dependencies_by_loading ) { int loop_count = 0; - for ( const auto& body : dependent_bodies ) { + for ( const auto& body : props.bodies ) { + if ( !path_counts.contains( body.node_id ) ) { + continue; + } + auto& [ left_count, right_count ] = path_counts[ body.node_id ]; loop_count = std::max( loop_count, left_count + right_count ); } - if ( loop_head.second ) { - path_counts[ loop_head.first ] = { loop_count, 1 }; + if ( props.end_direction ) { + path_counts[ loading_head ] = { loop_count, 1 }; } else { - path_counts[ loop_head.first ] = { 1, loop_count }; + path_counts[ loading_head ] = { 1, loop_count }; } } } - - - for ( const auto& [ loading_head, loading_bodies ] : dependencies_by_loading ) { - int loop_count = 0; - for ( const auto& body : loading_bodies ) { - if ( !path_counts.contains( body.node_id ) ) { - continue; + for ( auto* map : { &dependencies_by_loops, &dependencies_by_loading } ) { + for ( auto& [ loop_head, props ] : *map ) { + auto it = path_counts.find( loop_head ); + if ( it != path_counts.end() ) { + props.previous_counts.push_back( it->second ); } - - auto& [ left_count, right_count ] = path_counts[ body.node_id ]; - loop_count = std::max( loop_count, left_count + right_count ); - } - - if ( loading_head.second ) { - path_counts[ loading_head.first ] = { loop_count, 1 }; - } else { - path_counts[ loading_head.first ] = { 1, loop_count }; } } @@ -562,7 +652,8 @@ fuzzing::iid_node_dependence_props::compute_path_counts( const equation& path, // ------------------------------------------------------------------------------------------------ fuzzing::equation fuzzing::iid_node_dependence_props::get_best_vector( const std::map< equation, int >& vectors_with_hits, bool use_random, - int desired_direction ) + int desired_direction, + float biggest_branching_value ) { if ( vectors_with_hits.empty() ) { throw std::invalid_argument( "Input map is empty." ); @@ -583,6 +674,11 @@ fuzzing::equation fuzzing::iid_node_dependence_props::get_best_vector( const std filtered_vectors_with_hits = vectors_with_hits; } + biggest_branching_value = std::abs( biggest_branching_value ); + std::erase_if( filtered_vectors_with_hits, [ biggest_branching_value ]( const auto& pair ) { + return std::abs( pair.first.best_value ) > biggest_branching_value; + } ); + if ( filtered_vectors_with_hits.empty() ) { throw std::invalid_argument( "No vectors match the desired direction." ); } @@ -593,9 +689,47 @@ fuzzing::equation fuzzing::iid_node_dependence_props::get_best_vector( const std auto max_it = std::max_element( filtered_vectors_with_hits.begin(), filtered_vectors_with_hits.end(), - []( const auto& a, const auto& b ) { return a.second < b.second; } ); + []( const auto& a, const auto& b ) { + if ( a.second == b.second ) { + return std::abs( a.first.best_value ) > std::abs( b.first.best_value ); + } + return a.second < b.second; + } ); + + equation best_vector = max_it->first; + + // std::map< equation, int > dependent_vectors_with_hits = + // get_linear_dependent_vector( filtered_vectors_with_hits, best_vector ); + + // if ( !dependent_vectors_with_hits.empty() ) { + // auto min_it = std::min_element( dependent_vectors_with_hits.begin(), + // dependent_vectors_with_hits.end(), + // []( const auto& a, const auto& b ) { + // return a.first.get_vector_length() < + // b.first.get_vector_length(); + // } ); + + // best_vector = min_it->first; + // } + + return best_vector; +} + +// ------------------------------------------------------------------------------------------------ +std::map< fuzzing::equation, int > fuzzing::iid_node_dependence_props::get_linear_dependent_vector( + const std::map< equation, int >& vectors_with_hits, + equation& best_vector ) +{ + std::map< equation, int > dependent_vectors_with_hits; + + for ( const auto& [ vector, hits ] : vectors_with_hits ) { + equation quotient = best_vector / vector; + if ( quotient.same_values() ) { + dependent_vectors_with_hits[ vector ] = hits; + } + } - return max_it->first; + return dependent_vectors_with_hits; } // ------------------------------------------------------------------------------------------------ @@ -629,11 +763,10 @@ fuzzing::equation fuzzing::iid_node_dependence_props::get_random_vector( const s std::set< fuzzing::node_direction > fuzzing::iid_node_dependence_props::get_leaf_subsets() { std::set< node_direction > all_leafs; - for ( const auto& [ _, loop_bodies ] : dependencies_by_loops ) { - for ( const auto& body : loop_bodies ) { + for ( const auto& [ _, props ] : dependencies_by_loops ) { + for ( const auto& body : props.bodies ) { location_id body_id = body.node_id; - if ( !dependencies_by_loops.contains( { body_id, false } ) && - !dependencies_by_loops.contains( { body_id, true } ) ) { + if ( !dependencies_by_loops.contains( body_id ) ) { all_leafs.insert( body ); } } @@ -728,7 +861,7 @@ void fuzzing::iid_node_dependence_props::compute_dependencies_by_loading( branch node_direction node_id_direction = { node_id, direction }; if ( matrix.contains( node_id_direction ) ) { - dependencies_by_loading[ { loop_head, loop_head_direction } ].insert( node_id_direction ); + dependencies_by_loading[ loop_head ].bodies.insert( node_id_direction ); } } } @@ -754,7 +887,7 @@ void fuzzing::iid_node_dependence_props::compute_dependencies_by_loops( const lo node_direction node_id_direction = { body, direction }; if ( matrix.contains( node_id_direction ) ) - dependencies_by_loops[ { loop_head, loop_head_end_direction } ].insert( node_id_direction ); + dependencies_by_loops[ loop_head ].bodies.insert( node_id_direction ); } } } @@ -773,13 +906,12 @@ fuzzing::iid_node_dependence_props::generate_path_from_node_counts( const nodes_ bool loop_head_direction = false; bool is_loop_head = false; - for ( bool direction : { false, true } ) { - for ( const auto& map : { dependencies_by_loops, dependencies_by_loading } ) { - if ( map.contains( std::make_pair( id, direction ) ) ) { - is_loop_head = true; - loop_head_direction = direction; - break; - } + for ( const auto& map : { dependencies_by_loops, dependencies_by_loading } ) { + auto it = map.find( id ); + if ( it != map.end() ) { + is_loop_head = true; + loop_head_direction = it->second.end_direction; + break; } } @@ -790,6 +922,23 @@ fuzzing::iid_node_dependence_props::generate_path_from_node_counts( const nodes_ return possible_path( path ); } +// ------------------------------------------------------------------------------------------------ +std::set< location_id > fuzzing::iid_node_dependence_props::get_loop_heads( bool include_loading_loops ) +{ + std::set< location_id > loop_heads; + for ( const auto& [ loop_head, _ ] : dependencies_by_loops ) { + loop_heads.insert( loop_head ); + } + + if ( include_loading_loops ) { + for ( const auto& [ loading_head, _ ] : dependencies_by_loading ) { + loop_heads.insert( loading_head ); + } + } + + return loop_heads; +} + // iid_dependencies // ------------------------------------------------------------------------------------------------ void fuzzing::iid_dependencies::update_non_iid_nodes( sensitivity_analysis& sensitivity ) From af2d4d9e35b9dbda4657b6aca46edddc2d49545d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sun, 19 Jan 2025 20:42:35 +0100 Subject: [PATCH 080/144] feat: update benchmark JSON files to include detailed output statistics for various analyses --- .../iid_testing/combination_needed.json | 25 +++++++++++++++-- .../iid_testing/decrement_in_condition.json | 25 +++++++++++++++-- benchmarks/iid_testing/decrement_in_else.json | 25 +++++++++++++++-- benchmarks/iid_testing/decrement_on_b.json | 25 +++++++++++++++-- benchmarks/iid_testing/double_loop.json | 25 +++++++++++++++-- benchmarks/iid_testing/iid.json | 28 ++++++++++++++++--- benchmarks/iid_testing/iid_in_condition.json | 20 +++++++++++-- benchmarks/iid_testing/iid_multiple.json | 25 +++++++++++++++-- benchmarks/iid_testing/iid_twice.json | 25 +++++++++++++++-- benchmarks/iid_testing/input_cycle.json | 25 +++++++++++++++-- benchmarks/iid_testing/input_cycle_dec_b.json | 25 +++++++++++++++-- .../lots_of_non_iid_conditions.json | 25 +++++++++++++++-- .../iid_testing/more_iid_condition.json | 25 +++++++++++++++-- .../more_iid_condition_nested.json | 25 +++++++++++++++-- .../iid_testing/multiplication_in_iid.json | 25 +++++++++++++++-- .../iid_testing/same_loop_dependent.json | 25 +++++++++++++++-- .../iid_testing/same_loop_dependent_2.json | 25 +++++++++++++++-- .../iid_testing/same_loop_independent.json | 25 +++++++++++++++-- 18 files changed, 393 insertions(+), 55 deletions(-) diff --git a/benchmarks/iid_testing/combination_needed.json b/benchmarks/iid_testing/combination_needed.json index f35f0e9d..89fda376 100644 --- a/benchmarks/iid_testing/combination_needed.json +++ b/benchmarks/iid_testing/combination_needed.json @@ -37,8 +37,27 @@ 8, 0 ], - "num_generated_tests": 8, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/iid_testing/decrement_in_condition.json b/benchmarks/iid_testing/decrement_in_condition.json index 9866f6c3..87bb1359 100644 --- a/benchmarks/iid_testing/decrement_in_condition.json +++ b/benchmarks/iid_testing/decrement_in_condition.json @@ -39,8 +39,27 @@ 9, 0 ], - "num_generated_tests": 9, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 5, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/iid_testing/decrement_in_else.json b/benchmarks/iid_testing/decrement_in_else.json index 1bc13e3e..2c9853ad 100644 --- a/benchmarks/iid_testing/decrement_in_else.json +++ b/benchmarks/iid_testing/decrement_in_else.json @@ -35,8 +35,27 @@ 7, 0 ], - "num_generated_tests": 7, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/iid_testing/decrement_on_b.json b/benchmarks/iid_testing/decrement_on_b.json index e5ce7d8b..a0a83fbc 100644 --- a/benchmarks/iid_testing/decrement_on_b.json +++ b/benchmarks/iid_testing/decrement_on_b.json @@ -37,8 +37,27 @@ 8, 0 ], - "num_generated_tests": 8, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/iid_testing/double_loop.json b/benchmarks/iid_testing/double_loop.json index 2dc5d8a4..551d55c6 100644 --- a/benchmarks/iid_testing/double_loop.json +++ b/benchmarks/iid_testing/double_loop.json @@ -43,8 +43,27 @@ 11, 0 ], - "num_generated_tests": 10, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/iid_testing/iid.json b/benchmarks/iid_testing/iid.json index eb0cc95d..5bbee2f3 100644 --- a/benchmarks/iid_testing/iid.json +++ b/benchmarks/iid_testing/iid.json @@ -11,7 +11,8 @@ "stdout_model": "stdout_void", "optimizer_max_seconds": 10, "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 + "optimizer_max_stdin_bytes": 1000000, + "m32": false }, "num_executions_original": 6044, "results": { @@ -35,8 +36,27 @@ 7, 0 ], - "num_generated_tests": 7, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/iid_testing/iid_in_condition.json b/benchmarks/iid_testing/iid_in_condition.json index 7cb98f87..7adf874e 100644 --- a/benchmarks/iid_testing/iid_in_condition.json +++ b/benchmarks/iid_testing/iid_in_condition.json @@ -39,8 +39,22 @@ 9, 0 ], - "num_generated_tests": 8, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 6, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/iid_testing/iid_multiple.json b/benchmarks/iid_testing/iid_multiple.json index 8cc8ddfe..c141381d 100644 --- a/benchmarks/iid_testing/iid_multiple.json +++ b/benchmarks/iid_testing/iid_multiple.json @@ -53,8 +53,27 @@ 16, 0 ], - "num_generated_tests": 16, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 10, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/iid_testing/iid_twice.json b/benchmarks/iid_testing/iid_twice.json index e0908c22..b3ce5c66 100644 --- a/benchmarks/iid_testing/iid_twice.json +++ b/benchmarks/iid_testing/iid_twice.json @@ -41,8 +41,27 @@ 10, 0 ], - "num_generated_tests": 9, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/iid_testing/input_cycle.json b/benchmarks/iid_testing/input_cycle.json index 313fc712..30f8783f 100644 --- a/benchmarks/iid_testing/input_cycle.json +++ b/benchmarks/iid_testing/input_cycle.json @@ -39,8 +39,27 @@ 9, 0 ], - "num_generated_tests": 9, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/iid_testing/input_cycle_dec_b.json b/benchmarks/iid_testing/input_cycle_dec_b.json index 454ff514..7e6b4e41 100644 --- a/benchmarks/iid_testing/input_cycle_dec_b.json +++ b/benchmarks/iid_testing/input_cycle_dec_b.json @@ -41,8 +41,27 @@ 10, 0 ], - "num_generated_tests": 10, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/iid_testing/lots_of_non_iid_conditions.json b/benchmarks/iid_testing/lots_of_non_iid_conditions.json index 2ec248ec..e82fd758 100644 --- a/benchmarks/iid_testing/lots_of_non_iid_conditions.json +++ b/benchmarks/iid_testing/lots_of_non_iid_conditions.json @@ -49,8 +49,27 @@ 14, 0 ], - "num_generated_tests": 14, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/iid_testing/more_iid_condition.json b/benchmarks/iid_testing/more_iid_condition.json index 79c37bc3..3b4aa87b 100644 --- a/benchmarks/iid_testing/more_iid_condition.json +++ b/benchmarks/iid_testing/more_iid_condition.json @@ -39,8 +39,27 @@ 9, 0 ], - "num_generated_tests": 9, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/iid_testing/more_iid_condition_nested.json b/benchmarks/iid_testing/more_iid_condition_nested.json index 6e9de3a8..109ac279 100644 --- a/benchmarks/iid_testing/more_iid_condition_nested.json +++ b/benchmarks/iid_testing/more_iid_condition_nested.json @@ -38,8 +38,27 @@ 9, 0 ], - "num_generated_tests": 9, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/iid_testing/multiplication_in_iid.json b/benchmarks/iid_testing/multiplication_in_iid.json index 8302a11b..a65fc3b0 100644 --- a/benchmarks/iid_testing/multiplication_in_iid.json +++ b/benchmarks/iid_testing/multiplication_in_iid.json @@ -35,8 +35,27 @@ 7, 0 ], - "num_generated_tests": 7, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/iid_testing/same_loop_dependent.json b/benchmarks/iid_testing/same_loop_dependent.json index 82000af4..37502f5e 100644 --- a/benchmarks/iid_testing/same_loop_dependent.json +++ b/benchmarks/iid_testing/same_loop_dependent.json @@ -43,8 +43,27 @@ 11, 0 ], - "num_generated_tests": 8, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/iid_testing/same_loop_dependent_2.json b/benchmarks/iid_testing/same_loop_dependent_2.json index 5efb5bdd..6649b7a3 100644 --- a/benchmarks/iid_testing/same_loop_dependent_2.json +++ b/benchmarks/iid_testing/same_loop_dependent_2.json @@ -42,8 +42,27 @@ 11, 0 ], - "num_generated_tests": 10, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file diff --git a/benchmarks/iid_testing/same_loop_independent.json b/benchmarks/iid_testing/same_loop_independent.json index 8b93505a..61be981e 100644 --- a/benchmarks/iid_testing/same_loop_independent.json +++ b/benchmarks/iid_testing/same_loop_independent.json @@ -43,8 +43,27 @@ 11, 0 ], - "num_generated_tests": 10, - "num_crashes": 0, - "num_boundary_violations": 0 + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } } } \ No newline at end of file From db45978d5ecdef5fe25491144d8b3b7c3292b56d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sun, 19 Jan 2025 20:42:44 +0100 Subject: [PATCH 081/144] fix: correct CMakeLists.txt formatting by removing unnecessary newline --- src/fuzzing/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fuzzing/CMakeLists.txt b/src/fuzzing/CMakeLists.txt index 5702a5e6..8245ef50 100644 --- a/src/fuzzing/CMakeLists.txt +++ b/src/fuzzing/CMakeLists.txt @@ -51,7 +51,7 @@ add_library(${THIS_TARGET_NAME} ./include/fuzzing/iid_vector_analysis.hpp ./src/iid_vector_analysis.cpp - +) set_target_properties(${THIS_TARGET_NAME} PROPERTIES DEBUG_OUTPUT_NAME "${THIS_TARGET_NAME}_${CMAKE_SYSTEM_NAME}_Debug" RELEASE_OUTPUT_NAME "${THIS_TARGET_NAME}_${CMAKE_SYSTEM_NAME}_Release" From 7f2719a886d8ef203f873d50a0ad0453259dd103 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Tue, 21 Jan 2025 15:12:32 +0100 Subject: [PATCH 082/144] feat: update after merge --- .../include/fuzzing/iid_vector_analysis.hpp | 1 + src/fuzzing/src/fuzzer.cpp | 10 +- src/fuzzing/src/iid_vector_analysis.cpp | 110 ++++++++---------- 3 files changed, 55 insertions(+), 66 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index f4fda606..2aa1227e 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -148,6 +148,7 @@ struct equation_matrix { void print_matrix(); + BRANCHING_PREDICATE get_branching_predicate(); private: void recompute_matrix(); diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index 9a198e3d..98407dcf 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -736,7 +736,7 @@ branching_node* fuzzer::monte_carlo_search( TMPROF_BLOCK(); ASSUMPTION(start_node != nullptr && !start_node->is_closed()); - + branching_node* pivot = start_node; while (true) { @@ -1213,7 +1213,11 @@ execution_record::execution_flags fuzzer::process_execution_results() num_driver_executions, succ_info.xor_like_branching_function, succ_info.predicate - ) + ); + + construction_props.leaf->set_successor(info.direction, { + branching_node::successor_pointer::VISITED, + new_node }); iid_dependences.process_node_dependence(new_node); @@ -1740,7 +1744,7 @@ branching_node* fuzzer::select_iid_coverage_target() 0.75f, entry_branching ); - + if ( !path.get_path().empty() ) { start_node = select_start_node_for_monte_carlo_search_with_vector( path, diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 42b795f5..dea60ff7 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -412,6 +412,13 @@ void fuzzing::equation_matrix::print_matrix() } +// ------------------------------------------------------------------------------------------------ +BRANCHING_PREDICATE fuzzing::equation_matrix::get_branching_predicate() +{ + ASSUMPTION( all_paths.size() > 0 ); + return all_paths[ 0 ]->branching_predicate; +} + // ------------------------------------------------------------------------------------------------ void fuzzing::equation_matrix::recompute_matrix() { @@ -435,7 +442,6 @@ fuzzing::possible_path fuzzing::iid_node_dependence_props::generate_probabilitie equation_matrix submatrix = matrix.get_submatrix( all_leafs, true ); std::map< equation, int > vectors = submatrix.compute_vectors(); if ( vectors.empty() ) { - // std::cout << "No vectors found." << std::endl; return {}; } @@ -445,36 +451,12 @@ fuzzing::possible_path fuzzing::iid_node_dependence_props::generate_probabilitie std::optional< equation > new_path = submatrix.get_new_path_from_vector( best_vector ); if ( !new_path.has_value() ) { - // std::cout << "Desired vector direction: " << desired_vector_direction << std::endl; - // std::cout << "Biggest branching value: " << biggest_branching_value << std::endl; - // std::cout << "Vectors:" << std::endl; - // for ( const auto& [ vector, hits ] : vectors ) { - // std::cout << "Vector: "; - // for ( const auto& value : vector.values ) { - // std::cout << value << " "; - // } - // std::cout << "-> " << vector.best_value << " "; - // std::cout << "Hits: " << hits << std::endl; - // } - // std::cout << "Best vector: "; - // for ( const auto& value : best_vector.values ) { - // std::cout << value << " "; - // } - // std::cout << "-> " << best_vector.best_value << " "; - // std::cout << "Hits: " << vectors[ best_vector ] << std::endl; return {}; } nodes_to_counts path_counts = compute_path_counts( new_path.value(), all_leafs ); possible_path path = generate_path_from_node_counts( path_counts ); - // std::cout << "Best vector: "; - // for ( const auto& value : best_vector.values ) { - // std::cout << value << " "; - // } - // std::cout << "-> " << best_vector.best_value << " "; - // std::cout << "Hits: " << vectors[ best_vector ] << std::endl; - // std::cout << "Generated path:" << std::endl << path << std::endl; return path; } @@ -586,36 +568,36 @@ fuzzing::iid_node_dependence_props::compute_path_counts( const equation& path, } // Second - for ( const auto& [ loop_head, props ] : std::ranges::views::reverse( dependencies_by_loops ) ) { - int non_loop_child_max_count = 1; - std::map< location_id, int > counts; - - for ( const auto& body : props.bodies ) { - auto& [ left_count, right_count ] = path_counts[ body.node_id ]; - - if ( !loop_heads.contains( body.node_id ) ) { - non_loop_child_max_count = std::max( non_loop_child_max_count, left_count + right_count ); - } else { - counts[ body.node_id ] = std::max( left_count, right_count ); - } - } - - int new_count = compute_path_counts_for_nested_loops( counts, non_loop_child_max_count ); - if ( props.end_direction ) { - path_counts[ loop_head ] = { new_count, 1 }; - } else { - path_counts[ loop_head ] = { 1, new_count }; - } - - for ( const auto& [ node_id, count ] : counts ) { - const loop_dependencies_props& o_props = dependencies_by_loops.at( node_id ); - if ( o_props.end_direction ) { - path_counts[ node_id ] = { count, 1 }; - } else { - path_counts[ node_id ] = { 1, count }; - } - } - } + // for ( const auto& [ loop_head, props ] : std::ranges::views::reverse( dependencies_by_loops ) ) { + // int non_loop_child_max_count = 1; + // std::map< location_id, int > counts; + + // for ( const auto& body : props.bodies ) { + // auto& [ left_count, right_count ] = path_counts[ body.node_id ]; + + // if ( !loop_heads.contains( body.node_id ) ) { + // non_loop_child_max_count = std::max( non_loop_child_max_count, left_count + right_count ); + // } else { + // counts[ body.node_id ] = std::max( left_count, right_count ); + // } + // } + + // int new_count = compute_path_counts_for_nested_loops( counts, non_loop_child_max_count ); + // if ( props.end_direction ) { + // path_counts[ loop_head ] = { new_count, 1 }; + // } else { + // path_counts[ loop_head ] = { 1, new_count }; + // } + + // for ( const auto& [ node_id, count ] : counts ) { + // const loop_dependencies_props& o_props = dependencies_by_loops.at( node_id ); + // if ( o_props.end_direction ) { + // path_counts[ node_id ] = { count, 1 }; + // } else { + // path_counts[ node_id ] = { 1, count }; + // } + // } + // } for ( int i = 0; i < dependencies_by_loading.size(); ++i ) { for ( const auto& [ loading_head, props ] : dependencies_by_loading ) { @@ -637,14 +619,14 @@ fuzzing::iid_node_dependence_props::compute_path_counts( const equation& path, } } - for ( auto* map : { &dependencies_by_loops, &dependencies_by_loading } ) { - for ( auto& [ loop_head, props ] : *map ) { - auto it = path_counts.find( loop_head ); - if ( it != path_counts.end() ) { - props.previous_counts.push_back( it->second ); - } - } - } + // for ( auto* map : { &dependencies_by_loops, &dependencies_by_loading } ) { + // for ( auto& [ loop_head, props ] : *map ) { + // auto it = path_counts.find( loop_head ); + // if ( it != path_counts.end() ) { + // props.previous_counts.push_back( it->second ); + // } + // } + // } return path_counts; } @@ -862,6 +844,7 @@ void fuzzing::iid_node_dependence_props::compute_dependencies_by_loading( branch if ( matrix.contains( node_id_direction ) ) { dependencies_by_loading[ loop_head ].bodies.insert( node_id_direction ); + dependencies_by_loading[ loop_head ].end_direction = loop_head_direction; } } } @@ -888,6 +871,7 @@ void fuzzing::iid_node_dependence_props::compute_dependencies_by_loops( const lo if ( matrix.contains( node_id_direction ) ) dependencies_by_loops[ loop_head ].bodies.insert( node_id_direction ); + dependencies_by_loops[ loop_head ].end_direction = loop_head_end_direction; } } } From 902c833162169a8b66ecb658946b25e846abaa04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Tue, 21 Jan 2025 15:12:44 +0100 Subject: [PATCH 083/144] feat: update output statistics in benchmark JSON files for improved analysis accuracy --- benchmarks/iid_testing/iid_twice.json | 2 +- benchmarks/iid_testing/input_cycle.json | 7 +------ benchmarks/iid_testing/input_cycle_dec_b.json | 7 +------ benchmarks/iid_testing/lots_of_non_iid_conditions.json | 9 ++------- benchmarks/iid_testing/more_iid_condition.json | 4 ++-- benchmarks/iid_testing/multiplication_in_iid.json | 2 +- benchmarks/iid_testing/same_loop_dependent_2.json | 2 +- benchmarks/iid_testing/same_loop_independent.json | 2 +- 8 files changed, 10 insertions(+), 25 deletions(-) diff --git a/benchmarks/iid_testing/iid_twice.json b/benchmarks/iid_testing/iid_twice.json index b3ce5c66..56b6faba 100644 --- a/benchmarks/iid_testing/iid_twice.json +++ b/benchmarks/iid_testing/iid_twice.json @@ -43,7 +43,7 @@ ], "output_statistics": { "bitshare_analysis": { - "num_generated_tests": 1, + "num_generated_tests": 2, "num_crashes": 0, "num_boundary_violations": 0 }, diff --git a/benchmarks/iid_testing/input_cycle.json b/benchmarks/iid_testing/input_cycle.json index 30f8783f..b570da02 100644 --- a/benchmarks/iid_testing/input_cycle.json +++ b/benchmarks/iid_testing/input_cycle.json @@ -40,13 +40,8 @@ 0 ], "output_statistics": { - "bitshare_analysis": { - "num_generated_tests": 1, - "num_crashes": 0, - "num_boundary_violations": 0 - }, "sensitivity_analysis": { - "num_generated_tests": 4, + "num_generated_tests": 6, "num_crashes": 0, "num_boundary_violations": 0 }, diff --git a/benchmarks/iid_testing/input_cycle_dec_b.json b/benchmarks/iid_testing/input_cycle_dec_b.json index 7e6b4e41..9738ca0a 100644 --- a/benchmarks/iid_testing/input_cycle_dec_b.json +++ b/benchmarks/iid_testing/input_cycle_dec_b.json @@ -42,13 +42,8 @@ 0 ], "output_statistics": { - "bitshare_analysis": { - "num_generated_tests": 1, - "num_crashes": 0, - "num_boundary_violations": 0 - }, "sensitivity_analysis": { - "num_generated_tests": 4, + "num_generated_tests": 7, "num_crashes": 0, "num_boundary_violations": 0 }, diff --git a/benchmarks/iid_testing/lots_of_non_iid_conditions.json b/benchmarks/iid_testing/lots_of_non_iid_conditions.json index e82fd758..be0942cf 100644 --- a/benchmarks/iid_testing/lots_of_non_iid_conditions.json +++ b/benchmarks/iid_testing/lots_of_non_iid_conditions.json @@ -50,18 +50,13 @@ 0 ], "output_statistics": { - "bitshare_analysis": { - "num_generated_tests": 1, - "num_crashes": 0, - "num_boundary_violations": 0 - }, "sensitivity_analysis": { - "num_generated_tests": 4, + "num_generated_tests": 3, "num_crashes": 0, "num_boundary_violations": 0 }, "typed_minimization_analysis": { - "num_generated_tests": 2, + "num_generated_tests": 10, "num_crashes": 0, "num_boundary_violations": 0 }, diff --git a/benchmarks/iid_testing/more_iid_condition.json b/benchmarks/iid_testing/more_iid_condition.json index 3b4aa87b..e8d50842 100644 --- a/benchmarks/iid_testing/more_iid_condition.json +++ b/benchmarks/iid_testing/more_iid_condition.json @@ -41,7 +41,7 @@ ], "output_statistics": { "bitshare_analysis": { - "num_generated_tests": 1, + "num_generated_tests": 3, "num_crashes": 0, "num_boundary_violations": 0 }, @@ -51,7 +51,7 @@ "num_boundary_violations": 0 }, "typed_minimization_analysis": { - "num_generated_tests": 2, + "num_generated_tests": 1, "num_crashes": 0, "num_boundary_violations": 0 }, diff --git a/benchmarks/iid_testing/multiplication_in_iid.json b/benchmarks/iid_testing/multiplication_in_iid.json index a65fc3b0..21311f2a 100644 --- a/benchmarks/iid_testing/multiplication_in_iid.json +++ b/benchmarks/iid_testing/multiplication_in_iid.json @@ -47,7 +47,7 @@ "num_boundary_violations": 0 }, "typed_minimization_analysis": { - "num_generated_tests": 2, + "num_generated_tests": 1, "num_crashes": 0, "num_boundary_violations": 0 }, diff --git a/benchmarks/iid_testing/same_loop_dependent_2.json b/benchmarks/iid_testing/same_loop_dependent_2.json index 6649b7a3..08a0afd8 100644 --- a/benchmarks/iid_testing/same_loop_dependent_2.json +++ b/benchmarks/iid_testing/same_loop_dependent_2.json @@ -54,7 +54,7 @@ "num_boundary_violations": 0 }, "typed_minimization_analysis": { - "num_generated_tests": 2, + "num_generated_tests": 4, "num_crashes": 0, "num_boundary_violations": 0 }, diff --git a/benchmarks/iid_testing/same_loop_independent.json b/benchmarks/iid_testing/same_loop_independent.json index 61be981e..4d67f76c 100644 --- a/benchmarks/iid_testing/same_loop_independent.json +++ b/benchmarks/iid_testing/same_loop_independent.json @@ -55,7 +55,7 @@ "num_boundary_violations": 0 }, "typed_minimization_analysis": { - "num_generated_tests": 2, + "num_generated_tests": 4, "num_crashes": 0, "num_boundary_violations": 0 }, From bd56a7969eca80f1d62cf153773db70ad1ce1a66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Wed, 22 Jan 2025 10:49:50 +0100 Subject: [PATCH 084/144] feat: add double scalar multiplication and update vector handling in iid_vector_analysis - use branching predicate --- .../include/fuzzing/iid_vector_analysis.hpp | 17 ++- src/fuzzing/src/iid_vector_analysis.cpp | 124 +++++++++++++----- 2 files changed, 99 insertions(+), 42 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index 2aa1227e..53e050a4 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -93,6 +93,7 @@ struct equation { equation operator+( const equation& other ) const; equation operator-( const equation& other ) const; equation operator*( int scalar ) const; + equation operator*( double scalar ) const; equation operator/( const equation& other ) const; auto operator<=>( const equation& other ) const = default; bool operator==( const equation& other ) const = default; @@ -128,6 +129,7 @@ struct loop_dependencies_props { std::vector< node_counts > previous_counts; }; + using loop_ending_to_bodies = std::map< location_id, loop_dependencies_props >; using loop_endings = std::map< location_id, bool >; using loop_head_to_bodies_t = std::unordered_map< location_id, std::unordered_set< location_id > >; @@ -142,13 +144,14 @@ struct equation_matrix { std::pair< std::size_t, std::size_t > get_dimensions() const; std::map< equation, int > compute_vectors(); std::vector< equation >& get_matrix(); - std::optional< equation > get_new_path_from_vector( const equation& vector ); + std::optional< equation > get_new_path_from_vector( const std::vector< equation >& vector ); int get_desired_vector_direction() const; float get_biggest_branching_value() const; void print_matrix(); BRANCHING_PREDICATE get_branching_predicate(); + private: void recompute_matrix(); @@ -167,13 +170,15 @@ struct iid_node_dependence_props { private: int compute_path_counts_for_nested_loops( std::map< location_id, int >& counts, int minimum_count ); nodes_to_counts compute_path_counts( const equation& path, std::set< node_direction > const& all_leafs ); - equation get_best_vector( const std::map< equation, int >& vectors_with_hits, - bool use_random, - int desired_direction, - float biggest_branching_value ); + std::vector< equation > get_best_vectors( const std::map< equation, int >& vectors_with_hits, + int number_of_vectors, + bool use_random, + int desired_direction, + float biggest_branching_value ); std::map< equation, int > get_linear_dependent_vector( const std::map< equation, int >& vectors_with_hits, equation& best_vector ); - equation get_random_vector( const std::map< equation, int >& vectors_with_hits ); + std::vector< equation > get_random_vector( const std::map< equation, int >& vectors_with_hits, + int number_of_vectors ); std::set< node_direction > get_leaf_subsets(); loop_endings get_loop_heads_ending( branching_node* end_node, loop_head_to_bodies_t& loop_heads_to_bodies ); void compute_dependencies_by_loading( branching_node* end_node, diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index dea60ff7..438405f9 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -140,6 +140,17 @@ fuzzing::equation fuzzing::equation::operator*( int scalar ) const return { new_values, best_value * scalar }; } +// ------------------------------------------------------------------------------------------------ +fuzzing::equation fuzzing::equation::operator*( double scalar ) const +{ + std::vector< int > new_values; + for ( int i = 0; i < values.size(); ++i ) { + new_values.push_back( values[ i ] * scalar ); + } + + return { new_values, best_value * scalar }; +} + // ------------------------------------------------------------------------------------------------ fuzzing::equation fuzzing::equation::operator/( const equation& other ) const { @@ -348,13 +359,18 @@ float fuzzing::equation_matrix::get_biggest_branching_value() const } // ------------------------------------------------------------------------------------------------ -std::optional< fuzzing::equation > fuzzing::equation_matrix::get_new_path_from_vector( const equation& vector ) +std::optional< fuzzing::equation > +fuzzing::equation_matrix::get_new_path_from_vector( const std::vector< equation >& vectors ) { - INVARIANT( vector.values.size() == nodes.size() ); + INVARIANT( !vectors.empty() ); + INVARIANT( vectors[ 0 ].values.size() == nodes.size() ); + std::vector< equation > paths; std::pair< std::size_t, std::size_t > dimensions = get_dimensions(); + bool get_precise = true; + bool is_equal_sign = get_branching_predicate() == BRANCHING_PREDICATE::BP_EQUAL; auto add_if_positive = [ &paths ]( const equation& new_path ) { if ( !new_path.is_any_negative() ) { @@ -362,21 +378,28 @@ std::optional< fuzzing::equation > fuzzing::equation_matrix::get_new_path_from_v } }; - for ( const auto& row : matrix ) { - double counts = std::abs( row.best_value ) / vector.best_value; - int rounded_counts = static_cast< int >( std::round( counts ) ); + for ( const auto& vector : vectors ) { + for ( const auto& row : matrix ) { + double counts = std::abs( row.best_value ) / std::abs( vector.best_value ); + int rounded_counts = static_cast< int >( std::round( counts ) ); + + if ( get_precise ) { + if ( std::abs( counts - double( rounded_counts ) ) > 1e-6 ) { + continue; + } + + equation new_path = vector * rounded_counts + row; + if ( !is_equal_sign ) { + new_path = new_path * 1.25; + } - if ( get_precise ) { - if ( std::abs( counts - double( rounded_counts ) ) > 1e-6 ) { - continue; - } - equation new_path = vector * rounded_counts + row; - add_if_positive( new_path ); - } else { - for ( int vector_increment = 1; vector_increment <= rounded_counts; ++vector_increment ) { - equation new_path = ( vector * vector_increment ) + row; add_if_positive( new_path ); + } else { + for ( int vector_increment = 1; vector_increment <= rounded_counts; ++vector_increment ) { + equation new_path = ( vector * vector_increment ) + row; + add_if_positive( new_path ); + } } } } @@ -413,7 +436,7 @@ void fuzzing::equation_matrix::print_matrix() // ------------------------------------------------------------------------------------------------ -BRANCHING_PREDICATE fuzzing::equation_matrix::get_branching_predicate() +BRANCHING_PREDICATE fuzzing::equation_matrix::get_branching_predicate() { ASSUMPTION( all_paths.size() > 0 ); return all_paths[ 0 ]->branching_predicate; @@ -447,13 +470,27 @@ fuzzing::possible_path fuzzing::iid_node_dependence_props::generate_probabilitie int desired_vector_direction = submatrix.get_desired_vector_direction(); float biggest_branching_value = submatrix.get_biggest_branching_value(); - equation best_vector = get_best_vector( vectors, false, desired_vector_direction, biggest_branching_value ); + std::vector< equation > best_vectors = + get_best_vectors( vectors, 1, false, desired_vector_direction, biggest_branching_value ); - std::optional< equation > new_path = submatrix.get_new_path_from_vector( best_vector ); + // std::cout << "Best vectors: " << std::endl; + // for (const auto& vector : best_vectors) { + // for (const auto& value : vector.values) { + // std::cout << value << " "; + // } + // std::cout << "-> | " << vector.best_value << std::endl; + // } + std::optional< equation > new_path = submatrix.get_new_path_from_vector( best_vectors ); if ( !new_path.has_value() ) { return {}; } + // std::cout << "New path: "; + // for (const auto& value : new_path->values) { + // std::cout << value << " "; + // } + // std::cout << "-> | " << new_path->best_value << std::endl; + nodes_to_counts path_counts = compute_path_counts( new_path.value(), all_leafs ); possible_path path = generate_path_from_node_counts( path_counts ); @@ -492,7 +529,8 @@ void fuzzing::iid_node_dependence_props::print_dependencies() const } // ------------------------------------------------------------------------------------------------ -int fuzzing::iid_node_dependence_props::compute_path_counts_for_nested_loops( std::map< location_id, int >& counts, int minimum_count ) +int fuzzing::iid_node_dependence_props::compute_path_counts_for_nested_loops( std::map< location_id, int >& counts, + int minimum_count ) { int previous_count = 1; @@ -632,10 +670,12 @@ fuzzing::iid_node_dependence_props::compute_path_counts( const equation& path, } // ------------------------------------------------------------------------------------------------ -fuzzing::equation fuzzing::iid_node_dependence_props::get_best_vector( const std::map< equation, int >& vectors_with_hits, - bool use_random, - int desired_direction, - float biggest_branching_value ) +std::vector< fuzzing::equation > +fuzzing::iid_node_dependence_props::get_best_vectors( const std::map< equation, int >& vectors_with_hits, + int number_of_vectors, + bool use_random, + int desired_direction, + float biggest_branching_value ) { if ( vectors_with_hits.empty() ) { throw std::invalid_argument( "Input map is empty." ); @@ -666,19 +706,24 @@ fuzzing::equation fuzzing::iid_node_dependence_props::get_best_vector( const std } if ( use_random ) { - return get_random_vector( filtered_vectors_with_hits ); + return get_random_vector( filtered_vectors_with_hits, number_of_vectors ); } - auto max_it = std::max_element( filtered_vectors_with_hits.begin(), - filtered_vectors_with_hits.end(), - []( const auto& a, const auto& b ) { - if ( a.second == b.second ) { - return std::abs( a.first.best_value ) > std::abs( b.first.best_value ); - } - return a.second < b.second; - } ); + std::vector< std::pair< equation, int > > sorted_vectors( filtered_vectors_with_hits.begin(), + filtered_vectors_with_hits.end() ); + + std::sort( sorted_vectors.begin(), sorted_vectors.end(), []( const auto& a, const auto& b ) { + if ( a.second == b.second ) { + return std::abs( a.first.best_value ) > std::abs( b.first.best_value ); + } + return a.second > b.second; + } ); + + std::vector< equation > best_vectors; + for ( int i = 0; i < number_of_vectors && i < sorted_vectors.size(); ++i ) { + best_vectors.push_back( sorted_vectors[ i ].first ); + } - equation best_vector = max_it->first; // std::map< equation, int > dependent_vectors_with_hits = // get_linear_dependent_vector( filtered_vectors_with_hits, best_vector ); @@ -694,7 +739,7 @@ fuzzing::equation fuzzing::iid_node_dependence_props::get_best_vector( const std // best_vector = min_it->first; // } - return best_vector; + return best_vectors; } // ------------------------------------------------------------------------------------------------ @@ -715,7 +760,9 @@ std::map< fuzzing::equation, int > fuzzing::iid_node_dependence_props::get_linea } // ------------------------------------------------------------------------------------------------ -fuzzing::equation fuzzing::iid_node_dependence_props::get_random_vector( const std::map< equation, int >& vectors_with_hits ) +std::vector< fuzzing::equation > +fuzzing::iid_node_dependence_props::get_random_vector( const std::map< equation, int >& vectors_with_hits, + int number_of_vectors ) { std::vector< equation > equations; std::vector< double > probabilities; @@ -738,7 +785,12 @@ fuzzing::equation fuzzing::iid_node_dependence_props::get_random_vector( const s std::mt19937 gen( rd() ); std::discrete_distribution<> dist( probabilities.begin(), probabilities.end() ); - return equations[ dist( gen ) ]; + std::vector< equation > selected_equations; + for ( int i = 0; i < number_of_vectors; ++i ) { + selected_equations.push_back( equations[ dist( gen ) ] ); + } + + return selected_equations; } // ------------------------------------------------------------------------------------------------ @@ -871,7 +923,7 @@ void fuzzing::iid_node_dependence_props::compute_dependencies_by_loops( const lo if ( matrix.contains( node_id_direction ) ) dependencies_by_loops[ loop_head ].bodies.insert( node_id_direction ); - dependencies_by_loops[ loop_head ].end_direction = loop_head_end_direction; + dependencies_by_loops[ loop_head ].end_direction = loop_head_end_direction; } } } From 066634b73643d4bd05966438b50e7a0a9e710d6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Wed, 22 Jan 2025 10:50:01 +0100 Subject: [PATCH 085/144] feat: add new benchmarks for different predicates in IID testing --- .../iid_testing/different_predicates_in_iid.c | 85 +++++++++++++++++++ .../different_predicates_in_iid.json | 85 +++++++++++++++++++ .../iid_testing/iid_reverse_predicate.c | 35 ++++++++ .../iid_testing/iid_reverse_predicate.json | 61 +++++++++++++ 4 files changed, 266 insertions(+) create mode 100644 benchmarks/iid_testing/different_predicates_in_iid.c create mode 100644 benchmarks/iid_testing/different_predicates_in_iid.json create mode 100644 benchmarks/iid_testing/iid_reverse_predicate.c create mode 100644 benchmarks/iid_testing/iid_reverse_predicate.json diff --git a/benchmarks/iid_testing/different_predicates_in_iid.c b/benchmarks/iid_testing/different_predicates_in_iid.c new file mode 100644 index 00000000..bed5237b --- /dev/null +++ b/benchmarks/iid_testing/different_predicates_in_iid.c @@ -0,0 +1,85 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + ++i; + } + if ( k == 10 ) // ID: 7 + return 1; + + i = 0; + k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 8 + break; + if ( s[ i ] == 'B' ) // ID: 9 + ++k; + ++i; + } + if ( k > 10 ) // ID: 10 + return 1; + + i = 0; + k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 11 + break; + if ( s[ i ] == 'C' ) // ID: 12 + ++k; + ++i; + } + if ( k >= 10 ) // ID: 13 + return 1; + + i = 0; + k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 14 + break; + if ( s[ i ] == 'D' ) // ID: 15 + ++k; + ++i; + } + // if ( 10 > k ) // ID: 16 + if ( 10 < k ) // ID: 16 + return 1; + + i = 0; + k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 17 + break; + if ( s[ i ] == 'E' ) // ID: 18 + ++k; + ++i; + } + if ( 10 <= k ) // ID: 19 + return 1; + + return 0; + } +} diff --git a/benchmarks/iid_testing/different_predicates_in_iid.json b/benchmarks/iid_testing/different_predicates_in_iid.json new file mode 100644 index 00000000..45b43ccf --- /dev/null +++ b/benchmarks/iid_testing/different_predicates_in_iid.json @@ -0,0 +1,85 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "num_executions_original": 7000, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 58926, + "num_covered_branchings": 19, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0, + 11, + 0, + 12, + 0, + 13, + 0, + 14, + 0, + 15, + 0, + 16, + 0, + 17, + 0, + 18, + 0, + 19, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 5, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 5, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/iid_reverse_predicate.c b/benchmarks/iid_testing/iid_reverse_predicate.c new file mode 100644 index 00000000..d196bddb --- /dev/null +++ b/benchmarks/iid_testing/iid_reverse_predicate.c @@ -0,0 +1,35 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + ++i; + } + if ( 10 < k ) // ID: 7 + return 1; + return 0; + } +} diff --git a/benchmarks/iid_testing/iid_reverse_predicate.json b/benchmarks/iid_testing/iid_reverse_predicate.json new file mode 100644 index 00000000..6077caf1 --- /dev/null +++ b/benchmarks/iid_testing/iid_reverse_predicate.json @@ -0,0 +1,61 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 4108, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file From 7445203f21f50fa4537307534a744667e874a57f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Wed, 22 Jan 2025 19:23:18 +0100 Subject: [PATCH 086/144] refactor: rename loop_head_direction to loop_head_end_direction and update related logic for clarity --- .../include/fuzzing/iid_vector_analysis.hpp | 11 +-- src/fuzzing/src/iid_vector_analysis.cpp | 81 +++++++++++-------- 2 files changed, 52 insertions(+), 40 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index 53e050a4..71a7c3f4 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -21,11 +21,11 @@ struct node_counts { struct path_node_props { - path_node_props( node_counts computed_counts, bool is_loop_head, bool loop_head_direction ) + path_node_props( node_counts computed_counts, bool is_loop_head, bool loop_head_end_direction ) : computed_counts( computed_counts ) , taken_counts( { 0, 0 } ) , is_loop_head( is_loop_head ) - , loop_head_direction( loop_head_direction ) + , loop_head_end_direction( loop_head_end_direction ) {} bool get_desired_direction() const; @@ -40,7 +40,7 @@ struct path_node_props { os << "L: (" << eq.computed_counts.left_count << " | " << eq.taken_counts.left_count << ") "; os << "R: (" << eq.computed_counts.right_count << " | " << eq.taken_counts.right_count << ") "; if ( eq.is_loop_head ) { - os << "Loop head: " << ( eq.loop_head_direction ? "R" : "L" ); + os << "Loop head: " << ( eq.loop_head_end_direction ? "R" : "L" ); } return os; } @@ -49,7 +49,7 @@ struct path_node_props { node_counts computed_counts; node_counts taken_counts; bool is_loop_head; - bool loop_head_direction; + bool loop_head_end_direction; bool get_preferred_direction_loop_head() const; }; @@ -98,7 +98,8 @@ struct equation { auto operator<=>( const equation& other ) const = default; bool operator==( const equation& other ) const = default; - int get_vector_length() const; + int get_vector_size() const; + int get_one_way_branching_count() const; bool is_any_negative() const; bool same_values() const; diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 438405f9..d1f69fd5 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -19,24 +19,24 @@ bool fuzzing::path_node_props::get_desired_direction() const return get_preferred_direction_loop_head(); } - if ( taken_counts.left_count < computed_counts.left_count ) { - return false; - } + bool can_go_left = taken_counts.left_count < computed_counts.left_count; + bool can_go_right = taken_counts.right_count < computed_counts.right_count; - if ( taken_counts.right_count < computed_counts.right_count ) { - return true; + if ( can_go_left && can_go_right ) { + return rand() % 2 == 0; } - // TODO: Should this be here? - float total_count = computed_counts.left_count + computed_counts.right_count; - float left_probability = static_cast< float >( computed_counts.left_count ) / total_count; - float random_value = static_cast< float >( rand() ) / static_cast< float >( RAND_MAX ); - - if ( random_value < left_probability ) { + if ( can_go_left ) { return false; - } else { + } + + if ( can_go_right ) { return true; } + + std::cout << "Error: get_desired_direction() called without any possible direction" << std::endl; + std::cout << *this << std::endl; + return false; } // ------------------------------------------------------------------------------------------------ @@ -58,8 +58,10 @@ void fuzzing::path_node_props::go_direction( bool direction ) taken_counts.left_count++; } - // TODO: Maybe taken count should be reset after the loop is finished, in case this loop is visited again - // later in the code. + // Reset the counts if the loop ended + if ( is_loop_head && direction == loop_head_end_direction ) { + taken_counts = { 0, 0 }; + } } // ------------------------------------------------------------------------------------------------ @@ -83,7 +85,7 @@ bool fuzzing::path_node_props::get_preferred_direction_loop_head() const { auto is_depleted = []( int computed, int taken ) { return computed == taken; }; - if ( !loop_head_direction ) { + if ( !loop_head_end_direction ) { INVARIANT( computed_counts.left_count == 1 ); return !is_depleted( computed_counts.right_count, taken_counts.right_count ); } else { @@ -170,9 +172,15 @@ fuzzing::equation fuzzing::equation::operator/( const equation& other ) const } // ------------------------------------------------------------------------------------------------ -int fuzzing::equation::get_vector_length() const +int fuzzing::equation::get_vector_size() const { - return std::sqrt( std::inner_product( values.begin(), values.end(), values.begin(), 0 ) ); + return std::accumulate( values.begin(), values.end(), 0, []( int sum, int val ) { return sum + val; } ); +} + +// ------------------------------------------------------------------------------------------------ +int fuzzing::equation::get_one_way_branching_count() const +{ + return std::count_if( values.begin(), values.end(), []( int val ) { return val == 0; } ); } // ------------------------------------------------------------------------------------------------ @@ -296,11 +304,10 @@ std::pair< std::size_t, std::size_t > fuzzing::equation_matrix::get_dimensions() // ------------------------------------------------------------------------------------------------ std::map< fuzzing::equation, int > fuzzing::equation_matrix::compute_vectors() { - std::pair< std::size_t, std::size_t > dimensions = get_dimensions(); std::map< equation, int > vectors_with_hits; - for ( int i = 0; i < dimensions.first; ++i ) { - for ( int j = 0; j < dimensions.first; ++j ) { + for ( int i = 0; i < matrix.size(); ++i ) { + for ( int j = 0; j < matrix.size(); ++j ) { if ( i == j ) continue; @@ -365,9 +372,7 @@ fuzzing::equation_matrix::get_new_path_from_vector( const std::vector< equation INVARIANT( !vectors.empty() ); INVARIANT( vectors[ 0 ].values.size() == nodes.size() ); - std::vector< equation > paths; - std::pair< std::size_t, std::size_t > dimensions = get_dimensions(); bool get_precise = true; bool is_equal_sign = get_branching_predicate() == BRANCHING_PREDICATE::BP_EQUAL; @@ -408,13 +413,19 @@ fuzzing::equation_matrix::get_new_path_from_vector( const std::vector< equation return std::nullopt; } - auto min_it = std::min_element( paths.begin(), paths.end(), []( const equation& a, const equation& b ) { - if ( std::abs( a.best_value ) == std::abs( b.best_value ) ) { - return a.get_vector_length() < b.get_vector_length(); + auto compare_equations = []( const equation& a, const equation& b ) { + if ( std::abs( a.best_value ) != std::abs( b.best_value ) ) { + return std::abs( a.best_value ) < std::abs( b.best_value ); } - return std::abs( a.best_value ) < std::abs( b.best_value ); - } ); + if ( a.get_vector_size() != b.get_vector_size() ) { + return a.get_vector_size() < b.get_vector_size(); + } + return a.get_one_way_branching_count() > b.get_one_way_branching_count(); + }; + + auto min_it = std::min_element( paths.begin(), paths.end(), compare_equations ); + INVARIANT( min_it != paths.end() ); return *min_it; } @@ -724,7 +735,6 @@ fuzzing::iid_node_dependence_props::get_best_vectors( const std::map< equation, best_vectors.push_back( sorted_vectors[ i ].first ); } - // std::map< equation, int > dependent_vectors_with_hits = // get_linear_dependent_vector( filtered_vectors_with_hits, best_vector ); @@ -880,7 +890,7 @@ void fuzzing::iid_node_dependence_props::compute_dependencies_by_loading( branch continue; } - bool loop_head_direction = loop_heads_ending.at( loop_head ); + bool loop_head_end_direction = loop_heads_ending.at( loop_head ); const auto& [ min, max ] = values; @@ -896,7 +906,7 @@ void fuzzing::iid_node_dependence_props::compute_dependencies_by_loading( branch if ( matrix.contains( node_id_direction ) ) { dependencies_by_loading[ loop_head ].bodies.insert( node_id_direction ); - dependencies_by_loading[ loop_head ].end_direction = loop_head_direction; + dependencies_by_loading[ loop_head ].end_direction = loop_head_end_direction; } } } @@ -921,9 +931,10 @@ void fuzzing::iid_node_dependence_props::compute_dependencies_by_loops( const lo for ( bool direction : { true, false } ) { node_direction node_id_direction = { body, direction }; - if ( matrix.contains( node_id_direction ) ) + if ( matrix.contains( node_id_direction ) ) { dependencies_by_loops[ loop_head ].bodies.insert( node_id_direction ); - dependencies_by_loops[ loop_head ].end_direction = loop_head_end_direction; + dependencies_by_loops[ loop_head ].end_direction = loop_head_end_direction; + } } } } @@ -939,19 +950,19 @@ fuzzing::iid_node_dependence_props::generate_path_from_node_counts( const nodes_ continue; } - bool loop_head_direction = false; + bool loop_head_end_direction = false; bool is_loop_head = false; for ( const auto& map : { dependencies_by_loops, dependencies_by_loading } ) { auto it = map.find( id ); if ( it != map.end() ) { is_loop_head = true; - loop_head_direction = it->second.end_direction; + loop_head_end_direction = it->second.end_direction; break; } } - path_node_props props = { counts, is_loop_head, loop_head_direction }; + path_node_props props = { counts, is_loop_head, loop_head_end_direction }; path.emplace( id.id, props ); } From bbd157fd661e8ed07c9654ab0b4cec59992f4281 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Wed, 22 Jan 2025 20:13:20 +0100 Subject: [PATCH 087/144] refactor: renaming functions --- .../include/fuzzing/iid_vector_analysis.hpp | 14 ++++---- src/fuzzing/src/iid_vector_analysis.cpp | 34 +++++++++++-------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index 71a7c3f4..857e8ee8 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -143,9 +143,9 @@ struct equation_matrix { void add_equation( branching_node* end_node ); bool contains( node_direction const& node ) const; std::pair< std::size_t, std::size_t > get_dimensions() const; - std::map< equation, int > compute_vectors(); + std::map< equation, int > compute_vectors_with_hits(); std::vector< equation >& get_matrix(); - std::optional< equation > get_new_path_from_vector( const std::vector< equation >& vector ); + std::optional< equation > get_new_leaf_counts_from_vectors( const std::vector< equation >& vector ); int get_desired_vector_direction() const; float get_biggest_branching_value() const; @@ -171,11 +171,11 @@ struct iid_node_dependence_props { private: int compute_path_counts_for_nested_loops( std::map< location_id, int >& counts, int minimum_count ); nodes_to_counts compute_path_counts( const equation& path, std::set< node_direction > const& all_leafs ); - std::vector< equation > get_best_vectors( const std::map< equation, int >& vectors_with_hits, - int number_of_vectors, - bool use_random, - int desired_direction, - float biggest_branching_value ); + std::vector< equation > compute_best_vectors( const std::map< equation, int >& vectors_with_hits, + int number_of_vectors, + bool use_random, + int desired_direction, + float biggest_branching_value ); std::map< equation, int > get_linear_dependent_vector( const std::map< equation, int >& vectors_with_hits, equation& best_vector ); std::vector< equation > get_random_vector( const std::map< equation, int >& vectors_with_hits, diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index d1f69fd5..41ac5337 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -302,7 +302,7 @@ std::pair< std::size_t, std::size_t > fuzzing::equation_matrix::get_dimensions() } // ------------------------------------------------------------------------------------------------ -std::map< fuzzing::equation, int > fuzzing::equation_matrix::compute_vectors() +std::map< fuzzing::equation, int > fuzzing::equation_matrix::compute_vectors_with_hits() { std::map< equation, int > vectors_with_hits; @@ -367,7 +367,7 @@ float fuzzing::equation_matrix::get_biggest_branching_value() const // ------------------------------------------------------------------------------------------------ std::optional< fuzzing::equation > -fuzzing::equation_matrix::get_new_path_from_vector( const std::vector< equation >& vectors ) +fuzzing::equation_matrix::get_new_leaf_counts_from_vectors( const std::vector< equation >& vectors ) { INVARIANT( !vectors.empty() ); INVARIANT( vectors[ 0 ].values.size() == nodes.size() ); @@ -474,7 +474,7 @@ fuzzing::possible_path fuzzing::iid_node_dependence_props::generate_probabilitie std::set< node_direction > all_leafs = get_leaf_subsets(); equation_matrix submatrix = matrix.get_submatrix( all_leafs, true ); - std::map< equation, int > vectors = submatrix.compute_vectors(); + std::map< equation, int > vectors = submatrix.compute_vectors_with_hits(); if ( vectors.empty() ) { return {}; } @@ -482,7 +482,7 @@ fuzzing::possible_path fuzzing::iid_node_dependence_props::generate_probabilitie int desired_vector_direction = submatrix.get_desired_vector_direction(); float biggest_branching_value = submatrix.get_biggest_branching_value(); std::vector< equation > best_vectors = - get_best_vectors( vectors, 1, false, desired_vector_direction, biggest_branching_value ); + compute_best_vectors( vectors, 1, false, desired_vector_direction, biggest_branching_value ); // std::cout << "Best vectors: " << std::endl; // for (const auto& vector : best_vectors) { @@ -491,19 +491,23 @@ fuzzing::possible_path fuzzing::iid_node_dependence_props::generate_probabilitie // } // std::cout << "-> | " << vector.best_value << std::endl; // } - std::optional< equation > new_path = submatrix.get_new_path_from_vector( best_vectors ); - if ( !new_path.has_value() ) { + + std::optional< equation > new_leaf_counts = submatrix.get_new_leaf_counts_from_vectors( best_vectors ); + if ( !new_leaf_counts.has_value() ) { return {}; } // std::cout << "New path: "; - // for (const auto& value : new_path->values) { + // for (const auto& value : new_leaf_counts->values) { // std::cout << value << " "; // } - // std::cout << "-> | " << new_path->best_value << std::endl; + // std::cout << "-> | " << new_leaf_counts->best_value << std::endl; + + nodes_to_counts node_counts = compute_path_counts( new_leaf_counts.value(), all_leafs ); + possible_path path = generate_path_from_node_counts( node_counts ); - nodes_to_counts path_counts = compute_path_counts( new_path.value(), all_leafs ); - possible_path path = generate_path_from_node_counts( path_counts ); + // std::cout << "New path: " << std::endl; + // std::cout << path << std::endl; return path; } @@ -682,11 +686,11 @@ fuzzing::iid_node_dependence_props::compute_path_counts( const equation& path, // ------------------------------------------------------------------------------------------------ std::vector< fuzzing::equation > -fuzzing::iid_node_dependence_props::get_best_vectors( const std::map< equation, int >& vectors_with_hits, - int number_of_vectors, - bool use_random, - int desired_direction, - float biggest_branching_value ) +fuzzing::iid_node_dependence_props::compute_best_vectors( const std::map< equation, int >& vectors_with_hits, + int number_of_vectors, + bool use_random, + int desired_direction, + float biggest_branching_value ) { if ( vectors_with_hits.empty() ) { throw std::invalid_argument( "Input map is empty." ); From 4bd896a590cb843a053272ee950fa3cc57985051 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Thu, 23 Jan 2025 20:10:47 +0100 Subject: [PATCH 088/144] feat: add support for programs with loops inside loops --- .../include/fuzzing/iid_vector_analysis.hpp | 23 +- src/fuzzing/src/fuzzer.cpp | 15 +- src/fuzzing/src/iid_vector_analysis.cpp | 215 +++++++++++------- 3 files changed, 156 insertions(+), 97 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index 857e8ee8..9d5d6914 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -17,6 +17,8 @@ namespace fuzzing struct node_counts { int left_count; int right_count; + + int get_max_count() const; }; @@ -37,11 +39,11 @@ struct path_node_props { friend std::ostream& operator<<( std::ostream& os, const path_node_props& eq ) { - os << "L: (" << eq.computed_counts.left_count << " | " << eq.taken_counts.left_count << ") "; - os << "R: (" << eq.computed_counts.right_count << " | " << eq.taken_counts.right_count << ") "; + os << "L-" << eq.computed_counts.left_count << " R-" << eq.computed_counts.right_count; if ( eq.is_loop_head ) { - os << "Loop head: " << ( eq.loop_head_end_direction ? "R" : "L" ); + os << " " << ( eq.loop_head_end_direction ? "R" : "L" ); } + return os; } @@ -69,8 +71,7 @@ struct possible_path { friend std::ostream& operator<<( std::ostream& os, const possible_path& eq ) { for ( const auto& [ id, props ] : eq.path ) { - os << id << ":" << std::endl; - os << props << std::endl; + os << id << ": " << props << std::endl; } return os; @@ -169,7 +170,17 @@ struct iid_node_dependence_props { void print_dependencies() const; private: - int compute_path_counts_for_nested_loops( std::map< location_id, int >& counts, int minimum_count ); + void compute_path_counts_for_nested_loops( nodes_to_counts& path_counts, + std::map< location_id, int >& child_loop_counts, + location_id loop_head_id, + int minimum_count, + bool use_random = false ); + void compute_path_counts_loading( nodes_to_counts& path_counts, + const equation& path, + const std::set< location_id >& loop_heads ); + void compute_path_counts_loops( nodes_to_counts& path_counts, + const equation& path, + const std::set< location_id >& loop_heads ); nodes_to_counts compute_path_counts( const equation& path, std::set< node_direction > const& all_leafs ); std::vector< equation > compute_best_vectors( const std::map< equation, int >& vectors_with_hits, int number_of_vectors, diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index 98407dcf..e2ca3a54 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -10,6 +10,9 @@ #include #include +constexpr bool use_vector_analysis = true; +constexpr bool use_only_probabilities = false; + namespace fuzzing { @@ -738,10 +741,14 @@ branching_node* fuzzer::monte_carlo_search( ASSUMPTION(start_node != nullptr && !start_node->is_closed()); branching_node* pivot = start_node; + branching_node* successor; while (true) { - // branching_node* const successor{ monte_carlo_step(pivot, histogram, generators, location_miss_generator) }; - branching_node* const successor{ monte_carlo_step_with_path(pivot, histogram, generators, location_miss_generator, path) }; + if ( use_vector_analysis && !use_only_probabilities ) { + successor = monte_carlo_step_with_path( pivot, histogram, generators, location_miss_generator, path ); + } else { + successor = monte_carlo_step( pivot, histogram, generators, location_miss_generator ); + } if (successor == nullptr) break; pivot = successor; @@ -1691,7 +1698,7 @@ branching_node* fuzzer::select_iid_coverage_target() possible_path path; - if ( !iid_locations.empty() ) { + if ( use_vector_analysis && !iid_locations.empty() ) { iid_node_dependence_props& node_props = iid_dependences.get_props( iid_locations[0] ); // std::cout << "IID Location: " << iid_locations[0].id << std::endl; path = node_props.generate_probabilities(); @@ -1745,7 +1752,7 @@ branching_node* fuzzer::select_iid_coverage_target() entry_branching ); - if ( !path.get_path().empty() ) { + if ( use_vector_analysis && !use_only_probabilities && !path.get_path().empty() ) { start_node = select_start_node_for_monte_carlo_search_with_vector( path, it_pivot->second.loop_boundaries, diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 41ac5337..3126c2e9 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -9,6 +9,10 @@ #include #include +// node_counts +// ------------------------------------------------------------------------------------------------ +int fuzzing::node_counts::get_max_count() const { return std::max( left_count, right_count ); } + // path_node_props // ------------------------------------------------------------------------------------------------ bool fuzzing::path_node_props::get_desired_direction() const @@ -474,6 +478,9 @@ fuzzing::possible_path fuzzing::iid_node_dependence_props::generate_probabilitie std::set< node_direction > all_leafs = get_leaf_subsets(); equation_matrix submatrix = matrix.get_submatrix( all_leafs, true ); + + // submatrix.print_matrix(); + std::map< equation, int > vectors = submatrix.compute_vectors_with_hits(); if ( vectors.empty() ) { return {}; @@ -484,6 +491,10 @@ fuzzing::possible_path fuzzing::iid_node_dependence_props::generate_probabilitie std::vector< equation > best_vectors = compute_best_vectors( vectors, 1, false, desired_vector_direction, biggest_branching_value ); + if ( best_vectors.empty() ) { + return {}; + } + // std::cout << "Best vectors: " << std::endl; // for (const auto& vector : best_vectors) { // for (const auto& value : vector.values) { @@ -544,20 +555,29 @@ void fuzzing::iid_node_dependence_props::print_dependencies() const } // ------------------------------------------------------------------------------------------------ -int fuzzing::iid_node_dependence_props::compute_path_counts_for_nested_loops( std::map< location_id, int >& counts, - int minimum_count ) +void fuzzing::iid_node_dependence_props::compute_path_counts_for_nested_loops( nodes_to_counts& path_counts, + std::map< location_id, int >& child_loop_counts, + location_id loop_head_id, + int minimum_count, + bool use_random ) { + INVARIANT( !child_loop_counts.empty() ); + int previous_count = 1; + if ( dependencies_by_loops.at( loop_head_id ).previous_counts.size() > 0 ) { + previous_count = dependencies_by_loops.at( loop_head_id ).previous_counts.back().get_max_count(); + } - int max_count = std::max_element( counts.begin(), counts.end(), []( const auto& a, const auto& b ) { - return a.second < b.second; - } )->second; + int max_child_count = + std::max_element( child_loop_counts.begin(), child_loop_counts.end(), []( const auto& a, const auto& b ) { + return a.second < b.second; + } )->second; - int highest_count = 1; - for ( int i = minimum_count; i <= max_count; ++i ) { + std::set< int > possible_counts; + for ( int i = minimum_count; i <= max_child_count; ++i ) { bool is_good = true; - for ( const auto& [ node_id, count ] : counts ) { + for ( const auto& [ node_id, count ] : child_loop_counts ) { if ( count % i != 0 ) { is_good = false; break; @@ -565,16 +585,87 @@ int fuzzing::iid_node_dependence_props::compute_path_counts_for_nested_loops( st } if ( is_good ) { - highest_count = i; - break; + possible_counts.insert( i ); + } + } + + + if ( possible_counts.empty() ) { + path_counts[ loop_head_id ] = { 1, 1 }; + return; + } + + int highest_count = use_random ? *std::next( possible_counts.begin(), rand() % possible_counts.size() ) : + *possible_counts.rbegin(); + + for ( auto& [ node_id, count ] : child_loop_counts ) { + auto& [ left_count, right_count ] = path_counts[ node_id ]; + + if ( dependencies_by_loops.at( node_id ).end_direction ) { + left_count = count / highest_count; + } else { + right_count = count / highest_count; } } - for ( auto& [ node_id, count ] : counts ) { - count /= highest_count; + if ( dependencies_by_loops.at( loop_head_id ).end_direction ) { + path_counts[ loop_head_id ] = { highest_count, 1 }; + } else { + path_counts[ loop_head_id ] = { 1, highest_count }; } +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::iid_node_dependence_props::compute_path_counts_loading( nodes_to_counts& path_counts, + const equation& path, + const std::set< location_id >& loop_heads ) +{ + // TODO + for ( const auto& [ loading_head, props ] : dependencies_by_loading ) { + int loop_count = 0; + + for ( const auto& body : props.bodies ) { + if ( !path_counts.contains( body.node_id ) || !loop_heads.contains( body.node_id ) ) { + continue; + } + + auto& [ left_count, right_count ] = path_counts[ body.node_id ]; + loop_count = std::max( loop_count, left_count + right_count ); + } + + if ( props.end_direction ) { + path_counts[ loading_head ] = { loop_count, 1 }; + } else { + path_counts[ loading_head ] = { 1, loop_count }; + } + } +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::iid_node_dependence_props::compute_path_counts_loops( nodes_to_counts& path_counts, + const equation& path, + const std::set< location_id >& loop_heads ) +{ + for ( const auto& [ loop_head, props ] : std::ranges::views::reverse( dependencies_by_loops ) ) { + std::map< location_id, int > child_loop_counts; + int non_loop_child_max_count = 1; + + for ( const auto& body : props.bodies ) { + auto& [ left_count, right_count ] = path_counts[ body.node_id ]; + + if ( loop_heads.contains( body.node_id ) ) { + child_loop_counts[ body.node_id ] = std::max( left_count, right_count ); + } else { + non_loop_child_max_count = std::max( non_loop_child_max_count, left_count + right_count ); + } + } - return highest_count; + if ( child_loop_counts.empty() ) { + continue; + } + + compute_path_counts_for_nested_loops( path_counts, child_loop_counts, loop_head, non_loop_child_max_count ); + } } // ------------------------------------------------------------------------------------------------ @@ -585,10 +676,10 @@ fuzzing::iid_node_dependence_props::compute_path_counts( const equation& path, nodes_to_counts path_counts; std::vector< node_direction > leafs = std::vector< node_direction >( all_leafs.begin(), all_leafs.end() ); - std::set< location_id > loop_heads = get_loop_heads( false ); - INVARIANT( leafs.size() == path.values.size() ); + std::set< location_id > loop_heads = get_loop_heads( false ); + for ( int i = 0; i < leafs.size(); ++i ) { auto& [ left_count, right_count ] = path_counts[ leafs[ i ].node_id ]; if ( leafs[ i ].branching_direction ) { @@ -598,89 +689,38 @@ fuzzing::iid_node_dependence_props::compute_path_counts( const equation& path, } } - // First - for ( int i = 0; i < dependencies_by_loops.size(); ++i ) { - for ( const auto& [ loop_head, props ] : dependencies_by_loops ) { - int loop_count = 0; - for ( const auto& body : props.bodies ) { - auto& [ left_count, right_count ] = path_counts[ body.node_id ]; + for ( const auto& [ loop_head, props ] : std::ranges::views::reverse( dependencies_by_loops ) ) { + int loop_count = 0; - if ( loop_heads.contains( body.node_id ) ) { - loop_count = std::max( loop_count, 1 ); - } else { - loop_count = std::max( loop_count, left_count + right_count ); - } - } + for ( const auto& body : props.bodies ) { + auto& [ left_count, right_count ] = path_counts[ body.node_id ]; - if ( props.end_direction ) { - path_counts[ loop_head ] = { loop_count, 1 }; + if ( loop_heads.contains( body.node_id ) ) { + loop_count = std::max( loop_count, 1 ); } else { - path_counts[ loop_head ] = { 1, loop_count }; + loop_count = std::max( loop_count, left_count + right_count ); } } - } - - // Second - // for ( const auto& [ loop_head, props ] : std::ranges::views::reverse( dependencies_by_loops ) ) { - // int non_loop_child_max_count = 1; - // std::map< location_id, int > counts; - - // for ( const auto& body : props.bodies ) { - // auto& [ left_count, right_count ] = path_counts[ body.node_id ]; - // if ( !loop_heads.contains( body.node_id ) ) { - // non_loop_child_max_count = std::max( non_loop_child_max_count, left_count + right_count ); - // } else { - // counts[ body.node_id ] = std::max( left_count, right_count ); - // } - // } - - // int new_count = compute_path_counts_for_nested_loops( counts, non_loop_child_max_count ); - // if ( props.end_direction ) { - // path_counts[ loop_head ] = { new_count, 1 }; - // } else { - // path_counts[ loop_head ] = { 1, new_count }; - // } - - // for ( const auto& [ node_id, count ] : counts ) { - // const loop_dependencies_props& o_props = dependencies_by_loops.at( node_id ); - // if ( o_props.end_direction ) { - // path_counts[ node_id ] = { count, 1 }; - // } else { - // path_counts[ node_id ] = { 1, count }; - // } - // } - // } - - for ( int i = 0; i < dependencies_by_loading.size(); ++i ) { - for ( const auto& [ loading_head, props ] : dependencies_by_loading ) { - int loop_count = 0; - for ( const auto& body : props.bodies ) { - if ( !path_counts.contains( body.node_id ) ) { - continue; - } + if ( props.end_direction ) { + path_counts[ loop_head ] = { loop_count, 1 }; + } else { + path_counts[ loop_head ] = { 1, loop_count }; + } + } - auto& [ left_count, right_count ] = path_counts[ body.node_id ]; - loop_count = std::max( loop_count, left_count + right_count ); - } + compute_path_counts_loops( path_counts, path, loop_heads ); + compute_path_counts_loading( path_counts, path, loop_heads ); - if ( props.end_direction ) { - path_counts[ loading_head ] = { loop_count, 1 }; - } else { - path_counts[ loading_head ] = { 1, loop_count }; + for ( auto* map : { &dependencies_by_loops, &dependencies_by_loading } ) { + for ( auto& [ loop_head, props ] : *map ) { + auto it = path_counts.find( loop_head ); + if ( it != path_counts.end() ) { + props.previous_counts.push_back( it->second ); } } } - // for ( auto* map : { &dependencies_by_loops, &dependencies_by_loading } ) { - // for ( auto& [ loop_head, props ] : *map ) { - // auto it = path_counts.find( loop_head ); - // if ( it != path_counts.end() ) { - // props.previous_counts.push_back( it->second ); - // } - // } - // } - return path_counts; } @@ -717,7 +757,8 @@ fuzzing::iid_node_dependence_props::compute_best_vectors( const std::map< equati } ); if ( filtered_vectors_with_hits.empty() ) { - throw std::invalid_argument( "No vectors match the desired direction." ); + return {}; + // throw std::invalid_argument( "No vectors match the desired direction." ); } if ( use_random ) { From be4cdc8103e7a1737895a40ccf9c8892af488621 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 25 Jan 2025 10:45:00 +0100 Subject: [PATCH 089/144] feat: generate more data after iid node is covered --- .../include/fuzzing/iid_vector_analysis.hpp | 42 ++++- src/fuzzing/src/fuzzer.cpp | 19 +-- src/fuzzing/src/iid_vector_analysis.cpp | 158 ++++++++++++++++-- 3 files changed, 192 insertions(+), 27 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index 9d5d6914..c9eca3ac 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -92,6 +92,7 @@ struct equation { double best_value; equation operator+( const equation& other ) const; + equation operator+( int scalar ) const; equation operator-( const equation& other ) const; equation operator*( int scalar ) const; equation operator*( double scalar ) const; @@ -99,8 +100,10 @@ struct equation { auto operator<=>( const equation& other ) const = default; bool operator==( const equation& other ) const = default; + equation add_to_positive( int value ) const; int get_vector_size() const; int get_one_way_branching_count() const; + int get_biggest_value() const; bool is_any_negative() const; bool same_values() const; @@ -146,7 +149,8 @@ struct equation_matrix { std::pair< std::size_t, std::size_t > get_dimensions() const; std::map< equation, int > compute_vectors_with_hits(); std::vector< equation >& get_matrix(); - std::optional< equation > get_new_leaf_counts_from_vectors( const std::vector< equation >& vector ); + std::optional< equation > get_new_leaf_counts_from_vectors( const std::vector< equation >& vector, + int generated_after_covered ); int get_desired_vector_direction() const; float get_biggest_branching_value() const; @@ -162,14 +166,38 @@ struct equation_matrix { std::set< node_direction > nodes; }; +enum generation_state { + NOT_COVERED, + GENERATION_MORE, + COVERED, + GENERATION_DATA_FOR_NEXT_NODE +}; + +struct iid_node_generations_stats { + int generation_starts = 0; + int successful_generations = 0; + int failed_generations = 0; + + int failed_generations_in_row = 0; + + int generated_after_covered_max = 0; + int generated_after_covered = 0; + + generation_state state = generation_state::NOT_COVERED; +}; struct iid_node_dependence_props { possible_path generate_probabilities(); void process_node( branching_node* end_node ); + iid_node_generations_stats& get_generations_stats() { return stats; } + bool should_generate() const; void print_dependencies() const; + void print_stats() const; private: + possible_path return_empty_path(); + possible_path return_path( const possible_path& path ); void compute_path_counts_for_nested_loops( nodes_to_counts& path_counts, std::map< location_id, int >& child_loop_counts, location_id loop_head_id, @@ -204,6 +232,8 @@ struct iid_node_dependence_props { equation_matrix matrix; loop_ending_to_bodies dependencies_by_loops; loop_ending_to_bodies dependencies_by_loading; + + iid_node_generations_stats stats; }; struct iid_dependencies { @@ -212,10 +242,18 @@ struct iid_dependencies { void remove_node_dependence( location_id id ); iid_node_dependence_props& get_props( location_id id ); std::vector< location_id > get_iid_nodes(); + std::optional< location_id > get_next_iid_node(); private: - std::unordered_map< location_id, iid_node_dependence_props > id_to_equation_map; + std::map< location_id, iid_node_dependence_props > id_to_equation_map; std::set< location_id > non_iid_nodes; + + int max_failed_generations = 10; + int current_failed_generations = 0; + + // Settings + bool generate_more_data_after_coverage = true; + int minimal_max_generation_after_covered = 10; }; std::vector< node_direction > get_path( branching_node* node ); diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index e2ca3a54..703ee692 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -1687,27 +1687,18 @@ branching_node* fuzzer::select_iid_coverage_target() histogram_of_hit_counts_per_direction::hit_counts_map hit_counts; it_pivot->second.histogram_ptr->merge(hit_counts); - - std::vector< location_id > iid_locations = iid_dependences.get_iid_nodes(); - { - // std::cout << "IID Locations:" << std::endl; - // for (const auto& loc : iid_locations) { - // std::cout << loc.id << std::endl; - // } - } - possible_path path; - - if ( use_vector_analysis && !iid_locations.empty() ) { - iid_node_dependence_props& node_props = iid_dependences.get_props( iid_locations[0] ); - // std::cout << "IID Location: " << iid_locations[0].id << std::endl; + std::optional< location_id > iid_location = iid_dependences.get_next_iid_node(); + + if ( use_vector_analysis && iid_location.has_value() ) { + iid_node_dependence_props& node_props = iid_dependences.get_props( *iid_location ); + std::cout << "Computing probabilities for location " << ( *iid_location ).id << std::endl; path = node_props.generate_probabilities(); for ( const auto& path_props : path.get_path() ) { auto it = histogram.find( path_props.first ); if ( it != histogram.end() ) { it->second = path_props.second.get_false_direction_probability(); - // std::cout << "IID Location: " << path_props.first << " Probability: " << it->second << std::endl; } } } diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 3126c2e9..b816c127 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -122,6 +122,17 @@ fuzzing::equation fuzzing::equation::operator+( const equation& other ) const return { new_values, best_value + other.best_value }; } +// ------------------------------------------------------------------------------------------------ +fuzzing::equation fuzzing::equation::operator+( int scalar ) const +{ + std::vector< int > new_values; + for ( int i = 0; i < values.size(); ++i ) { + new_values.push_back( values[ i ] + scalar ); + } + + return { new_values, best_value }; +} + // ------------------------------------------------------------------------------------------------ fuzzing::equation fuzzing::equation::operator-( const equation& other ) const { @@ -175,6 +186,21 @@ fuzzing::equation fuzzing::equation::operator/( const equation& other ) const return { new_values, best_value / other.best_value }; } +// ------------------------------------------------------------------------------------------------ +fuzzing::equation fuzzing::equation::add_to_positive( int value ) const +{ + std::vector< int > new_values; + for ( int i = 0; i < values.size(); ++i ) { + if ( values[ i ] != 0 ) { + new_values.push_back( values[ i ] + value ); + } else { + new_values.push_back( values[ i ] ); + } + } + + return { new_values, best_value }; +} + // ------------------------------------------------------------------------------------------------ int fuzzing::equation::get_vector_size() const { @@ -187,6 +213,9 @@ int fuzzing::equation::get_one_way_branching_count() const return std::count_if( values.begin(), values.end(), []( int val ) { return val == 0; } ); } +// ------------------------------------------------------------------------------------------------ +int fuzzing::equation::get_biggest_value() const { return *std::max_element( values.begin(), values.end() ); } + // ------------------------------------------------------------------------------------------------ bool fuzzing::equation::is_any_negative() const { @@ -371,7 +400,8 @@ float fuzzing::equation_matrix::get_biggest_branching_value() const // ------------------------------------------------------------------------------------------------ std::optional< fuzzing::equation > -fuzzing::equation_matrix::get_new_leaf_counts_from_vectors( const std::vector< equation >& vectors ) +fuzzing::equation_matrix::get_new_leaf_counts_from_vectors( const std::vector< equation >& vectors, + int generation_count_after_covered ) { INVARIANT( !vectors.empty() ); INVARIANT( vectors[ 0 ].values.size() == nodes.size() ); @@ -399,10 +429,11 @@ fuzzing::equation_matrix::get_new_leaf_counts_from_vectors( const std::vector< e equation new_path = vector * rounded_counts + row; if ( !is_equal_sign ) { - new_path = new_path * 1.25; + int limit = new_path.get_biggest_value() / 2; + limit = 1; + new_path = new_path.add_to_positive( 1 + rand() % limit ); } - add_if_positive( new_path ); } else { for ( int vector_increment = 1; vector_increment <= rounded_counts; ++vector_increment ) { @@ -473,9 +504,11 @@ void fuzzing::equation_matrix::recompute_matrix() // ------------------------------------------------------------------------------------------------ fuzzing::possible_path fuzzing::iid_node_dependence_props::generate_probabilities() { + print_stats(); // print_dependencies(); // matrix.print_matrix(); + stats.generation_starts++; std::set< node_direction > all_leafs = get_leaf_subsets(); equation_matrix submatrix = matrix.get_submatrix( all_leafs, true ); @@ -483,7 +516,7 @@ fuzzing::possible_path fuzzing::iid_node_dependence_props::generate_probabilitie std::map< equation, int > vectors = submatrix.compute_vectors_with_hits(); if ( vectors.empty() ) { - return {}; + return return_empty_path(); } int desired_vector_direction = submatrix.get_desired_vector_direction(); @@ -492,7 +525,7 @@ fuzzing::possible_path fuzzing::iid_node_dependence_props::generate_probabilitie compute_best_vectors( vectors, 1, false, desired_vector_direction, biggest_branching_value ); if ( best_vectors.empty() ) { - return {}; + return return_empty_path(); } // std::cout << "Best vectors: " << std::endl; @@ -503,9 +536,10 @@ fuzzing::possible_path fuzzing::iid_node_dependence_props::generate_probabilitie // std::cout << "-> | " << vector.best_value << std::endl; // } - std::optional< equation > new_leaf_counts = submatrix.get_new_leaf_counts_from_vectors( best_vectors ); + std::optional< equation > new_leaf_counts = + submatrix.get_new_leaf_counts_from_vectors( best_vectors, stats.generated_after_covered ); if ( !new_leaf_counts.has_value() ) { - return {}; + return return_empty_path(); } // std::cout << "New path: "; @@ -520,7 +554,7 @@ fuzzing::possible_path fuzzing::iid_node_dependence_props::generate_probabilitie // std::cout << "New path: " << std::endl; // std::cout << path << std::endl; - return path; + return return_path( path ); } // ------------------------------------------------------------------------------------------------ @@ -535,6 +569,12 @@ void fuzzing::iid_node_dependence_props::process_node( branching_node* end_node compute_dependencies_by_loops( loop_heads_to_bodies, loop_heads_ending ); } +// ------------------------------------------------------------------------------------------------ +bool fuzzing::iid_node_dependence_props::should_generate() const +{ + return stats.state == generation_state::NOT_COVERED || stats.state == generation_state::GENERATION_MORE || + stats.state == generation_state::GENERATION_DATA_FOR_NEXT_NODE; +} // ------------------------------------------------------------------------------------------------ void fuzzing::iid_node_dependence_props::print_dependencies() const { @@ -554,6 +594,53 @@ void fuzzing::iid_node_dependence_props::print_dependencies() const } } +// ------------------------------------------------------------------------------------------------ +void fuzzing::iid_node_dependence_props::print_stats() const +{ + switch ( stats.state ) { + case generation_state::NOT_COVERED: + std::cout << "Status: NOT_COVERED" << std::endl; + std::cout << "Failed generations/Total generations: " << stats.failed_generations << "/" + << stats.generation_starts << std::endl; + std::cout << "Failed generations in row: " << stats.failed_generations_in_row << std::endl; + break; + case generation_state::GENERATION_MORE: + std::cout << "Status GENERATION_MORE" << std::endl; + std::cout << "Generated after covered: " << stats.generated_after_covered << "/" + << stats.generated_after_covered_max << std::endl; + break; + case generation_state::COVERED: std::cout << "COVERED" << std::endl; break; + case generation_state::GENERATION_DATA_FOR_NEXT_NODE: + std::cout << "GENERATION_DATA_FOR_NEXT_NODE" << std::endl; + break; + } +} + +// ------------------------------------------------------------------------------------------------ +fuzzing::possible_path fuzzing::iid_node_dependence_props::return_empty_path() +{ + stats.failed_generations++; + stats.failed_generations_in_row++; + return possible_path(); +} + +// ------------------------------------------------------------------------------------------------ +fuzzing::possible_path fuzzing::iid_node_dependence_props::return_path( const possible_path& path ) +{ + stats.failed_generations_in_row = 0; + stats.successful_generations++; + + if ( stats.state == generation_state::GENERATION_MORE ) { + stats.generated_after_covered++; + + if ( stats.generated_after_covered > stats.generated_after_covered_max ) { + stats.state = generation_state::COVERED; + } + } + + return path; +} + // ------------------------------------------------------------------------------------------------ void fuzzing::iid_node_dependence_props::compute_path_counts_for_nested_loops( nodes_to_counts& path_counts, std::map< location_id, int >& child_loop_counts, @@ -1060,9 +1147,17 @@ void fuzzing::iid_dependencies::process_node_dependence( branching_node* node ) // ------------------------------------------------------------------------------------------------ void fuzzing::iid_dependencies::remove_node_dependence( location_id id ) { - if ( id_to_equation_map.contains( id ) ) { - id_to_equation_map.erase( id ); - non_iid_nodes.insert( id ); + auto it = id_to_equation_map.find( id ); + if ( it != id_to_equation_map.end() ) { + iid_node_generations_stats& stats = it->second.get_generations_stats(); + stats.state = generation_state::COVERED; + + if ( generate_more_data_after_coverage ) { + stats.state = generation_state::GENERATION_MORE; + int max_generation_after_covered = std::max( minimal_max_generation_after_covered, + stats.successful_generations / 2 ); + stats.generated_after_covered_max = max_generation_after_covered; + } } } @@ -1084,6 +1179,47 @@ std::vector< location_id > fuzzing::iid_dependencies::get_iid_nodes() return result; } +// ------------------------------------------------------------------------------------------------ +std::optional< location_id > fuzzing::iid_dependencies::get_next_iid_node() +{ + std::optional< location_id > best_id = std::nullopt; + + for ( const auto& [ id, props ] : id_to_equation_map ) { + if ( props.should_generate() ) { + return id; + } + } + + return best_id; + + + // bool previous_failed = false; + + // for ( auto it = id_to_equation_map.rbegin(); it != id_to_equation_map.rend(); ++it ) { + // if ( previous_failed ) { + // previous_failed = false; + // current_failed_generations++; + // return it->first; + // } + + // if ( it->second.get_number_of_failed_generations() >= max_failed_generations ) { + // if ( current_failed_generations >= max_failed_generations ) { + // it->second.get_number_of_failed_generations() = 0; + // current_failed_generations = 0; + // } else { + // previous_failed = true; + // continue; + // } + // } + + // if ( it->second.get_generation_count_after_covered() >= max_generation_count_after_covered ) { + // continue; + // } + + // best_id = it->first; + // } +} + // non member functions // ------------------------------------------------------------------------------------------------ std::vector< fuzzing::node_direction > fuzzing::get_path( branching_node* node ) From 4ed86520e118cb5c00f1396684d9dc3bbac161b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 25 Jan 2025 14:46:14 +0100 Subject: [PATCH 090/144] feat: update generated test counts and execution limits in IID benchmarks --- benchmarks/iid_testing/different_predicates_in_iid.json | 4 ++-- benchmarks/iid_testing/input_cycle_dec_b.json | 4 ++-- benchmarks/iid_testing/more_iid_condition.json | 2 +- benchmarks/iid_testing/more_iid_condition_nested.json | 8 ++++---- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/benchmarks/iid_testing/different_predicates_in_iid.json b/benchmarks/iid_testing/different_predicates_in_iid.json index 45b43ccf..db086d43 100644 --- a/benchmarks/iid_testing/different_predicates_in_iid.json +++ b/benchmarks/iid_testing/different_predicates_in_iid.json @@ -61,12 +61,12 @@ ], "output_statistics": { "bitshare_analysis": { - "num_generated_tests": 5, + "num_generated_tests": 3, "num_crashes": 0, "num_boundary_violations": 0 }, "sensitivity_analysis": { - "num_generated_tests": 4, + "num_generated_tests": 6, "num_crashes": 0, "num_boundary_violations": 0 }, diff --git a/benchmarks/iid_testing/input_cycle_dec_b.json b/benchmarks/iid_testing/input_cycle_dec_b.json index 9738ca0a..e00e68be 100644 --- a/benchmarks/iid_testing/input_cycle_dec_b.json +++ b/benchmarks/iid_testing/input_cycle_dec_b.json @@ -43,12 +43,12 @@ ], "output_statistics": { "sensitivity_analysis": { - "num_generated_tests": 7, + "num_generated_tests": 6, "num_crashes": 0, "num_boundary_violations": 0 }, "typed_minimization_analysis": { - "num_generated_tests": 2, + "num_generated_tests": 3, "num_crashes": 0, "num_boundary_violations": 0 }, diff --git a/benchmarks/iid_testing/more_iid_condition.json b/benchmarks/iid_testing/more_iid_condition.json index e8d50842..55f23df1 100644 --- a/benchmarks/iid_testing/more_iid_condition.json +++ b/benchmarks/iid_testing/more_iid_condition.json @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 4000, + "num_executions": 6900, "num_covered_branchings": 9, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/more_iid_condition_nested.json b/benchmarks/iid_testing/more_iid_condition_nested.json index 109ac279..65795f66 100644 --- a/benchmarks/iid_testing/more_iid_condition_nested.json +++ b/benchmarks/iid_testing/more_iid_condition_nested.json @@ -1,6 +1,6 @@ { "args": { - "max_executions": 10000, + "max_executions": 100000, "max_seconds": 300, "max_trace_length": 10000, "max_stack_size": 25, @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 10000, + "num_executions": 30000, "num_covered_branchings": 9, "covered_branchings": [ 1, @@ -40,7 +40,7 @@ ], "output_statistics": { "bitshare_analysis": { - "num_generated_tests": 1, + "num_generated_tests": 3, "num_crashes": 0, "num_boundary_violations": 0 }, @@ -50,7 +50,7 @@ "num_boundary_violations": 0 }, "typed_minimization_analysis": { - "num_generated_tests": 2, + "num_generated_tests": 1, "num_crashes": 0, "num_boundary_violations": 0 }, From 3c169381fafad6a99798c74e427457600dbe94c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 25 Jan 2025 14:46:42 +0100 Subject: [PATCH 091/144] feat: generate more data from previous iiid nodes if there not enough data --- .../include/fuzzing/iid_vector_analysis.hpp | 24 ++--- src/fuzzing/src/fuzzer.cpp | 2 +- src/fuzzing/src/iid_vector_analysis.cpp | 97 ++++++++++++++----- 3 files changed, 87 insertions(+), 36 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index c9eca3ac..ed2f6ee0 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -156,7 +156,7 @@ struct equation_matrix { void print_matrix(); - BRANCHING_PREDICATE get_branching_predicate(); + BRANCHING_PREDICATE get_branching_predicate() const; private: void recompute_matrix(); @@ -166,23 +166,21 @@ struct equation_matrix { std::set< node_direction > nodes; }; -enum generation_state { - NOT_COVERED, - GENERATION_MORE, - COVERED, - GENERATION_DATA_FOR_NEXT_NODE -}; +enum generation_state { NOT_COVERED, GENERATION_MORE, COVERED, GENERATION_DATA_FOR_NEXT_NODE }; struct iid_node_generations_stats { int generation_starts = 0; int successful_generations = 0; int failed_generations = 0; - + int failed_generations_in_row = 0; int generated_after_covered_max = 0; int generated_after_covered = 0; + int generated_for_other_node = 0; + int generated_for_other_node_max = 0; + generation_state state = generation_state::NOT_COVERED; }; @@ -190,11 +188,16 @@ struct iid_node_dependence_props { possible_path generate_probabilities(); void process_node( branching_node* end_node ); iid_node_generations_stats& get_generations_stats() { return stats; } + bool should_generate() const; + bool needs_data_from_other_node( int max_failed_generations_in_row ) const; + void set_as_generating_for_other_node( int minimal_max_generation_for_other_node ); + bool is_equal_branching_predicate() const; void print_dependencies() const; void print_stats() const; + private: possible_path return_empty_path(); possible_path return_path( const possible_path& path ); @@ -248,12 +251,11 @@ struct iid_dependencies { std::map< location_id, iid_node_dependence_props > id_to_equation_map; std::set< location_id > non_iid_nodes; - int max_failed_generations = 10; - int current_failed_generations = 0; - // Settings bool generate_more_data_after_coverage = true; int minimal_max_generation_after_covered = 10; + int max_failed_generations_in_row = 10; + int minimal_max_generation_for_other_node = 10; }; std::vector< node_direction > get_path( branching_node* node ); diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index 703ee692..fecc2c93 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -1692,7 +1692,7 @@ branching_node* fuzzer::select_iid_coverage_target() if ( use_vector_analysis && iid_location.has_value() ) { iid_node_dependence_props& node_props = iid_dependences.get_props( *iid_location ); - std::cout << "Computing probabilities for location " << ( *iid_location ).id << std::endl; + // std::cout << "Computing probabilities for location " << ( *iid_location ).id << std::endl; path = node_props.generate_probabilities(); for ( const auto& path_props : path.get_path() ) { diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index b816c127..008a3d64 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -408,7 +408,6 @@ fuzzing::equation_matrix::get_new_leaf_counts_from_vectors( const std::vector< e std::vector< equation > paths; - bool get_precise = true; bool is_equal_sign = get_branching_predicate() == BRANCHING_PREDICATE::BP_EQUAL; auto add_if_positive = [ &paths ]( const equation& new_path ) { @@ -422,25 +421,18 @@ fuzzing::equation_matrix::get_new_leaf_counts_from_vectors( const std::vector< e double counts = std::abs( row.best_value ) / std::abs( vector.best_value ); int rounded_counts = static_cast< int >( std::round( counts ) ); - if ( get_precise ) { - if ( std::abs( counts - double( rounded_counts ) ) > 1e-6 ) { - continue; - } - - equation new_path = vector * rounded_counts + row; - if ( !is_equal_sign ) { - int limit = new_path.get_biggest_value() / 2; - limit = 1; - new_path = new_path.add_to_positive( 1 + rand() % limit ); - } + if ( std::abs( counts - double( rounded_counts ) ) > 1e-6 ) { + continue; + } - add_if_positive( new_path ); - } else { - for ( int vector_increment = 1; vector_increment <= rounded_counts; ++vector_increment ) { - equation new_path = ( vector * vector_increment ) + row; - add_if_positive( new_path ); - } + equation new_path = vector * rounded_counts + row; + if ( !is_equal_sign ) { + int limit = std::max( new_path.get_biggest_value() / 4, 1 ); + new_path = new_path.add_to_positive( 1 + rand() % limit ); + // new_path = new_path.add_to_positive( 1 + limit ); } + + add_if_positive( new_path ); } } @@ -482,7 +474,7 @@ void fuzzing::equation_matrix::print_matrix() // ------------------------------------------------------------------------------------------------ -BRANCHING_PREDICATE fuzzing::equation_matrix::get_branching_predicate() +BRANCHING_PREDICATE fuzzing::equation_matrix::get_branching_predicate() const { ASSUMPTION( all_paths.size() > 0 ); return all_paths[ 0 ]->branching_predicate; @@ -504,7 +496,7 @@ void fuzzing::equation_matrix::recompute_matrix() // ------------------------------------------------------------------------------------------------ fuzzing::possible_path fuzzing::iid_node_dependence_props::generate_probabilities() { - print_stats(); + // print_stats(); // print_dependencies(); // matrix.print_matrix(); @@ -575,6 +567,37 @@ bool fuzzing::iid_node_dependence_props::should_generate() const return stats.state == generation_state::NOT_COVERED || stats.state == generation_state::GENERATION_MORE || stats.state == generation_state::GENERATION_DATA_FOR_NEXT_NODE; } + +// ------------------------------------------------------------------------------------------------ +bool fuzzing::iid_node_dependence_props::needs_data_from_other_node( int max_failed_generations_in_row ) const +{ + if ( stats.state != generation_state::NOT_COVERED ) { + return false; + } + + if ( stats.failed_generations_in_row > max_failed_generations_in_row ) { + return true; + } + + return false; +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::iid_node_dependence_props::set_as_generating_for_other_node( int minimal_max_generation_for_other_node ) +{ + INVARIANT( stats.state == generation_state::COVERED ); + + stats.state = generation_state::GENERATION_DATA_FOR_NEXT_NODE; + stats.generated_for_other_node_max = minimal_max_generation_for_other_node; + stats.generated_for_other_node = 0; +} + +// ------------------------------------------------------------------------------------------------ +bool fuzzing::iid_node_dependence_props::is_equal_branching_predicate() const +{ + return matrix.get_branching_predicate() == BRANCHING_PREDICATE::BP_EQUAL; +} + // ------------------------------------------------------------------------------------------------ void fuzzing::iid_node_dependence_props::print_dependencies() const { @@ -611,7 +634,9 @@ void fuzzing::iid_node_dependence_props::print_stats() const break; case generation_state::COVERED: std::cout << "COVERED" << std::endl; break; case generation_state::GENERATION_DATA_FOR_NEXT_NODE: - std::cout << "GENERATION_DATA_FOR_NEXT_NODE" << std::endl; + std::cout << "Status: GENERATION_DATA_FOR_NEXT_NODE" << std::endl; + std::cout << "Generated for other node: " << stats.generated_for_other_node << "/" + << stats.generated_for_other_node_max << std::endl; break; } } @@ -638,6 +663,14 @@ fuzzing::possible_path fuzzing::iid_node_dependence_props::return_path( const po } } + if ( stats.state == generation_state::GENERATION_DATA_FOR_NEXT_NODE ) { + stats.generated_for_other_node++; + + if ( stats.generated_for_other_node > stats.generated_for_other_node_max ) { + stats.state = generation_state::COVERED; + } + } + return path; } @@ -1152,7 +1185,7 @@ void fuzzing::iid_dependencies::remove_node_dependence( location_id id ) iid_node_generations_stats& stats = it->second.get_generations_stats(); stats.state = generation_state::COVERED; - if ( generate_more_data_after_coverage ) { + if ( generate_more_data_after_coverage && !it->second.is_equal_branching_predicate() ) { stats.state = generation_state::GENERATION_MORE; int max_generation_after_covered = std::max( minimal_max_generation_after_covered, stats.successful_generations / 2 ); @@ -1182,6 +1215,22 @@ std::vector< location_id > fuzzing::iid_dependencies::get_iid_nodes() // ------------------------------------------------------------------------------------------------ std::optional< location_id > fuzzing::iid_dependencies::get_next_iid_node() { + bool previous_needs_more_data = false; + for ( auto it = id_to_equation_map.rbegin(); it != id_to_equation_map.rend(); ++it ) { + iid_node_dependence_props& props = it->second; + + if ( previous_needs_more_data ) { + props.set_as_generating_for_other_node( minimal_max_generation_for_other_node ); + previous_needs_more_data = false; + } + + if ( props.needs_data_from_other_node( max_failed_generations_in_row ) ) { + previous_needs_more_data = true; + props.get_generations_stats().failed_generations_in_row = 0; + } + } + + std::optional< location_id > best_id = std::nullopt; for ( const auto& [ id, props ] : id_to_equation_map ) { @@ -1202,8 +1251,8 @@ std::optional< location_id > fuzzing::iid_dependencies::get_next_iid_node() // return it->first; // } - // if ( it->second.get_number_of_failed_generations() >= max_failed_generations ) { - // if ( current_failed_generations >= max_failed_generations ) { + // if ( it->second.get_number_of_failed_generations() >= max_failed_generations_in_row ) { + // if ( current_failed_generations >= max_failed_generations_in_row ) { // it->second.get_number_of_failed_generations() = 0; // current_failed_generations = 0; // } else { From bde3a88316848f03898f891f9893f03232e8e46b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 25 Jan 2025 15:17:49 +0100 Subject: [PATCH 092/144] feat: update execution counts in IID benchmark JSON files to correspond with new analysis --- benchmarks/iid_testing/combination_needed.json | 3 +-- benchmarks/iid_testing/decrement_in_condition.json | 3 +-- benchmarks/iid_testing/decrement_in_else.json | 3 +-- benchmarks/iid_testing/decrement_on_b.json | 3 +-- benchmarks/iid_testing/different_predicates_in_iid.json | 3 +-- benchmarks/iid_testing/double_loop.json | 3 +-- benchmarks/iid_testing/iid.json | 3 +-- benchmarks/iid_testing/iid_in_condition.json | 3 +-- benchmarks/iid_testing/iid_multiple.json | 3 +-- benchmarks/iid_testing/iid_reverse_predicate.json | 2 +- benchmarks/iid_testing/iid_twice.json | 3 +-- benchmarks/iid_testing/input_cycle.json | 3 +-- benchmarks/iid_testing/input_cycle_dec_b.json | 3 +-- benchmarks/iid_testing/lots_of_non_iid_conditions.json | 1 - benchmarks/iid_testing/more_iid_condition.json | 3 +-- benchmarks/iid_testing/more_iid_condition_nested.json | 2 +- benchmarks/iid_testing/multiplication_in_iid.json | 3 +-- benchmarks/iid_testing/same_loop_dependent.json | 3 +-- benchmarks/iid_testing/same_loop_dependent_2.json | 2 +- benchmarks/iid_testing/same_loop_independent.json | 3 +-- 20 files changed, 19 insertions(+), 36 deletions(-) diff --git a/benchmarks/iid_testing/combination_needed.json b/benchmarks/iid_testing/combination_needed.json index 89fda376..a984ac71 100644 --- a/benchmarks/iid_testing/combination_needed.json +++ b/benchmarks/iid_testing/combination_needed.json @@ -13,11 +13,10 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, - "num_executions_original": 10000, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 9100, + "num_executions": 1929, "num_covered_branchings": 8, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/decrement_in_condition.json b/benchmarks/iid_testing/decrement_in_condition.json index 87bb1359..541e5b8d 100644 --- a/benchmarks/iid_testing/decrement_in_condition.json +++ b/benchmarks/iid_testing/decrement_in_condition.json @@ -13,11 +13,10 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, - "num_executions_original": 7110, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 3981, + "num_executions": 3226, "num_covered_branchings": 9, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/decrement_in_else.json b/benchmarks/iid_testing/decrement_in_else.json index 2c9853ad..9db8041b 100644 --- a/benchmarks/iid_testing/decrement_in_else.json +++ b/benchmarks/iid_testing/decrement_in_else.json @@ -13,11 +13,10 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, - "num_executions_original": 1868, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 1868, + "num_executions": 1473, "num_covered_branchings": 7, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/decrement_on_b.json b/benchmarks/iid_testing/decrement_on_b.json index a0a83fbc..b44807f3 100644 --- a/benchmarks/iid_testing/decrement_on_b.json +++ b/benchmarks/iid_testing/decrement_on_b.json @@ -13,11 +13,10 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, - "num_executions_original": 7000, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 3710, + "num_executions": 3007, "num_covered_branchings": 8, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/different_predicates_in_iid.json b/benchmarks/iid_testing/different_predicates_in_iid.json index db086d43..39873d7f 100644 --- a/benchmarks/iid_testing/different_predicates_in_iid.json +++ b/benchmarks/iid_testing/different_predicates_in_iid.json @@ -13,11 +13,10 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, - "num_executions_original": 7000, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 58926, + "num_executions": 12981, "num_covered_branchings": 19, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/double_loop.json b/benchmarks/iid_testing/double_loop.json index 551d55c6..9d65986d 100644 --- a/benchmarks/iid_testing/double_loop.json +++ b/benchmarks/iid_testing/double_loop.json @@ -13,11 +13,10 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, - "num_executions_original": 9000, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 3600, + "num_executions": 3019, "num_covered_branchings": 11, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/iid.json b/benchmarks/iid_testing/iid.json index 5bbee2f3..c106e488 100644 --- a/benchmarks/iid_testing/iid.json +++ b/benchmarks/iid_testing/iid.json @@ -14,11 +14,10 @@ "optimizer_max_stdin_bytes": 1000000, "m32": false }, - "num_executions_original": 6044, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 3807, + "num_executions": 3001, "num_covered_branchings": 7, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/iid_in_condition.json b/benchmarks/iid_testing/iid_in_condition.json index 7adf874e..fd6e7b10 100644 --- a/benchmarks/iid_testing/iid_in_condition.json +++ b/benchmarks/iid_testing/iid_in_condition.json @@ -13,11 +13,10 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, - "num_executions_original": 6810, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 3850, + "num_executions": 3000, "num_covered_branchings": 9, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/iid_multiple.json b/benchmarks/iid_testing/iid_multiple.json index c141381d..658096c1 100644 --- a/benchmarks/iid_testing/iid_multiple.json +++ b/benchmarks/iid_testing/iid_multiple.json @@ -13,11 +13,10 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, - "num_executions_original": 6480, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 3693, + "num_executions": 3061, "num_covered_branchings": 16, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/iid_reverse_predicate.json b/benchmarks/iid_testing/iid_reverse_predicate.json index 6077caf1..b18e29d2 100644 --- a/benchmarks/iid_testing/iid_reverse_predicate.json +++ b/benchmarks/iid_testing/iid_reverse_predicate.json @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 4108, + "num_executions": 3764, "num_covered_branchings": 7, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/iid_twice.json b/benchmarks/iid_testing/iid_twice.json index 56b6faba..d92f6ffa 100644 --- a/benchmarks/iid_testing/iid_twice.json +++ b/benchmarks/iid_testing/iid_twice.json @@ -13,11 +13,10 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, - "num_executions_original": 1000, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 8000, + "num_executions": 5489, "num_covered_branchings": 10, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/input_cycle.json b/benchmarks/iid_testing/input_cycle.json index b570da02..bd182c8f 100644 --- a/benchmarks/iid_testing/input_cycle.json +++ b/benchmarks/iid_testing/input_cycle.json @@ -13,11 +13,10 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, - "num_executions_original": 2007, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 1556, + "num_executions": 1117, "num_covered_branchings": 9, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/input_cycle_dec_b.json b/benchmarks/iid_testing/input_cycle_dec_b.json index e00e68be..4a4a7e93 100644 --- a/benchmarks/iid_testing/input_cycle_dec_b.json +++ b/benchmarks/iid_testing/input_cycle_dec_b.json @@ -13,11 +13,10 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, - "num_executions_original": 2007, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 2007, + "num_executions": 1802, "num_covered_branchings": 10, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/lots_of_non_iid_conditions.json b/benchmarks/iid_testing/lots_of_non_iid_conditions.json index be0942cf..57e11217 100644 --- a/benchmarks/iid_testing/lots_of_non_iid_conditions.json +++ b/benchmarks/iid_testing/lots_of_non_iid_conditions.json @@ -13,7 +13,6 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, - "num_executions_original": 410, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", diff --git a/benchmarks/iid_testing/more_iid_condition.json b/benchmarks/iid_testing/more_iid_condition.json index 55f23df1..b19a5aea 100644 --- a/benchmarks/iid_testing/more_iid_condition.json +++ b/benchmarks/iid_testing/more_iid_condition.json @@ -13,11 +13,10 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, - "num_executions_original": 10000, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 6900, + "num_executions": 6114, "num_covered_branchings": 9, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/more_iid_condition_nested.json b/benchmarks/iid_testing/more_iid_condition_nested.json index 65795f66..809899d7 100644 --- a/benchmarks/iid_testing/more_iid_condition_nested.json +++ b/benchmarks/iid_testing/more_iid_condition_nested.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 30000, + "num_executions": 8612, "num_covered_branchings": 9, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/multiplication_in_iid.json b/benchmarks/iid_testing/multiplication_in_iid.json index 21311f2a..9db8041b 100644 --- a/benchmarks/iid_testing/multiplication_in_iid.json +++ b/benchmarks/iid_testing/multiplication_in_iid.json @@ -13,11 +13,10 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, - "num_executions_original": 10000, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 2000, + "num_executions": 1473, "num_covered_branchings": 7, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/same_loop_dependent.json b/benchmarks/iid_testing/same_loop_dependent.json index 37502f5e..745179d8 100644 --- a/benchmarks/iid_testing/same_loop_dependent.json +++ b/benchmarks/iid_testing/same_loop_dependent.json @@ -13,11 +13,10 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, - "num_executions_original": 10000, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 1900, + "num_executions": 1479, "num_covered_branchings": 11, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/same_loop_dependent_2.json b/benchmarks/iid_testing/same_loop_dependent_2.json index 08a0afd8..c7c8e9a9 100644 --- a/benchmarks/iid_testing/same_loop_dependent_2.json +++ b/benchmarks/iid_testing/same_loop_dependent_2.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 7739, + "num_executions": 2085, "num_covered_branchings": 11, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/same_loop_independent.json b/benchmarks/iid_testing/same_loop_independent.json index 4d67f76c..9d65986d 100644 --- a/benchmarks/iid_testing/same_loop_independent.json +++ b/benchmarks/iid_testing/same_loop_independent.json @@ -13,11 +13,10 @@ "optimizer_max_trace_length": 1000000, "optimizer_max_stdin_bytes": 1000000 }, - "num_executions_original": 10000, "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 3600, + "num_executions": 3019, "num_covered_branchings": 11, "covered_branchings": [ 1, From 7ff172ceac2fd89da78d7b34ac6842bee0407e5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 25 Jan 2025 18:15:03 +0100 Subject: [PATCH 093/144] feat: add linear dependency check to equation --- .../include/fuzzing/iid_vector_analysis.hpp | 3 +- src/fuzzing/src/iid_vector_analysis.cpp | 143 ++++++++---------- 2 files changed, 66 insertions(+), 80 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index ed2f6ee0..4aed7753 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -106,6 +106,7 @@ struct equation { int get_biggest_value() const; bool is_any_negative() const; bool same_values() const; + bool is_linear_dependent( const equation& other ) const; friend std::ostream& operator<<( std::ostream& os, const equation& eq ) { @@ -254,7 +255,7 @@ struct iid_dependencies { // Settings bool generate_more_data_after_coverage = true; int minimal_max_generation_after_covered = 10; - int max_failed_generations_in_row = 10; + int max_failed_generations_in_row = 3; int minimal_max_generation_for_other_node = 10; }; diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 008a3d64..7e8cd6f0 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -38,9 +38,7 @@ bool fuzzing::path_node_props::get_desired_direction() const return true; } - std::cout << "Error: get_desired_direction() called without any possible direction" << std::endl; - std::cout << *this << std::endl; - return false; + throw std::runtime_error( "Error: get_desired_direction() called without any possible direction" ); } // ------------------------------------------------------------------------------------------------ @@ -226,7 +224,7 @@ bool fuzzing::equation::is_any_negative() const bool fuzzing::equation::same_values() const { for ( int i = 0; i < values.size(); ++i ) { - if ( values[ i ] != best_value ) { + if ( values[ i ] != best_value && values[ i ] != 0 ) { return false; } } @@ -234,6 +232,40 @@ bool fuzzing::equation::same_values() const return true; } +// ------------------------------------------------------------------------------------------------ +bool fuzzing::equation::is_linear_dependent( const equation& other ) const +{ + INVARIANT( values.size() == other.values.size() ); + + double ratio = std::numeric_limits::quiet_NaN(); + for ( int i = 0; i < values.size(); ++i ) { + if ( values[ i ] == 0 && other.values[ i ] == 0 ) { + continue; + } + + if ( values[ i ] == 0 || other.values[ i ] == 0 ) { + return false; + } + + double current_ratio = double( values[ i ] ) / other.values[ i ]; + if ( std::isnan( ratio ) ) { + ratio = current_ratio; + } else if ( std::abs( ratio - current_ratio ) > 1e-9 ) { + return false; + } + } + + if ( best_value == 0 && other.best_value == 0 ) { + return true; + } + + if ( best_value == 0 || other.best_value == 0 ) { + return false; + } + + return std::abs( best_value / other.best_value - ratio ) < 1e-9; +} + // node_direction // ------------------------------------------------------------------------------------------------ auto fuzzing::node_direction::operator<=>( node_direction const& other ) const @@ -410,12 +442,6 @@ fuzzing::equation_matrix::get_new_leaf_counts_from_vectors( const std::vector< e bool is_equal_sign = get_branching_predicate() == BRANCHING_PREDICATE::BP_EQUAL; - auto add_if_positive = [ &paths ]( const equation& new_path ) { - if ( !new_path.is_any_negative() ) { - paths.push_back( new_path ); - } - }; - for ( const auto& vector : vectors ) { for ( const auto& row : matrix ) { double counts = std::abs( row.best_value ) / std::abs( vector.best_value ); @@ -432,7 +458,9 @@ fuzzing::equation_matrix::get_new_leaf_counts_from_vectors( const std::vector< e // new_path = new_path.add_to_positive( 1 + limit ); } - add_if_positive( new_path ); + if ( !new_path.is_any_negative() ) { + paths.push_back( new_path ); + } } } @@ -496,15 +524,18 @@ void fuzzing::equation_matrix::recompute_matrix() // ------------------------------------------------------------------------------------------------ fuzzing::possible_path fuzzing::iid_node_dependence_props::generate_probabilities() { - // print_stats(); - // print_dependencies(); - // matrix.print_matrix(); + TMPROF_BLOCK(); stats.generation_starts++; std::set< node_direction > all_leafs = get_leaf_subsets(); equation_matrix submatrix = matrix.get_submatrix( all_leafs, true ); - // submatrix.print_matrix(); + { + // print_stats(); + // print_dependencies(); + // matrix.print_matrix(); + // submatrix.print_matrix(); + } std::map< equation, int > vectors = submatrix.compute_vectors_with_hits(); if ( vectors.empty() ) { @@ -520,32 +551,15 @@ fuzzing::possible_path fuzzing::iid_node_dependence_props::generate_probabilitie return return_empty_path(); } - // std::cout << "Best vectors: " << std::endl; - // for (const auto& vector : best_vectors) { - // for (const auto& value : vector.values) { - // std::cout << value << " "; - // } - // std::cout << "-> | " << vector.best_value << std::endl; - // } - std::optional< equation > new_leaf_counts = submatrix.get_new_leaf_counts_from_vectors( best_vectors, stats.generated_after_covered ); if ( !new_leaf_counts.has_value() ) { return return_empty_path(); } - // std::cout << "New path: "; - // for (const auto& value : new_leaf_counts->values) { - // std::cout << value << " "; - // } - // std::cout << "-> | " << new_leaf_counts->best_value << std::endl; - nodes_to_counts node_counts = compute_path_counts( new_leaf_counts.value(), all_leafs ); possible_path path = generate_path_from_node_counts( node_counts ); - // std::cout << "New path: " << std::endl; - // std::cout << path << std::endl; - return return_path( path ); } @@ -709,7 +723,6 @@ void fuzzing::iid_node_dependence_props::compute_path_counts_for_nested_loops( n } } - if ( possible_counts.empty() ) { path_counts[ loop_head_id ] = { 1, 1 }; return; @@ -878,7 +891,6 @@ fuzzing::iid_node_dependence_props::compute_best_vectors( const std::map< equati if ( filtered_vectors_with_hits.empty() ) { return {}; - // throw std::invalid_argument( "No vectors match the desired direction." ); } if ( use_random ) { @@ -895,25 +907,26 @@ fuzzing::iid_node_dependence_props::compute_best_vectors( const std::map< equati return a.second > b.second; } ); + bool use_linear_dependency = true; std::vector< equation > best_vectors; for ( int i = 0; i < number_of_vectors && i < sorted_vectors.size(); ++i ) { - best_vectors.push_back( sorted_vectors[ i ].first ); + if ( use_linear_dependency ) { + std::map< equation, int > dependent_vectors_with_hits = + get_linear_dependent_vector( filtered_vectors_with_hits, sorted_vectors[ i ].first ); + + auto it = std::min_element( dependent_vectors_with_hits.begin(), + dependent_vectors_with_hits.end(), + []( const auto& a, const auto& b ) { + return a.first.get_vector_size() < b.first.get_vector_size(); + } ); + best_vectors.push_back( it->first ); + // std::cout << "Chosen vector : " << it->first << std::endl; + // std::cout << "Original vector: " << sorted_vectors[ i ].first << std::endl; + } else { + best_vectors.push_back( sorted_vectors[ i ].first ); + } } - // std::map< equation, int > dependent_vectors_with_hits = - // get_linear_dependent_vector( filtered_vectors_with_hits, best_vector ); - - // if ( !dependent_vectors_with_hits.empty() ) { - // auto min_it = std::min_element( dependent_vectors_with_hits.begin(), - // dependent_vectors_with_hits.end(), - // []( const auto& a, const auto& b ) { - // return a.first.get_vector_length() < - // b.first.get_vector_length(); - // } ); - - // best_vector = min_it->first; - // } - return best_vectors; } @@ -925,12 +938,12 @@ std::map< fuzzing::equation, int > fuzzing::iid_node_dependence_props::get_linea std::map< equation, int > dependent_vectors_with_hits; for ( const auto& [ vector, hits ] : vectors_with_hits ) { - equation quotient = best_vector / vector; - if ( quotient.same_values() ) { + if ( best_vector.is_linear_dependent( vector ) ) { dependent_vectors_with_hits[ vector ] = hits; } } + INVARIANT( !dependent_vectors_with_hits.empty() ); return dependent_vectors_with_hits; } @@ -1230,7 +1243,6 @@ std::optional< location_id > fuzzing::iid_dependencies::get_next_iid_node() } } - std::optional< location_id > best_id = std::nullopt; for ( const auto& [ id, props ] : id_to_equation_map ) { @@ -1240,33 +1252,6 @@ std::optional< location_id > fuzzing::iid_dependencies::get_next_iid_node() } return best_id; - - - // bool previous_failed = false; - - // for ( auto it = id_to_equation_map.rbegin(); it != id_to_equation_map.rend(); ++it ) { - // if ( previous_failed ) { - // previous_failed = false; - // current_failed_generations++; - // return it->first; - // } - - // if ( it->second.get_number_of_failed_generations() >= max_failed_generations_in_row ) { - // if ( current_failed_generations >= max_failed_generations_in_row ) { - // it->second.get_number_of_failed_generations() = 0; - // current_failed_generations = 0; - // } else { - // previous_failed = true; - // continue; - // } - // } - - // if ( it->second.get_generation_count_after_covered() >= max_generation_count_after_covered ) { - // continue; - // } - - // best_id = it->first; - // } } // non member functions From cb71d9cfeab8928393765dac65b9a7898864ef2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 25 Jan 2025 18:15:34 +0100 Subject: [PATCH 094/144] feat: update execution counts in IID benchmark JSON files after adding dependency --- benchmarks/iid_testing/input_cycle_dec_b.json | 2 +- benchmarks/iid_testing/more_iid_condition_nested.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/iid_testing/input_cycle_dec_b.json b/benchmarks/iid_testing/input_cycle_dec_b.json index 4a4a7e93..e4ec9c23 100644 --- a/benchmarks/iid_testing/input_cycle_dec_b.json +++ b/benchmarks/iid_testing/input_cycle_dec_b.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 1802, + "num_executions": 1470, "num_covered_branchings": 10, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/more_iid_condition_nested.json b/benchmarks/iid_testing/more_iid_condition_nested.json index 809899d7..539b45e1 100644 --- a/benchmarks/iid_testing/more_iid_condition_nested.json +++ b/benchmarks/iid_testing/more_iid_condition_nested.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 8612, + "num_executions": 7836, "num_covered_branchings": 9, "covered_branchings": [ 1, From b49b836fc8986022a26c6f3dddd0ed201523b075 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 25 Jan 2025 18:27:08 +0100 Subject: [PATCH 095/144] feat: adjust project declaration in CMakeLists.txt --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 049b514b..db2dca1f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ -project(fizzer) - cmake_minimum_required(VERSION 3.20 FATAL_ERROR) +project(fizzer) + macro(append_compiler_flags FLAGS) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FLAGS}") endmacro() From 2d898c2b7ae88262f2d54ef318d4736f2ba4ee29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sun, 26 Jan 2025 16:34:49 +0100 Subject: [PATCH 096/144] feat: use namespace --- src/fuzzing/src/iid_vector_analysis.cpp | 196 ++++++++++++------------ 1 file changed, 96 insertions(+), 100 deletions(-) diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 7e8cd6f0..cccf5322 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -9,13 +9,17 @@ #include #include +namespace fuzzing +{ + + // node_counts // ------------------------------------------------------------------------------------------------ -int fuzzing::node_counts::get_max_count() const { return std::max( left_count, right_count ); } +int node_counts::get_max_count() const { return std::max( left_count, right_count ); } // path_node_props // ------------------------------------------------------------------------------------------------ -bool fuzzing::path_node_props::get_desired_direction() const +bool path_node_props::get_desired_direction() const { INVARIANT( computed_counts.left_count + computed_counts.right_count > 0 ); @@ -42,7 +46,7 @@ bool fuzzing::path_node_props::get_desired_direction() const } // ------------------------------------------------------------------------------------------------ -bool fuzzing::path_node_props::can_go_direction( bool direction ) const +bool path_node_props::can_go_direction( bool direction ) const { if ( direction ) { return taken_counts.right_count < computed_counts.right_count; @@ -52,7 +56,7 @@ bool fuzzing::path_node_props::can_go_direction( bool direction ) const } // ------------------------------------------------------------------------------------------------ -void fuzzing::path_node_props::go_direction( bool direction ) +void path_node_props::go_direction( bool direction ) { if ( direction ) { taken_counts.right_count++; @@ -67,14 +71,14 @@ void fuzzing::path_node_props::go_direction( bool direction ) } // ------------------------------------------------------------------------------------------------ -bool fuzzing::path_node_props::can_take_next_direction() const +bool path_node_props::can_take_next_direction() const { return taken_counts.left_count < computed_counts.left_count || taken_counts.right_count < computed_counts.right_count; } // ------------------------------------------------------------------------------------------------ -float_32_bit fuzzing::path_node_props::get_false_direction_probability() const +float_32_bit path_node_props::get_false_direction_probability() const { INVARIANT( computed_counts.left_count + computed_counts.right_count > 0 ); @@ -83,7 +87,7 @@ float_32_bit fuzzing::path_node_props::get_false_direction_probability() const } // ------------------------------------------------------------------------------------------------ -bool fuzzing::path_node_props::get_preferred_direction_loop_head() const +bool path_node_props::get_preferred_direction_loop_head() const { auto is_depleted = []( int computed, int taken ) { return computed == taken; }; @@ -98,17 +102,14 @@ bool fuzzing::path_node_props::get_preferred_direction_loop_head() const // possible_path // ------------------------------------------------------------------------------------------------ -bool fuzzing::possible_path::contains( location_id::id_type id ) const { return path.contains( id ); } +bool possible_path::contains( location_id::id_type id ) const { return path.contains( id ); } // ------------------------------------------------------------------------------------------------ -std::map< location_id::id_type, fuzzing::path_node_props > fuzzing::possible_path::get_path() const -{ - return path; -} +std::map< location_id::id_type, path_node_props > possible_path::get_path() const { return path; } // equation // ------------------------------------------------------------------------------------------------ -fuzzing::equation fuzzing::equation::operator+( const equation& other ) const +equation equation::operator+( const equation& other ) const { INVARIANT( values.size() == other.values.size() ); @@ -121,7 +122,7 @@ fuzzing::equation fuzzing::equation::operator+( const equation& other ) const } // ------------------------------------------------------------------------------------------------ -fuzzing::equation fuzzing::equation::operator+( int scalar ) const +equation equation::operator+( int scalar ) const { std::vector< int > new_values; for ( int i = 0; i < values.size(); ++i ) { @@ -132,7 +133,7 @@ fuzzing::equation fuzzing::equation::operator+( int scalar ) const } // ------------------------------------------------------------------------------------------------ -fuzzing::equation fuzzing::equation::operator-( const equation& other ) const +equation equation::operator-( const equation& other ) const { INVARIANT( values.size() == other.values.size() ); @@ -145,7 +146,7 @@ fuzzing::equation fuzzing::equation::operator-( const equation& other ) const } // ------------------------------------------------------------------------------------------------ -fuzzing::equation fuzzing::equation::operator*( int scalar ) const +equation equation::operator*( int scalar ) const { std::vector< int > new_values; for ( int i = 0; i < values.size(); ++i ) { @@ -156,7 +157,7 @@ fuzzing::equation fuzzing::equation::operator*( int scalar ) const } // ------------------------------------------------------------------------------------------------ -fuzzing::equation fuzzing::equation::operator*( double scalar ) const +equation equation::operator*( double scalar ) const { std::vector< int > new_values; for ( int i = 0; i < values.size(); ++i ) { @@ -167,7 +168,7 @@ fuzzing::equation fuzzing::equation::operator*( double scalar ) const } // ------------------------------------------------------------------------------------------------ -fuzzing::equation fuzzing::equation::operator/( const equation& other ) const +equation equation::operator/( const equation& other ) const { INVARIANT( values.size() == other.values.size() ); @@ -185,7 +186,7 @@ fuzzing::equation fuzzing::equation::operator/( const equation& other ) const } // ------------------------------------------------------------------------------------------------ -fuzzing::equation fuzzing::equation::add_to_positive( int value ) const +equation equation::add_to_positive( int value ) const { std::vector< int > new_values; for ( int i = 0; i < values.size(); ++i ) { @@ -200,28 +201,28 @@ fuzzing::equation fuzzing::equation::add_to_positive( int value ) const } // ------------------------------------------------------------------------------------------------ -int fuzzing::equation::get_vector_size() const +int equation::get_vector_size() const { return std::accumulate( values.begin(), values.end(), 0, []( int sum, int val ) { return sum + val; } ); } // ------------------------------------------------------------------------------------------------ -int fuzzing::equation::get_one_way_branching_count() const +int equation::get_one_way_branching_count() const { return std::count_if( values.begin(), values.end(), []( int val ) { return val == 0; } ); } // ------------------------------------------------------------------------------------------------ -int fuzzing::equation::get_biggest_value() const { return *std::max_element( values.begin(), values.end() ); } +int equation::get_biggest_value() const { return *std::max_element( values.begin(), values.end() ); } // ------------------------------------------------------------------------------------------------ -bool fuzzing::equation::is_any_negative() const +bool equation::is_any_negative() const { return std::any_of( values.begin(), values.end(), []( int val ) { return val < 0; } ); } // ------------------------------------------------------------------------------------------------ -bool fuzzing::equation::same_values() const +bool equation::same_values() const { for ( int i = 0; i < values.size(); ++i ) { if ( values[ i ] != best_value && values[ i ] != 0 ) { @@ -233,11 +234,11 @@ bool fuzzing::equation::same_values() const } // ------------------------------------------------------------------------------------------------ -bool fuzzing::equation::is_linear_dependent( const equation& other ) const +bool equation::is_linear_dependent( const equation& other ) const { INVARIANT( values.size() == other.values.size() ); - double ratio = std::numeric_limits::quiet_NaN(); + double ratio = std::numeric_limits< double >::quiet_NaN(); for ( int i = 0; i < values.size(); ++i ) { if ( values[ i ] == 0 && other.values[ i ] == 0 ) { continue; @@ -268,7 +269,7 @@ bool fuzzing::equation::is_linear_dependent( const equation& other ) const // node_direction // ------------------------------------------------------------------------------------------------ -auto fuzzing::node_direction::operator<=>( node_direction const& other ) const +auto node_direction::operator<=>( node_direction const& other ) const { if ( auto const cmp = node_id.id <=> other.node_id.id; cmp != 0 ) return cmp; @@ -278,8 +279,7 @@ auto fuzzing::node_direction::operator<=>( node_direction const& other ) const // equation_matrix // ------------------------------------------------------------------------------------------------ -fuzzing::equation_matrix fuzzing::equation_matrix::get_submatrix( std::set< node_direction > const& subset, - bool unique ) const +equation_matrix equation_matrix::get_submatrix( std::set< node_direction > const& subset, bool unique ) const { equation_matrix result; result.nodes = subset; @@ -312,7 +312,7 @@ fuzzing::equation_matrix fuzzing::equation_matrix::get_submatrix( std::set< node } // ------------------------------------------------------------------------------------------------ -void fuzzing::equation_matrix::process_node( branching_node* end_node ) +void equation_matrix::process_node( branching_node* end_node ) { all_paths.push_back( end_node ); @@ -331,7 +331,7 @@ void fuzzing::equation_matrix::process_node( branching_node* end_node ) } // ------------------------------------------------------------------------------------------------ -void fuzzing::equation_matrix::add_equation( branching_node* end_node ) +void equation_matrix::add_equation( branching_node* end_node ) { TMPROF_BLOCK(); @@ -358,16 +358,16 @@ void fuzzing::equation_matrix::add_equation( branching_node* end_node ) } // ------------------------------------------------------------------------------------------------ -bool fuzzing::equation_matrix::contains( node_direction const& node ) const { return nodes.contains( node ); } +bool equation_matrix::contains( node_direction const& node ) const { return nodes.contains( node ); } // ------------------------------------------------------------------------------------------------ -std::pair< std::size_t, std::size_t > fuzzing::equation_matrix::get_dimensions() const +std::pair< std::size_t, std::size_t > equation_matrix::get_dimensions() const { return { matrix.size(), nodes.size() }; } // ------------------------------------------------------------------------------------------------ -std::map< fuzzing::equation, int > fuzzing::equation_matrix::compute_vectors_with_hits() +std::map< equation, int > equation_matrix::compute_vectors_with_hits() { std::map< equation, int > vectors_with_hits; @@ -399,10 +399,10 @@ std::map< fuzzing::equation, int > fuzzing::equation_matrix::compute_vectors_wit } // ------------------------------------------------------------------------------------------------ -std::vector< fuzzing::equation >& fuzzing::equation_matrix::get_matrix() { return matrix; } +std::vector< equation >& equation_matrix::get_matrix() { return matrix; } // ------------------------------------------------------------------------------------------------ -int fuzzing::equation_matrix::get_desired_vector_direction() const +int equation_matrix::get_desired_vector_direction() const { auto is_positive = []( const equation& eq ) { return eq.best_value > 0; }; auto is_negative = []( const equation& eq ) { return eq.best_value < 0; }; @@ -417,7 +417,7 @@ int fuzzing::equation_matrix::get_desired_vector_direction() const } // ------------------------------------------------------------------------------------------------ -float fuzzing::equation_matrix::get_biggest_branching_value() const +float equation_matrix::get_biggest_branching_value() const { float biggest_value = 0.0f; @@ -431,9 +431,8 @@ float fuzzing::equation_matrix::get_biggest_branching_value() const } // ------------------------------------------------------------------------------------------------ -std::optional< fuzzing::equation > -fuzzing::equation_matrix::get_new_leaf_counts_from_vectors( const std::vector< equation >& vectors, - int generation_count_after_covered ) +std::optional< equation > equation_matrix::get_new_leaf_counts_from_vectors( const std::vector< equation >& vectors, + int generation_count_after_covered ) { INVARIANT( !vectors.empty() ); INVARIANT( vectors[ 0 ].values.size() == nodes.size() ); @@ -485,7 +484,7 @@ fuzzing::equation_matrix::get_new_leaf_counts_from_vectors( const std::vector< e } // ------------------------------------------------------------------------------------------------ -void fuzzing::equation_matrix::print_matrix() +void equation_matrix::print_matrix() { std::cout << "# Matrix:" << std::endl; for ( const node_direction& nav : nodes ) { @@ -502,14 +501,14 @@ void fuzzing::equation_matrix::print_matrix() // ------------------------------------------------------------------------------------------------ -BRANCHING_PREDICATE fuzzing::equation_matrix::get_branching_predicate() const +BRANCHING_PREDICATE equation_matrix::get_branching_predicate() const { ASSUMPTION( all_paths.size() > 0 ); return all_paths[ 0 ]->branching_predicate; } // ------------------------------------------------------------------------------------------------ -void fuzzing::equation_matrix::recompute_matrix() +void equation_matrix::recompute_matrix() { TMPROF_BLOCK(); @@ -522,7 +521,7 @@ void fuzzing::equation_matrix::recompute_matrix() // iid_node_dependence_props // ------------------------------------------------------------------------------------------------ -fuzzing::possible_path fuzzing::iid_node_dependence_props::generate_probabilities() +possible_path iid_node_dependence_props::generate_probabilities() { TMPROF_BLOCK(); @@ -564,7 +563,7 @@ fuzzing::possible_path fuzzing::iid_node_dependence_props::generate_probabilitie } // ------------------------------------------------------------------------------------------------ -void fuzzing::iid_node_dependence_props::process_node( branching_node* end_node ) +void iid_node_dependence_props::process_node( branching_node* end_node ) { loop_head_to_bodies_t loop_heads_to_bodies; loop_endings loop_heads_ending = get_loop_heads_ending( end_node, loop_heads_to_bodies ); @@ -576,14 +575,14 @@ void fuzzing::iid_node_dependence_props::process_node( branching_node* end_node } // ------------------------------------------------------------------------------------------------ -bool fuzzing::iid_node_dependence_props::should_generate() const +bool iid_node_dependence_props::should_generate() const { return stats.state == generation_state::NOT_COVERED || stats.state == generation_state::GENERATION_MORE || stats.state == generation_state::GENERATION_DATA_FOR_NEXT_NODE; } // ------------------------------------------------------------------------------------------------ -bool fuzzing::iid_node_dependence_props::needs_data_from_other_node( int max_failed_generations_in_row ) const +bool iid_node_dependence_props::needs_data_from_other_node( int max_failed_generations_in_row ) const { if ( stats.state != generation_state::NOT_COVERED ) { return false; @@ -597,7 +596,7 @@ bool fuzzing::iid_node_dependence_props::needs_data_from_other_node( int max_fai } // ------------------------------------------------------------------------------------------------ -void fuzzing::iid_node_dependence_props::set_as_generating_for_other_node( int minimal_max_generation_for_other_node ) +void iid_node_dependence_props::set_as_generating_for_other_node( int minimal_max_generation_for_other_node ) { INVARIANT( stats.state == generation_state::COVERED ); @@ -607,13 +606,13 @@ void fuzzing::iid_node_dependence_props::set_as_generating_for_other_node( int m } // ------------------------------------------------------------------------------------------------ -bool fuzzing::iid_node_dependence_props::is_equal_branching_predicate() const +bool iid_node_dependence_props::is_equal_branching_predicate() const { return matrix.get_branching_predicate() == BRANCHING_PREDICATE::BP_EQUAL; } // ------------------------------------------------------------------------------------------------ -void fuzzing::iid_node_dependence_props::print_dependencies() const +void iid_node_dependence_props::print_dependencies() const { std::cout << "# Dependencies:" << std::endl; std::cout << "## Dependencies by loops:" << std::endl; @@ -632,7 +631,7 @@ void fuzzing::iid_node_dependence_props::print_dependencies() const } // ------------------------------------------------------------------------------------------------ -void fuzzing::iid_node_dependence_props::print_stats() const +void iid_node_dependence_props::print_stats() const { switch ( stats.state ) { case generation_state::NOT_COVERED: @@ -656,7 +655,7 @@ void fuzzing::iid_node_dependence_props::print_stats() const } // ------------------------------------------------------------------------------------------------ -fuzzing::possible_path fuzzing::iid_node_dependence_props::return_empty_path() +possible_path iid_node_dependence_props::return_empty_path() { stats.failed_generations++; stats.failed_generations_in_row++; @@ -664,7 +663,7 @@ fuzzing::possible_path fuzzing::iid_node_dependence_props::return_empty_path() } // ------------------------------------------------------------------------------------------------ -fuzzing::possible_path fuzzing::iid_node_dependence_props::return_path( const possible_path& path ) +possible_path iid_node_dependence_props::return_path( const possible_path& path ) { stats.failed_generations_in_row = 0; stats.successful_generations++; @@ -689,11 +688,11 @@ fuzzing::possible_path fuzzing::iid_node_dependence_props::return_path( const po } // ------------------------------------------------------------------------------------------------ -void fuzzing::iid_node_dependence_props::compute_path_counts_for_nested_loops( nodes_to_counts& path_counts, - std::map< location_id, int >& child_loop_counts, - location_id loop_head_id, - int minimum_count, - bool use_random ) +void iid_node_dependence_props::compute_path_counts_for_nested_loops( nodes_to_counts& path_counts, + std::map< location_id, int >& child_loop_counts, + location_id loop_head_id, + int minimum_count, + bool use_random ) { INVARIANT( !child_loop_counts.empty() ); @@ -749,9 +748,9 @@ void fuzzing::iid_node_dependence_props::compute_path_counts_for_nested_loops( n } // ------------------------------------------------------------------------------------------------ -void fuzzing::iid_node_dependence_props::compute_path_counts_loading( nodes_to_counts& path_counts, - const equation& path, - const std::set< location_id >& loop_heads ) +void iid_node_dependence_props::compute_path_counts_loading( nodes_to_counts& path_counts, + const equation& path, + const std::set< location_id >& loop_heads ) { // TODO for ( const auto& [ loading_head, props ] : dependencies_by_loading ) { @@ -775,9 +774,9 @@ void fuzzing::iid_node_dependence_props::compute_path_counts_loading( nodes_to_c } // ------------------------------------------------------------------------------------------------ -void fuzzing::iid_node_dependence_props::compute_path_counts_loops( nodes_to_counts& path_counts, - const equation& path, - const std::set< location_id >& loop_heads ) +void iid_node_dependence_props::compute_path_counts_loops( nodes_to_counts& path_counts, + const equation& path, + const std::set< location_id >& loop_heads ) { for ( const auto& [ loop_head, props ] : std::ranges::views::reverse( dependencies_by_loops ) ) { std::map< location_id, int > child_loop_counts; @@ -802,9 +801,8 @@ void fuzzing::iid_node_dependence_props::compute_path_counts_loops( nodes_to_cou } // ------------------------------------------------------------------------------------------------ -fuzzing::nodes_to_counts -fuzzing::iid_node_dependence_props::compute_path_counts( const equation& path, - std::set< node_direction > const& all_leafs ) +nodes_to_counts iid_node_dependence_props::compute_path_counts( const equation& path, + std::set< node_direction > const& all_leafs ) { nodes_to_counts path_counts; @@ -858,12 +856,11 @@ fuzzing::iid_node_dependence_props::compute_path_counts( const equation& path, } // ------------------------------------------------------------------------------------------------ -std::vector< fuzzing::equation > -fuzzing::iid_node_dependence_props::compute_best_vectors( const std::map< equation, int >& vectors_with_hits, - int number_of_vectors, - bool use_random, - int desired_direction, - float biggest_branching_value ) +std::vector< equation > iid_node_dependence_props::compute_best_vectors( const std::map< equation, int >& vectors_with_hits, + int number_of_vectors, + bool use_random, + int desired_direction, + float biggest_branching_value ) { if ( vectors_with_hits.empty() ) { throw std::invalid_argument( "Input map is empty." ); @@ -931,9 +928,9 @@ fuzzing::iid_node_dependence_props::compute_best_vectors( const std::map< equati } // ------------------------------------------------------------------------------------------------ -std::map< fuzzing::equation, int > fuzzing::iid_node_dependence_props::get_linear_dependent_vector( - const std::map< equation, int >& vectors_with_hits, - equation& best_vector ) +std::map< equation, int > +iid_node_dependence_props::get_linear_dependent_vector( const std::map< equation, int >& vectors_with_hits, + equation& best_vector ) { std::map< equation, int > dependent_vectors_with_hits; @@ -948,9 +945,8 @@ std::map< fuzzing::equation, int > fuzzing::iid_node_dependence_props::get_linea } // ------------------------------------------------------------------------------------------------ -std::vector< fuzzing::equation > -fuzzing::iid_node_dependence_props::get_random_vector( const std::map< equation, int >& vectors_with_hits, - int number_of_vectors ) +std::vector< equation > iid_node_dependence_props::get_random_vector( const std::map< equation, int >& vectors_with_hits, + int number_of_vectors ) { std::vector< equation > equations; std::vector< double > probabilities; @@ -982,7 +978,7 @@ fuzzing::iid_node_dependence_props::get_random_vector( const std::map< equation, } // ------------------------------------------------------------------------------------------------ -std::set< fuzzing::node_direction > fuzzing::iid_node_dependence_props::get_leaf_subsets() +std::set< node_direction > iid_node_dependence_props::get_leaf_subsets() { std::set< node_direction > all_leafs; for ( const auto& [ _, props ] : dependencies_by_loops ) { @@ -998,12 +994,11 @@ std::set< fuzzing::node_direction > fuzzing::iid_node_dependence_props::get_leaf } // ------------------------------------------------------------------------------------------------ -std::map< location_id, bool > -fuzzing::iid_node_dependence_props::get_loop_heads_ending( branching_node* end_node, - loop_head_to_bodies_t& loop_heads_to_bodies ) +std::map< location_id, bool > iid_node_dependence_props::get_loop_heads_ending( branching_node* end_node, + loop_head_to_bodies_t& loop_heads_to_bodies ) { std::vector< fuzzer::loop_boundary_props > loops; - fuzzing::fuzzer::detect_loops_along_path_to_node( end_node, loop_heads_to_bodies, &loops ); + fuzzer::detect_loops_along_path_to_node( end_node, loop_heads_to_bodies, &loops ); std::map< location_id, bool > loop_heads_ending; @@ -1033,9 +1028,9 @@ fuzzing::iid_node_dependence_props::get_loop_heads_ending( branching_node* end_n } // ------------------------------------------------------------------------------------------------ -void fuzzing::iid_node_dependence_props::compute_dependencies_by_loading( branching_node* end_node, - const loop_head_to_bodies_t& loop_heads_to_bodies, - const loop_endings& loop_heads_ending ) +void iid_node_dependence_props::compute_dependencies_by_loading( branching_node* end_node, + const loop_head_to_bodies_t& loop_heads_to_bodies, + const loop_endings& loop_heads_ending ) { loop_head_to_loaded_bits_t loading_loops; for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { @@ -1095,8 +1090,8 @@ void fuzzing::iid_node_dependence_props::compute_dependencies_by_loading( branch } // ------------------------------------------------------------------------------------------------ -void fuzzing::iid_node_dependence_props::compute_dependencies_by_loops( const loop_head_to_bodies_t& loop_heads_to_bodies, - const loop_endings& loop_heads_ending ) +void iid_node_dependence_props::compute_dependencies_by_loops( const loop_head_to_bodies_t& loop_heads_to_bodies, + const loop_endings& loop_heads_ending ) { for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { if ( !loop_heads_ending.contains( loop_head ) ) { @@ -1119,8 +1114,7 @@ void fuzzing::iid_node_dependence_props::compute_dependencies_by_loops( const lo } // ------------------------------------------------------------------------------------------------ -fuzzing::possible_path -fuzzing::iid_node_dependence_props::generate_path_from_node_counts( const nodes_to_counts& path_counts ) +possible_path iid_node_dependence_props::generate_path_from_node_counts( const nodes_to_counts& path_counts ) { std::map< location_id::id_type, path_node_props > path; for ( const auto& [ id, counts ] : path_counts ) { @@ -1148,7 +1142,7 @@ fuzzing::iid_node_dependence_props::generate_path_from_node_counts( const nodes_ } // ------------------------------------------------------------------------------------------------ -std::set< location_id > fuzzing::iid_node_dependence_props::get_loop_heads( bool include_loading_loops ) +std::set< location_id > iid_node_dependence_props::get_loop_heads( bool include_loading_loops ) { std::set< location_id > loop_heads; for ( const auto& [ loop_head, _ ] : dependencies_by_loops ) { @@ -1166,7 +1160,7 @@ std::set< location_id > fuzzing::iid_node_dependence_props::get_loop_heads( bool // iid_dependencies // ------------------------------------------------------------------------------------------------ -void fuzzing::iid_dependencies::update_non_iid_nodes( sensitivity_analysis& sensitivity ) +void iid_dependencies::update_non_iid_nodes( sensitivity_analysis& sensitivity ) { for ( branching_node* node : sensitivity.get_changed_nodes() ) { if ( node->is_did_branching() ) { @@ -1179,7 +1173,7 @@ void fuzzing::iid_dependencies::update_non_iid_nodes( sensitivity_analysis& sens } // ------------------------------------------------------------------------------------------------ -void fuzzing::iid_dependencies::process_node_dependence( branching_node* node ) +void iid_dependencies::process_node_dependence( branching_node* node ) { TMPROF_BLOCK(); @@ -1191,7 +1185,7 @@ void fuzzing::iid_dependencies::process_node_dependence( branching_node* node ) } // ------------------------------------------------------------------------------------------------ -void fuzzing::iid_dependencies::remove_node_dependence( location_id id ) +void iid_dependencies::remove_node_dependence( location_id id ) { auto it = id_to_equation_map.find( id ); if ( it != id_to_equation_map.end() ) { @@ -1208,13 +1202,13 @@ void fuzzing::iid_dependencies::remove_node_dependence( location_id id ) } // ------------------------------------------------------------------------------------------------ -fuzzing::iid_node_dependence_props& fuzzing::iid_dependencies::get_props( location_id id ) +iid_node_dependence_props& iid_dependencies::get_props( location_id id ) { return id_to_equation_map.at( id ); } // ------------------------------------------------------------------------------------------------ -std::vector< location_id > fuzzing::iid_dependencies::get_iid_nodes() +std::vector< location_id > iid_dependencies::get_iid_nodes() { std::vector< location_id > result; for ( const auto& [ key, _ ] : id_to_equation_map ) { @@ -1226,7 +1220,7 @@ std::vector< location_id > fuzzing::iid_dependencies::get_iid_nodes() } // ------------------------------------------------------------------------------------------------ -std::optional< location_id > fuzzing::iid_dependencies::get_next_iid_node() +std::optional< location_id > iid_dependencies::get_next_iid_node() { bool previous_needs_more_data = false; for ( auto it = id_to_equation_map.rbegin(); it != id_to_equation_map.rend(); ++it ) { @@ -1256,7 +1250,7 @@ std::optional< location_id > fuzzing::iid_dependencies::get_next_iid_node() // non member functions // ------------------------------------------------------------------------------------------------ -std::vector< fuzzing::node_direction > fuzzing::get_path( branching_node* node ) +std::vector< node_direction > get_path( branching_node* node ) { std::vector< node_direction > result; @@ -1271,4 +1265,6 @@ std::vector< fuzzing::node_direction > fuzzing::get_path( branching_node* node ) } return result; -} \ No newline at end of file +} + +} // namespace fuzzing \ No newline at end of file From be962fb536bb1303e127ece039bd75dee764371f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Mon, 27 Jan 2025 15:47:08 +0100 Subject: [PATCH 097/144] feat: add new benchmark for nested big values condition testing --- .../more_iid_condition_nested_big_values.c | 52 ++++++++++++++ .../more_iid_condition_nested_big_values.json | 70 +++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 benchmarks/iid_testing/more_iid_condition_nested_big_values.c create mode 100644 benchmarks/iid_testing/more_iid_condition_nested_big_values.json diff --git a/benchmarks/iid_testing/more_iid_condition_nested_big_values.c b/benchmarks/iid_testing/more_iid_condition_nested_big_values.c new file mode 100644 index 00000000..de13fdd7 --- /dev/null +++ b/benchmarks/iid_testing/more_iid_condition_nested_big_values.c @@ -0,0 +1,52 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + ++i; + } + + if ( k >= 6 ) // ID: 7 + { + if ( k > 8 ) // ID: 8 + { + if ( 12 < k ) // ID: 9 + { + if ( k == 13 ) // ID: 10 + return 1; + } + + if ( k >= 16 ) // ID: 11 + { + if ( k == 30 ) // ID: 12 + return 1; + } + } + } + + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/more_iid_condition_nested_big_values.json b/benchmarks/iid_testing/more_iid_condition_nested_big_values.json new file mode 100644 index 00000000..ab0f9a48 --- /dev/null +++ b/benchmarks/iid_testing/more_iid_condition_nested_big_values.json @@ -0,0 +1,70 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 40638, + "num_covered_branchings": 12, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0, + 11, + 0, + 12, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 6, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file From d522cea1b757fabc61f96f3d8dd57ba6118ceb2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Fri, 31 Jan 2025 16:26:58 +0100 Subject: [PATCH 098/144] feat: add new benchmarks for loading loop tests with varied conditions --- .../iid_testing/loading_loop_2_at_a_time.c | 40 ++++++++++++ .../iid_testing/loading_loop_2_at_a_time.json | 62 +++++++++++++++++++ benchmarks/iid_testing/loading_loop_plus_20.c | 39 ++++++++++++ .../iid_testing/loading_loop_plus_20.json | 60 ++++++++++++++++++ 4 files changed, 201 insertions(+) create mode 100644 benchmarks/iid_testing/loading_loop_2_at_a_time.c create mode 100644 benchmarks/iid_testing/loading_loop_2_at_a_time.json create mode 100644 benchmarks/iid_testing/loading_loop_plus_20.c create mode 100644 benchmarks/iid_testing/loading_loop_plus_20.json diff --git a/benchmarks/iid_testing/loading_loop_2_at_a_time.c b/benchmarks/iid_testing/loading_loop_2_at_a_time.c new file mode 100644 index 00000000..740c61a6 --- /dev/null +++ b/benchmarks/iid_testing/loading_loop_2_at_a_time.c @@ -0,0 +1,40 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n % 2 == 1 ) // ID: 3 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 4 + return -1; + for ( short i = 0; i < n / 2; ++i ) // ID: 5 + { + s[ 2 * i ] = __VERIFIER_nondet_char(); + s[ 2 * i + 1 ] = __VERIFIER_nondet_char(); + } + if ( s[ n - 1 ] != '\0' ) // ID: 6 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + ++i; + } + if ( k == 10 ) // ID: 7 + return 1; + return 0; + } +} diff --git a/benchmarks/iid_testing/loading_loop_2_at_a_time.json b/benchmarks/iid_testing/loading_loop_2_at_a_time.json new file mode 100644 index 00000000..bde808c9 --- /dev/null +++ b/benchmarks/iid_testing/loading_loop_2_at_a_time.json @@ -0,0 +1,62 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 6147, + "num_covered_branchings": 8, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 5, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/loading_loop_plus_20.c b/benchmarks/iid_testing/loading_loop_plus_20.c new file mode 100644 index 00000000..8e0d0935 --- /dev/null +++ b/benchmarks/iid_testing/loading_loop_plus_20.c @@ -0,0 +1,39 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + for ( short i = 0; i < 50; ++i ) // ID: 1 + s[ i ] = '\0'; + + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 2 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 3 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 4 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 5 + return -1; + } + { + int i = 20, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 6 + break; + if ( s[ i ] == 'A' ) // ID: 7 + ++k; + ++i; + } + if ( k == 10 ) // ID: 8 + return 1; + + return 0; + } +} diff --git a/benchmarks/iid_testing/loading_loop_plus_20.json b/benchmarks/iid_testing/loading_loop_plus_20.json new file mode 100644 index 00000000..c4bd44d2 --- /dev/null +++ b/benchmarks/iid_testing/loading_loop_plus_20.json @@ -0,0 +1,60 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 18267, + "num_covered_branchings": 7, + "covered_branchings": [ + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file From c6ad9156ae86b51859e8d6eb55fec3d9586246bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Fri, 31 Jan 2025 16:27:31 +0100 Subject: [PATCH 099/144] feat: update execution counts in loading loop benchmark JSON files --- benchmarks/iid_testing/loading_loop_2_at_a_time.json | 2 +- benchmarks/iid_testing/loading_loop_plus_20.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/iid_testing/loading_loop_2_at_a_time.json b/benchmarks/iid_testing/loading_loop_2_at_a_time.json index bde808c9..02cc3e73 100644 --- a/benchmarks/iid_testing/loading_loop_2_at_a_time.json +++ b/benchmarks/iid_testing/loading_loop_2_at_a_time.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 6147, + "num_executions": 3794, "num_covered_branchings": 8, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/loading_loop_plus_20.json b/benchmarks/iid_testing/loading_loop_plus_20.json index c4bd44d2..c1f5c1ff 100644 --- a/benchmarks/iid_testing/loading_loop_plus_20.json +++ b/benchmarks/iid_testing/loading_loop_plus_20.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 18267, + "num_executions": 8652, "num_covered_branchings": 7, "covered_branchings": [ 2, From d9336e36116096424ad2990f4c89cf69e6be9fb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Fri, 31 Jan 2025 17:43:09 +0100 Subject: [PATCH 100/144] feat: changed dependencies by loading to handle to use loaded bits and sensitive bits --- .../include/fuzzing/iid_vector_analysis.hpp | 34 ++- src/fuzzing/src/iid_vector_analysis.cpp | 240 ++++++++++++++++-- 2 files changed, 249 insertions(+), 25 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index 4aed7753..be3e72e1 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -14,6 +14,14 @@ namespace fuzzing { +template < typename T > +struct mean_counter { + T mean; + int count; + + void add( T value ); +}; + struct node_counts { int left_count; int right_count; @@ -129,17 +137,33 @@ struct node_direction { } }; +struct loaded_bits_props { + natural_32_bit min; + natural_32_bit max; + int loop_count; +}; + +struct loading_body_props { + mean_counter< float > average_bit_size; + natural_32_bit minimal_bit_offset = std::numeric_limits< natural_32_bit >::max(); +}; + struct loop_dependencies_props { bool end_direction; std::set< node_direction > bodies; std::vector< node_counts > previous_counts; }; +struct loading_loops_props : loop_dependencies_props { + mean_counter< float > average_bits_per_loop; + std::map< location_id, loading_body_props > bit_values; +}; + +using loop_head_to_loaded_bits_props = std::unordered_map< location_id, loaded_bits_props >; using loop_ending_to_bodies = std::map< location_id, loop_dependencies_props >; using loop_endings = std::map< location_id, bool >; using loop_head_to_bodies_t = std::unordered_map< location_id, std::unordered_set< location_id > >; -using loop_head_to_loaded_bits_t = std::unordered_map< location_id, std::tuple< natural_32_bit, natural_32_bit > >; using nodes_to_counts = std::map< location_id, node_counts >; struct equation_matrix { @@ -207,6 +231,10 @@ struct iid_node_dependence_props { location_id loop_head_id, int minimum_count, bool use_random = false ); + int compute_loop_count_loading( nodes_to_counts& path_counts, + location_id id, + const std::set< location_id >& loop_heads, + const loading_loops_props& props ); void compute_path_counts_loading( nodes_to_counts& path_counts, const equation& path, const std::set< location_id >& loop_heads ); @@ -225,6 +253,9 @@ struct iid_node_dependence_props { int number_of_vectors ); std::set< node_direction > get_leaf_subsets(); loop_endings get_loop_heads_ending( branching_node* end_node, loop_head_to_bodies_t& loop_heads_to_bodies ); + void compute_loading_loops( branching_node* end_node, + const loop_head_to_bodies_t& loop_heads_to_bodies, + loop_head_to_loaded_bits_props& loading_loops ); void compute_dependencies_by_loading( branching_node* end_node, const loop_head_to_bodies_t& loop_heads_to_bodies, const loop_endings& loop_heads_ending ); @@ -236,6 +267,7 @@ struct iid_node_dependence_props { equation_matrix matrix; loop_ending_to_bodies dependencies_by_loops; loop_ending_to_bodies dependencies_by_loading; + std::map< location_id, loading_loops_props > new_dependencies_by_loading; iid_node_generations_stats stats; }; diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index cccf5322..a73e7965 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -12,6 +12,15 @@ namespace fuzzing { +// mean_counter +// ------------------------------------------------------------------------------------------------ +template < typename T > +inline void fuzzing::mean_counter< T >::add( T value ) +{ + mean = ( mean * count + value ) / ( count + 1 ); + count++; +} + // node_counts // ------------------------------------------------------------------------------------------------ @@ -559,6 +568,8 @@ possible_path iid_node_dependence_props::generate_probabilities() nodes_to_counts node_counts = compute_path_counts( new_leaf_counts.value(), all_leafs ); possible_path path = generate_path_from_node_counts( node_counts ); + // std::cout << "Generated path: " << std::endl << path << std::endl; + return return_path( path ); } @@ -628,6 +639,18 @@ void iid_node_dependence_props::print_dependencies() const std::cout << "- " << "`(" << body << ") → " << loop_head.id << "`" << std::endl; } } + + std::cout << "## Dependencies by loading:" << std::endl; + for ( const auto& [ loop_head, props ] : new_dependencies_by_loading ) { + std::cout << "Loop ID: " << loop_head.id << std::endl; + std::cout << "Loaded bits per loop: " << props.average_bits_per_loop.mean << std::endl; + for ( const auto& body : props.bodies ) { + const auto& bit_props = props.bit_values.at( body.node_id ); + std::cout << "- " << "`(" << body << ") → " << loop_head.id << "`" + << ", Bits: " << bit_props.average_bit_size.mean + << ", offset: " << bit_props.minimal_bit_offset << std::endl; + } + } } // ------------------------------------------------------------------------------------------------ @@ -747,22 +770,58 @@ void iid_node_dependence_props::compute_path_counts_for_nested_loops( nodes_to_c } } +// ------------------------------------------------------------------------------------------------ +int fuzzing::iid_node_dependence_props::compute_loop_count_loading( nodes_to_counts& path_counts, + location_id id, + const std::set< location_id >& loop_heads, + const loading_loops_props& props ) +{ + float loaded_per_loop = props.average_bits_per_loop.mean; + + float average_bits = props.bit_values.at( id ).average_bit_size.mean; + natural_32_bit offset = props.bit_values.at( id ).minimal_bit_offset; + + bool is_loop_head = true; + if ( !loop_heads.contains( id ) ) { + for ( const auto& [ head_id, props ] : dependencies_by_loops ) { + for ( const auto& body : props.bodies ) { + if ( body.node_id == id ) { + id = head_id; + is_loop_head = false; + } + } + } + } + + int total_count = -1; + if ( is_loop_head ) { + total_count = path_counts[ id ].left_count + path_counts[ id ].right_count; + } else { + total_count = path_counts[ id ].get_max_count(); + } + + INVARIANT( total_count > 0 ); + float bits_needed = average_bits * total_count + offset; + + return std::ceil( bits_needed / loaded_per_loop ); +} + // ------------------------------------------------------------------------------------------------ void iid_node_dependence_props::compute_path_counts_loading( nodes_to_counts& path_counts, const equation& path, const std::set< location_id >& loop_heads ) { - // TODO - for ( const auto& [ loading_head, props ] : dependencies_by_loading ) { + for ( const auto& [ loading_head, props ] : new_dependencies_by_loading ) { int loop_count = 0; for ( const auto& body : props.bodies ) { - if ( !path_counts.contains( body.node_id ) || !loop_heads.contains( body.node_id ) ) { + if ( !path_counts.contains( body.node_id ) ) { continue; } - auto& [ left_count, right_count ] = path_counts[ body.node_id ]; - loop_count = std::max( loop_count, left_count + right_count ); + int minimal_count = compute_loop_count_loading( path_counts, body.node_id, loop_heads, props ); + // std::cout << "Minimal count: " << minimal_count << ", for ID: " << body.node_id.id << std::endl; + loop_count = std::max( loop_count, minimal_count ); } if ( props.end_direction ) { @@ -1028,11 +1087,10 @@ std::map< location_id, bool > iid_node_dependence_props::get_loop_heads_ending( } // ------------------------------------------------------------------------------------------------ -void iid_node_dependence_props::compute_dependencies_by_loading( branching_node* end_node, - const loop_head_to_bodies_t& loop_heads_to_bodies, - const loop_endings& loop_heads_ending ) +void fuzzing::iid_node_dependence_props::compute_loading_loops( branching_node* end_node, + const loop_head_to_bodies_t& loop_heads_to_bodies, + loop_head_to_loaded_bits_props& loading_loops ) { - loop_head_to_loaded_bits_t loading_loops; for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { loading_loops[ loop_head ] = { std::numeric_limits< natural_32_bit >::max(), std::numeric_limits< natural_32_bit >::min() }; @@ -1044,33 +1102,59 @@ void iid_node_dependence_props::compute_dependencies_by_loading( branching_node* if ( loop_head.id == node->get_location_id().id ) { natural_32_bit bits_count = node->get_num_stdin_bits(); - auto& [ min, max ] = loading_loops[ loop_head ]; - min = std::min( min, bits_count ); - max = std::max( max, bits_count ); + auto& props = loading_loops[ loop_head ]; + props.min = std::min( props.min, bits_count ); + props.max = std::max( props.max, bits_count ); + props.loop_count++; } } node = node->predecessor; } - node = end_node; + // Remove all loops that did not load any data inside + for ( auto it = loading_loops.begin(); it != loading_loops.end(); ) { + if ( it->second.min == it->second.max ) { + it = loading_loops.erase( it ); + } else { + ++it; + } + } + + // Remove one loop count for branching that ends the loop + for ( auto& [ id, props ] : loading_loops ) { + if ( props.loop_count != 0 ) + props.loop_count--; + } +} + +// ------------------------------------------------------------------------------------------------ +void iid_node_dependence_props::compute_dependencies_by_loading( branching_node* end_node, + const loop_head_to_bodies_t& loop_heads_to_bodies, + const loop_endings& loop_heads_ending ) +{ + loop_head_to_loaded_bits_props loading_loops; + compute_loading_loops( end_node, loop_heads_to_bodies, loading_loops ); + + branching_node* node = end_node; while ( node != nullptr ) { auto node_id = node->get_location_id(); - for ( const auto& [ loop_head, values ] : loading_loops ) { + for ( const auto& [ loop_head, props ] : loading_loops ) { if ( !loop_heads_ending.contains( loop_head ) ) { continue; } bool loop_head_end_direction = loop_heads_ending.at( loop_head ); - const auto& [ min, max ] = values; + auto min = props.min; + auto max = props.max; auto it = std::find_if( node->sensitive_stdin_bits.begin(), node->sensitive_stdin_bits.end(), [ & ]( natural_32_bit bit_index ) { - return bit_index >= min && bit_index <= max; + return bit_index >= min && bit_index < max; } ); if ( it != node->sensitive_stdin_bits.end() ) { @@ -1087,6 +1171,111 @@ void iid_node_dependence_props::compute_dependencies_by_loading( branching_node* node = node->predecessor; } + + + struct loading_body_props_tmp { + natural_32_bit min = std::numeric_limits< natural_32_bit >::max(); + natural_32_bit max = std::numeric_limits< natural_32_bit >::min(); + std::vector< natural_32_bit > sensitive_stdin_bit_counts; + }; + std::map< location_id, std::map< location_id, loading_body_props_tmp > > loop_to_props; + + node = end_node; + + while ( node != nullptr ) { + auto node_id = node->get_location_id(); + + for ( const auto& [ loop_head, props ] : loading_loops ) { + if ( !loop_heads_ending.contains( loop_head ) ) { + continue; + } + + auto min = props.min; + auto max = props.max; + + auto it = std::find_if( node->sensitive_stdin_bits.begin(), + node->sensitive_stdin_bits.end(), + [ & ]( natural_32_bit bit_index ) { + return bit_index >= min && bit_index < max; + } ); + if ( it == node->sensitive_stdin_bits.end() ) + continue; + + loading_body_props_tmp& loop_props = loop_to_props[ loop_head.id ][ node_id ]; + + auto min_it = std::min_element( node->sensitive_stdin_bits.begin(), node->sensitive_stdin_bits.end() ); + if ( min_it != node->sensitive_stdin_bits.end() ) { + loop_props.min = std::min( loop_props.min, *min_it ); + } + + auto max_it = std::max_element( node->sensitive_stdin_bits.begin(), node->sensitive_stdin_bits.end() ); + if ( max_it != node->sensitive_stdin_bits.end() ) { + loop_props.max = std::max( loop_props.max, *max_it ); + } + + loop_props.sensitive_stdin_bit_counts.push_back( node->sensitive_stdin_bits.size() ); + } + + node = node->predecessor; + } + + for ( const auto& [ loop_head_id, body ] : loop_to_props ) { + auto loading_props = loading_loops.at( loop_head_id ); + auto& dependencies = new_dependencies_by_loading[ loop_head_id ]; + + natural_32_bit loaded_bits = loading_props.max - loading_props.min; + double per_loop = double( loaded_bits ) / double( loading_props.loop_count ); + dependencies.average_bits_per_loop.add( per_loop ); + + bool loop_head_end_direction = loop_heads_ending.at( loop_head_id ); + + for ( const auto& [ body_id, props ] : body ) { + auto& body_props = dependencies.bit_values[ body_id ]; + natural_32_bit minimal_offset = props.min - loading_props.min; + INVARIANT( minimal_offset >= 0 ); + body_props.minimal_bit_offset = std::min( body_props.minimal_bit_offset, minimal_offset ); + + for ( const auto& count : props.sensitive_stdin_bit_counts ) { + body_props.average_bit_size.add( count ); + } + + for ( bool direction : { true, false } ) { + node_direction node_id_direction = { body_id, direction }; + + if ( matrix.contains( node_id_direction ) ) { + dependencies.bodies.insert( node_id_direction ); + dependencies.end_direction = loop_head_end_direction; + } + } + } + } + + + { + return; + for ( const auto& [ loop_head_id, body ] : loop_to_props ) { + auto loading_props = loading_loops.at( loop_head_id ); + natural_32_bit loaded_bits = loading_props.max - loading_props.min; + double per_loop = double( loaded_bits ) / double( loading_props.loop_count ); + + std::cout << "------------------------------" << std::endl; + std::cout << "Loop with id: " << loop_head_id.id << std::endl; + std::cout << "Loaded " << loaded_bits << " bits" << std::endl; + std::cout << "Min: " << loading_props.min << ", Max: " << loading_props.max << std::endl; + std::cout << "Loop count: " << loading_props.loop_count << ", bits per loop: " << per_loop + << std::endl; + + for ( const auto& [ id, props ] : body ) { + std::cout << "Body ID: " << id.id << std::endl; + std::cout << "Min: " << props.min << ", Max: " << props.max << std::endl; + std::cout << "Sensitive stdin bit counts: "; + for ( const auto& count : props.sensitive_stdin_bit_counts ) { + std::cout << count << " "; + } + std::cout << std::endl; + } + } + } } // ------------------------------------------------------------------------------------------------ @@ -1125,13 +1314,16 @@ possible_path iid_node_dependence_props::generate_path_from_node_counts( const n bool loop_head_end_direction = false; bool is_loop_head = false; - for ( const auto& map : { dependencies_by_loops, dependencies_by_loading } ) { - auto it = map.find( id ); - if ( it != map.end() ) { - is_loop_head = true; - loop_head_end_direction = it->second.end_direction; - break; - } + auto it_loops = dependencies_by_loops.find( id ); + if ( it_loops != dependencies_by_loops.end() ) { + is_loop_head = true; + loop_head_end_direction = it_loops->second.end_direction; + } + + auto it_loading = new_dependencies_by_loading.find( id ); + if ( it_loading != new_dependencies_by_loading.end() ) { + is_loop_head = true; + loop_head_end_direction = it_loading->second.end_direction; } path_node_props props = { counts, is_loop_head, loop_head_end_direction }; @@ -1150,7 +1342,7 @@ std::set< location_id > iid_node_dependence_props::get_loop_heads( bool include_ } if ( include_loading_loops ) { - for ( const auto& [ loading_head, _ ] : dependencies_by_loading ) { + for ( const auto& [ loading_head, _ ] : new_dependencies_by_loading ) { loop_heads.insert( loading_head ); } } From e73f4c96d72cfefa0bd34e57c4053d657696f936 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Fri, 31 Jan 2025 17:47:40 +0100 Subject: [PATCH 101/144] refactor: remove old dependencies_by_loading, that did not use sensitive bits --- .../include/fuzzing/iid_vector_analysis.hpp | 3 +- src/fuzzing/src/iid_vector_analysis.cpp | 70 ++++--------------- 2 files changed, 16 insertions(+), 57 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index be3e72e1..c8a628b9 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -266,8 +266,7 @@ struct iid_node_dependence_props { equation_matrix matrix; loop_ending_to_bodies dependencies_by_loops; - loop_ending_to_bodies dependencies_by_loading; - std::map< location_id, loading_loops_props > new_dependencies_by_loading; + std::map< location_id, loading_loops_props > dependencies_by_loading; iid_node_generations_stats stats; }; diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index a73e7965..cf0d247d 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -635,13 +635,6 @@ void iid_node_dependence_props::print_dependencies() const std::cout << "## Dependencies by loading:" << std::endl; for ( const auto& [ loop_head, props ] : dependencies_by_loading ) { - for ( const auto& body : props.bodies ) { - std::cout << "- " << "`(" << body << ") → " << loop_head.id << "`" << std::endl; - } - } - - std::cout << "## Dependencies by loading:" << std::endl; - for ( const auto& [ loop_head, props ] : new_dependencies_by_loading ) { std::cout << "Loop ID: " << loop_head.id << std::endl; std::cout << "Loaded bits per loop: " << props.average_bits_per_loop.mean << std::endl; for ( const auto& body : props.bodies ) { @@ -811,7 +804,7 @@ void iid_node_dependence_props::compute_path_counts_loading( nodes_to_counts& pa const equation& path, const std::set< location_id >& loop_heads ) { - for ( const auto& [ loading_head, props ] : new_dependencies_by_loading ) { + for ( const auto& [ loading_head, props ] : dependencies_by_loading ) { int loop_count = 0; for ( const auto& body : props.bodies ) { @@ -902,15 +895,19 @@ nodes_to_counts iid_node_dependence_props::compute_path_counts( const equation& compute_path_counts_loops( path_counts, path, loop_heads ); compute_path_counts_loading( path_counts, path, loop_heads ); - for ( auto* map : { &dependencies_by_loops, &dependencies_by_loading } ) { - for ( auto& [ loop_head, props ] : *map ) { - auto it = path_counts.find( loop_head ); - if ( it != path_counts.end() ) { - props.previous_counts.push_back( it->second ); - } + for ( auto& [ loop_head, props ] : dependencies_by_loops ) { + auto it = path_counts.find( loop_head ); + if ( it != path_counts.end() ) { + props.previous_counts.push_back( it->second ); } } + for ( auto& [ loop_head, props ] : dependencies_by_loading ) { + auto it = path_counts.find( loop_head ); + if ( it != path_counts.end() ) { + props.previous_counts.push_back( it->second ); + } + } return path_counts; } @@ -1138,41 +1135,6 @@ void iid_node_dependence_props::compute_dependencies_by_loading( branching_node* branching_node* node = end_node; - while ( node != nullptr ) { - auto node_id = node->get_location_id(); - - for ( const auto& [ loop_head, props ] : loading_loops ) { - if ( !loop_heads_ending.contains( loop_head ) ) { - continue; - } - - bool loop_head_end_direction = loop_heads_ending.at( loop_head ); - - auto min = props.min; - auto max = props.max; - - auto it = std::find_if( node->sensitive_stdin_bits.begin(), - node->sensitive_stdin_bits.end(), - [ & ]( natural_32_bit bit_index ) { - return bit_index >= min && bit_index < max; - } ); - - if ( it != node->sensitive_stdin_bits.end() ) { - for ( bool direction : { true, false } ) { - node_direction node_id_direction = { node_id, direction }; - - if ( matrix.contains( node_id_direction ) ) { - dependencies_by_loading[ loop_head ].bodies.insert( node_id_direction ); - dependencies_by_loading[ loop_head ].end_direction = loop_head_end_direction; - } - } - } - } - - node = node->predecessor; - } - - struct loading_body_props_tmp { natural_32_bit min = std::numeric_limits< natural_32_bit >::max(); natural_32_bit max = std::numeric_limits< natural_32_bit >::min(); @@ -1180,8 +1142,6 @@ void iid_node_dependence_props::compute_dependencies_by_loading( branching_node* }; std::map< location_id, std::map< location_id, loading_body_props_tmp > > loop_to_props; - node = end_node; - while ( node != nullptr ) { auto node_id = node->get_location_id(); @@ -1221,7 +1181,7 @@ void iid_node_dependence_props::compute_dependencies_by_loading( branching_node* for ( const auto& [ loop_head_id, body ] : loop_to_props ) { auto loading_props = loading_loops.at( loop_head_id ); - auto& dependencies = new_dependencies_by_loading[ loop_head_id ]; + auto& dependencies = dependencies_by_loading[ loop_head_id ]; natural_32_bit loaded_bits = loading_props.max - loading_props.min; double per_loop = double( loaded_bits ) / double( loading_props.loop_count ); @@ -1320,8 +1280,8 @@ possible_path iid_node_dependence_props::generate_path_from_node_counts( const n loop_head_end_direction = it_loops->second.end_direction; } - auto it_loading = new_dependencies_by_loading.find( id ); - if ( it_loading != new_dependencies_by_loading.end() ) { + auto it_loading = dependencies_by_loading.find( id ); + if ( it_loading != dependencies_by_loading.end() ) { is_loop_head = true; loop_head_end_direction = it_loading->second.end_direction; } @@ -1342,7 +1302,7 @@ std::set< location_id > iid_node_dependence_props::get_loop_heads( bool include_ } if ( include_loading_loops ) { - for ( const auto& [ loading_head, _ ] : new_dependencies_by_loading ) { + for ( const auto& [ loading_head, _ ] : dependencies_by_loading ) { loop_heads.insert( loading_head ); } } From fe3f3fb86af4dab2927e64f8d80cb460a72dfb1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 1 Feb 2025 13:12:19 +0100 Subject: [PATCH 102/144] feat: add new benchmarks for input cycle tests with nested loops --- benchmarks/iid_testing/input_cycle_2_loops.c | 54 +++++++++++++++ .../iid_testing/input_cycle_2_loops.json | 63 +++++++++++++++++ .../input_cycle_2_loops_combination.c | 63 +++++++++++++++++ .../input_cycle_2_loops_combination.json | 67 +++++++++++++++++++ 4 files changed, 247 insertions(+) create mode 100644 benchmarks/iid_testing/input_cycle_2_loops.c create mode 100644 benchmarks/iid_testing/input_cycle_2_loops.json create mode 100644 benchmarks/iid_testing/input_cycle_2_loops_combination.c create mode 100644 benchmarks/iid_testing/input_cycle_2_loops_combination.json diff --git a/benchmarks/iid_testing/input_cycle_2_loops.c b/benchmarks/iid_testing/input_cycle_2_loops.c new file mode 100644 index 00000000..d12a2229 --- /dev/null +++ b/benchmarks/iid_testing/input_cycle_2_loops.c @@ -0,0 +1,54 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + short loop_count_1; + loop_count_1 = __VERIFIER_nondet_short(); + short loop_count_2; + loop_count_2 = __VERIFIER_nondet_short(); + + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + if ( loop_count_1 > 20 ) // ID: 5 + return -1; + + if ( loop_count_2 > 20 ) // ID: 6 + return -1; + + int i = 0, k = 0; + + for ( short index_1 = 0; index_1 < loop_count_1; ++index_1 ) { // ID: 7 + for ( short index_2 = 0; index_2 < loop_count_2; ++index_2 ) { // ID: 8 + i = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 9 + break; + if ( s[ i ] == 'A' ) // ID: 10 + ++k; + ++i; + } + } + } + + if ( k == 10 ) // ID: 11 + return 1; + + return 0; + } +} diff --git a/benchmarks/iid_testing/input_cycle_2_loops.json b/benchmarks/iid_testing/input_cycle_2_loops.json new file mode 100644 index 00000000..6da59eb6 --- /dev/null +++ b/benchmarks/iid_testing/input_cycle_2_loops.json @@ -0,0 +1,63 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 34874, + "num_covered_branchings": 11, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0, + 11, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 8, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/input_cycle_2_loops_combination.c b/benchmarks/iid_testing/input_cycle_2_loops_combination.c new file mode 100644 index 00000000..27261d3b --- /dev/null +++ b/benchmarks/iid_testing/input_cycle_2_loops_combination.c @@ -0,0 +1,63 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + short loop_count_1; + loop_count_1 = __VERIFIER_nondet_short(); + short loop_count_2; + loop_count_2 = __VERIFIER_nondet_short(); + + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + if ( loop_count_1 > 20 ) // ID: 5 + return -1; + + if ( loop_count_2 > 20 ) // ID: 6 + return -1; + + int i = 0, k = 0; + + for ( short index_1 = 0; index_1 < loop_count_1; ++index_1 ) { // ID: 7 + for ( short index_2 = 0; index_2 < loop_count_2; ++index_2 ) { // ID: 8 + i = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 9 + break; + if ( s[ i ] == 'A' ) // ID: 10 + ++k; + ++i; + } + } + + i = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 11 + break; + if ( s[ i ] == 'B' ) // ID: 12 + --k; + ++i; + } + } + + if ( k == 10 ) // ID: 13 + return 1; + + return 0; + } +} diff --git a/benchmarks/iid_testing/input_cycle_2_loops_combination.json b/benchmarks/iid_testing/input_cycle_2_loops_combination.json new file mode 100644 index 00000000..76cbab44 --- /dev/null +++ b/benchmarks/iid_testing/input_cycle_2_loops_combination.json @@ -0,0 +1,67 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 14306, + "num_covered_branchings": 13, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0, + 11, + 0, + 12, + 0, + 13, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 10, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file From d489f20fcddd0db70fcc03c38dbdc70e5479f18c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 1 Feb 2025 13:13:53 +0100 Subject: [PATCH 103/144] feat: update execution counts in benchmark JSON files for input cycle tests --- benchmarks/iid_testing/input_cycle_2_loops.json | 2 +- benchmarks/iid_testing/input_cycle_2_loops_combination.json | 2 +- benchmarks/iid_testing/input_cycle_dec_b.json | 2 +- .../iid_testing/more_iid_condition_nested_big_values.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmarks/iid_testing/input_cycle_2_loops.json b/benchmarks/iid_testing/input_cycle_2_loops.json index 6da59eb6..e32906ff 100644 --- a/benchmarks/iid_testing/input_cycle_2_loops.json +++ b/benchmarks/iid_testing/input_cycle_2_loops.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 34874, + "num_executions": 2230, "num_covered_branchings": 11, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/input_cycle_2_loops_combination.json b/benchmarks/iid_testing/input_cycle_2_loops_combination.json index 76cbab44..9cb5b2af 100644 --- a/benchmarks/iid_testing/input_cycle_2_loops_combination.json +++ b/benchmarks/iid_testing/input_cycle_2_loops_combination.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 14306, + "num_executions": 5542, "num_covered_branchings": 13, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/input_cycle_dec_b.json b/benchmarks/iid_testing/input_cycle_dec_b.json index e4ec9c23..be4d51d9 100644 --- a/benchmarks/iid_testing/input_cycle_dec_b.json +++ b/benchmarks/iid_testing/input_cycle_dec_b.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 1470, + "num_executions": 1100, "num_covered_branchings": 10, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/more_iid_condition_nested_big_values.json b/benchmarks/iid_testing/more_iid_condition_nested_big_values.json index ab0f9a48..9b3ce7d9 100644 --- a/benchmarks/iid_testing/more_iid_condition_nested_big_values.json +++ b/benchmarks/iid_testing/more_iid_condition_nested_big_values.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 40638, + "num_executions": 43225, "num_covered_branchings": 12, "covered_branchings": [ 1, From 5a90565ec0fbc532c368379d4583699c54291d43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 1 Feb 2025 14:39:39 +0100 Subject: [PATCH 104/144] feat: add generation if there are not enough data --- .../include/fuzzing/iid_vector_analysis.hpp | 22 +- src/fuzzing/src/iid_vector_analysis.cpp | 188 +++++++++++++----- 2 files changed, 160 insertions(+), 50 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index c8a628b9..f47761b6 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -175,7 +175,8 @@ struct equation_matrix { std::map< equation, int > compute_vectors_with_hits(); std::vector< equation >& get_matrix(); std::optional< equation > get_new_leaf_counts_from_vectors( const std::vector< equation >& vector, - int generated_after_covered ); + int generated_after_covered, + bool generate_more_data ); int get_desired_vector_direction() const; float get_biggest_branching_value() const; @@ -191,7 +192,13 @@ struct equation_matrix { std::set< node_direction > nodes; }; -enum generation_state { NOT_COVERED, GENERATION_MORE, COVERED, GENERATION_DATA_FOR_NEXT_NODE }; +enum generation_state { + STATE_NOT_COVERED, + STATE_GENERATION_MORE, + STATE_COVERED, + STATE_GENERATION_DATA_FOR_NEXT_NODE, + STATE_GENERATING_ARTIFICIAL_DATA +}; struct iid_node_generations_stats { int generation_starts = 0; @@ -206,7 +213,12 @@ struct iid_node_generations_stats { int generated_for_other_node = 0; int generated_for_other_node_max = 0; - generation_state state = generation_state::NOT_COVERED; + int generate_artificial_data = 0; + int generate_artificial_data_max = 0; + + generation_state state = generation_state::STATE_NOT_COVERED; + + generation_state failed_state = generation_state::STATE_GENERATING_ARTIFICIAL_DATA; }; struct iid_node_dependence_props { @@ -220,10 +232,12 @@ struct iid_node_dependence_props { bool is_equal_branching_predicate() const; void print_dependencies() const; - void print_stats() const; + void print_stats( bool only_state = false ) const; private: + void generate_vectors_if_not_enough_data( std::vector< equation >& best_vectors, equation_matrix& submatrix ); + std::optional< std::vector< equation > > get_best_vectors( equation_matrix& submatrix, int number_of_vectors ); possible_path return_empty_path(); possible_path return_path( const possible_path& path ); void compute_path_counts_for_nested_loops( nodes_to_counts& path_counts, diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index cf0d247d..875b5025 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -21,7 +21,6 @@ inline void fuzzing::mean_counter< T >::add( T value ) count++; } - // node_counts // ------------------------------------------------------------------------------------------------ int node_counts::get_max_count() const { return std::max( left_count, right_count ); } @@ -39,6 +38,8 @@ bool path_node_props::get_desired_direction() const bool can_go_left = taken_counts.left_count < computed_counts.left_count; bool can_go_right = taken_counts.right_count < computed_counts.right_count; + INVARIANT( can_go_left || can_go_right ); + if ( can_go_left && can_go_right ) { return rand() % 2 == 0; } @@ -51,7 +52,7 @@ bool path_node_props::get_desired_direction() const return true; } - throw std::runtime_error( "Error: get_desired_direction() called without any possible direction" ); + assert( false ); } // ------------------------------------------------------------------------------------------------ @@ -441,7 +442,8 @@ float equation_matrix::get_biggest_branching_value() const // ------------------------------------------------------------------------------------------------ std::optional< equation > equation_matrix::get_new_leaf_counts_from_vectors( const std::vector< equation >& vectors, - int generation_count_after_covered ) + int generation_count_after_covered, + bool generate_more_data ) { INVARIANT( !vectors.empty() ); INVARIANT( vectors[ 0 ].values.size() == nodes.size() ); @@ -455,10 +457,14 @@ std::optional< equation > equation_matrix::get_new_leaf_counts_from_vectors( con double counts = std::abs( row.best_value ) / std::abs( vector.best_value ); int rounded_counts = static_cast< int >( std::round( counts ) ); - if ( std::abs( counts - double( rounded_counts ) ) > 1e-6 ) { + if ( std::abs( counts - double( rounded_counts ) ) > 1e-6 && !generate_more_data ) { continue; } + if ( generate_more_data ) { + rounded_counts++; + } + equation new_path = vector * rounded_counts + row; if ( !is_equal_sign ) { int limit = std::max( new_path.get_biggest_value() / 4, 1 ); @@ -539,28 +545,28 @@ possible_path iid_node_dependence_props::generate_probabilities() equation_matrix submatrix = matrix.get_submatrix( all_leafs, true ); { - // print_stats(); + // print_stats( true ); // print_dependencies(); // matrix.print_matrix(); // submatrix.print_matrix(); } - std::map< equation, int > vectors = submatrix.compute_vectors_with_hits(); - if ( vectors.empty() ) { - return return_empty_path(); - } + std::optional< std::vector< equation > > best_vectors = get_best_vectors( submatrix, 1 ); - int desired_vector_direction = submatrix.get_desired_vector_direction(); - float biggest_branching_value = submatrix.get_biggest_branching_value(); - std::vector< equation > best_vectors = - compute_best_vectors( vectors, 1, false, desired_vector_direction, biggest_branching_value ); + if ( !best_vectors.has_value() ) { + if ( stats.state != generation_state::STATE_GENERATING_ARTIFICIAL_DATA ) { + return return_empty_path(); + } - if ( best_vectors.empty() ) { - return return_empty_path(); + best_vectors = std::vector< equation >(); + generate_vectors_if_not_enough_data( *best_vectors, submatrix ); } - std::optional< equation > new_leaf_counts = - submatrix.get_new_leaf_counts_from_vectors( best_vectors, stats.generated_after_covered ); + bool generate_more_data = stats.state == generation_state::STATE_GENERATION_MORE || + stats.state == generation_state::STATE_GENERATING_ARTIFICIAL_DATA; + std::optional< equation > new_leaf_counts = submatrix.get_new_leaf_counts_from_vectors( + *best_vectors, stats.generated_after_covered, generate_more_data ); + if ( !new_leaf_counts.has_value() ) { return return_empty_path(); } @@ -588,14 +594,13 @@ void iid_node_dependence_props::process_node( branching_node* end_node ) // ------------------------------------------------------------------------------------------------ bool iid_node_dependence_props::should_generate() const { - return stats.state == generation_state::NOT_COVERED || stats.state == generation_state::GENERATION_MORE || - stats.state == generation_state::GENERATION_DATA_FOR_NEXT_NODE; + return stats.state != generation_state::STATE_COVERED; } // ------------------------------------------------------------------------------------------------ bool iid_node_dependence_props::needs_data_from_other_node( int max_failed_generations_in_row ) const { - if ( stats.state != generation_state::NOT_COVERED ) { + if ( stats.state != generation_state::STATE_NOT_COVERED ) { return false; } @@ -609,9 +614,9 @@ bool iid_node_dependence_props::needs_data_from_other_node( int max_failed_gener // ------------------------------------------------------------------------------------------------ void iid_node_dependence_props::set_as_generating_for_other_node( int minimal_max_generation_for_other_node ) { - INVARIANT( stats.state == generation_state::COVERED ); + INVARIANT( stats.state == generation_state::STATE_COVERED ); - stats.state = generation_state::GENERATION_DATA_FOR_NEXT_NODE; + stats.state = generation_state::STATE_GENERATION_DATA_FOR_NEXT_NODE; stats.generated_for_other_node_max = minimal_max_generation_for_other_node; stats.generated_for_other_node = 0; } @@ -647,29 +652,97 @@ void iid_node_dependence_props::print_dependencies() const } // ------------------------------------------------------------------------------------------------ -void iid_node_dependence_props::print_stats() const +void iid_node_dependence_props::print_stats( bool only_state ) const { switch ( stats.state ) { - case generation_state::NOT_COVERED: - std::cout << "Status: NOT_COVERED" << std::endl; - std::cout << "Failed generations/Total generations: " << stats.failed_generations << "/" - << stats.generation_starts << std::endl; - std::cout << "Failed generations in row: " << stats.failed_generations_in_row << std::endl; + case generation_state::STATE_NOT_COVERED: + std::cout << "Status: STATE_NOT_COVERED" << std::endl; + if ( !only_state ) { + std::cout << "Failed generations/Total generations: " << stats.failed_generations << "/" + << stats.generation_starts << std::endl; + std::cout << "Failed generations in row: " << stats.failed_generations_in_row << std::endl; + } break; - case generation_state::GENERATION_MORE: - std::cout << "Status GENERATION_MORE" << std::endl; - std::cout << "Generated after covered: " << stats.generated_after_covered << "/" - << stats.generated_after_covered_max << std::endl; + case generation_state::STATE_GENERATION_MORE: + std::cout << "Status STATE_GENERATION_MORE" << std::endl; + if ( !only_state ) { + std::cout << "Generated after covered: " << stats.generated_after_covered << "/" + << stats.generated_after_covered_max << std::endl; + } break; - case generation_state::COVERED: std::cout << "COVERED" << std::endl; break; - case generation_state::GENERATION_DATA_FOR_NEXT_NODE: - std::cout << "Status: GENERATION_DATA_FOR_NEXT_NODE" << std::endl; - std::cout << "Generated for other node: " << stats.generated_for_other_node << "/" - << stats.generated_for_other_node_max << std::endl; + case generation_state::STATE_COVERED: std::cout << "STATE_COVERED" << std::endl; break; + case generation_state::STATE_GENERATION_DATA_FOR_NEXT_NODE: { + std::cout << "Status: STATE_GENERATION_DATA_FOR_NEXT_NODE" << std::endl; + if ( !only_state ) + std::cout << "Generated for other node: " << stats.generated_for_other_node << "/" + << stats.generated_for_other_node_max << std::endl; + } break; + case generation_state::STATE_GENERATING_ARTIFICIAL_DATA: + std::cout << "Status: STATE_GENERATING_ARTIFICIAL_DATA" << std::endl; + if ( !only_state ) { + std::cout << "Generated artificial data: " << stats.generate_artificial_data << "/" + << stats.generate_artificial_data_max << std::endl; + } break; } } +// ------------------------------------------------------------------------------------------------ +void fuzzing::iid_node_dependence_props::generate_vectors_if_not_enough_data( std::vector< equation >& best_vectors, + equation_matrix& submatrix ) +{ + best_vectors = std::vector< equation >(); + int desired_direction = submatrix.get_desired_vector_direction(); + std::map< equation, int > vectors = submatrix.compute_vectors_with_hits(); + + auto add_to_best_vectors = [ & ]( std::vector< int > values ) { + if ( desired_direction == 0 ) { + best_vectors.emplace_back( values, 1 ); + best_vectors.emplace_back( values, -1 ); + } else { + best_vectors.emplace_back( values, desired_direction ); + } + }; + + if ( vectors.empty() ) { + std::vector< int > values( submatrix.get_dimensions().second, 1 ); + add_to_best_vectors( values ); + } else { + for ( auto& [ vector, hits ] : vectors ) { + equation modified_vector = vector; + + for ( auto& value : modified_vector.values ) { + if ( value != 0 ) { + value = 1; + } + } + + add_to_best_vectors( modified_vector.values ); + } + } +} + +// ------------------------------------------------------------------------------------------------ +std::optional< std::vector< equation > > +fuzzing::iid_node_dependence_props::get_best_vectors( equation_matrix& submatrix, int number_of_vectors ) +{ + std::map< equation, int > vectors = submatrix.compute_vectors_with_hits(); + if ( vectors.empty() ) { + return std::nullopt; + } + + int desired_vector_direction = submatrix.get_desired_vector_direction(); + float biggest_branching_value = submatrix.get_biggest_branching_value(); + std::vector< equation > best_vectors = + compute_best_vectors( vectors, number_of_vectors, false, desired_vector_direction, biggest_branching_value ); + + if ( best_vectors.empty() ) { + return std::nullopt; + } + + return best_vectors; +} + // ------------------------------------------------------------------------------------------------ possible_path iid_node_dependence_props::return_empty_path() { @@ -684,19 +757,29 @@ possible_path iid_node_dependence_props::return_path( const possible_path& path stats.failed_generations_in_row = 0; stats.successful_generations++; - if ( stats.state == generation_state::GENERATION_MORE ) { + if ( stats.state == generation_state::STATE_GENERATION_MORE ) { stats.generated_after_covered++; if ( stats.generated_after_covered > stats.generated_after_covered_max ) { - stats.state = generation_state::COVERED; + stats.state = generation_state::STATE_COVERED; } } - if ( stats.state == generation_state::GENERATION_DATA_FOR_NEXT_NODE ) { + if ( stats.state == generation_state::STATE_GENERATION_DATA_FOR_NEXT_NODE ) { stats.generated_for_other_node++; if ( stats.generated_for_other_node > stats.generated_for_other_node_max ) { - stats.state = generation_state::COVERED; + stats.state = generation_state::STATE_COVERED; + stats.generated_for_other_node = 0; + } + } + + if ( stats.state == generation_state::STATE_GENERATING_ARTIFICIAL_DATA ) { + stats.generate_artificial_data++; + + if ( stats.generate_artificial_data > stats.generate_artificial_data_max ) { + stats.state = generation_state::STATE_NOT_COVERED; + stats.generate_artificial_data = 0; } } @@ -813,7 +896,6 @@ void iid_node_dependence_props::compute_path_counts_loading( nodes_to_counts& pa } int minimal_count = compute_loop_count_loading( path_counts, body.node_id, loop_heads, props ); - // std::cout << "Minimal count: " << minimal_count << ", for ID: " << body.node_id.id << std::endl; loop_count = std::max( loop_count, minimal_count ); } @@ -1342,10 +1424,10 @@ void iid_dependencies::remove_node_dependence( location_id id ) auto it = id_to_equation_map.find( id ); if ( it != id_to_equation_map.end() ) { iid_node_generations_stats& stats = it->second.get_generations_stats(); - stats.state = generation_state::COVERED; + stats.state = generation_state::STATE_COVERED; if ( generate_more_data_after_coverage && !it->second.is_equal_branching_predicate() ) { - stats.state = generation_state::GENERATION_MORE; + stats.state = generation_state::STATE_GENERATION_MORE; int max_generation_after_covered = std::max( minimal_max_generation_after_covered, stats.successful_generations / 2 ); stats.generated_after_covered_max = max_generation_after_covered; @@ -1384,8 +1466,22 @@ std::optional< location_id > iid_dependencies::get_next_iid_node() } if ( props.needs_data_from_other_node( max_failed_generations_in_row ) ) { - previous_needs_more_data = true; props.get_generations_stats().failed_generations_in_row = 0; + + if ( std::next( it ) == id_to_equation_map.rend() ) { + props.get_generations_stats().state = generation_state::STATE_GENERATING_ARTIFICIAL_DATA; + props.get_generations_stats().generate_artificial_data_max = 5; + } else { + if ( props.get_generations_stats().failed_state == + generation_state::STATE_GENERATION_DATA_FOR_NEXT_NODE ) { + props.get_generations_stats().state = generation_state::STATE_GENERATING_ARTIFICIAL_DATA; + props.get_generations_stats().generate_artificial_data_max = 5; + props.get_generations_stats().failed_state = generation_state::STATE_GENERATING_ARTIFICIAL_DATA; + } else { + previous_needs_more_data = true; + props.get_generations_stats().failed_state = generation_state::STATE_GENERATION_DATA_FOR_NEXT_NODE; + } + } } } @@ -1419,4 +1515,4 @@ std::vector< node_direction > get_path( branching_node* node ) return result; } -} // namespace fuzzing \ No newline at end of file +} // namespace fuzzing From 7f5bf5a3e0cae7e28f0ad2b4575fe0e4eb1ffbf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 1 Feb 2025 22:22:50 +0100 Subject: [PATCH 105/144] feat: update execution counts in benchmark JSON files for IID testing --- benchmarks/iid_testing/different_predicates_in_iid.json | 6 +++--- benchmarks/iid_testing/input_cycle_2_loops.json | 2 +- benchmarks/iid_testing/input_cycle_2_loops_combination.json | 2 +- benchmarks/iid_testing/more_iid_condition_nested.json | 2 +- .../iid_testing/more_iid_condition_nested_big_values.json | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/benchmarks/iid_testing/different_predicates_in_iid.json b/benchmarks/iid_testing/different_predicates_in_iid.json index 39873d7f..43554337 100644 --- a/benchmarks/iid_testing/different_predicates_in_iid.json +++ b/benchmarks/iid_testing/different_predicates_in_iid.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 12981, + "num_executions": 15347, "num_covered_branchings": 19, "covered_branchings": [ 1, @@ -60,12 +60,12 @@ ], "output_statistics": { "bitshare_analysis": { - "num_generated_tests": 3, + "num_generated_tests": 5, "num_crashes": 0, "num_boundary_violations": 0 }, "sensitivity_analysis": { - "num_generated_tests": 6, + "num_generated_tests": 4, "num_crashes": 0, "num_boundary_violations": 0 }, diff --git a/benchmarks/iid_testing/input_cycle_2_loops.json b/benchmarks/iid_testing/input_cycle_2_loops.json index e32906ff..8d080249 100644 --- a/benchmarks/iid_testing/input_cycle_2_loops.json +++ b/benchmarks/iid_testing/input_cycle_2_loops.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 2230, + "num_executions": 1912, "num_covered_branchings": 11, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/input_cycle_2_loops_combination.json b/benchmarks/iid_testing/input_cycle_2_loops_combination.json index 9cb5b2af..faa90799 100644 --- a/benchmarks/iid_testing/input_cycle_2_loops_combination.json +++ b/benchmarks/iid_testing/input_cycle_2_loops_combination.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 5542, + "num_executions": 2855, "num_covered_branchings": 13, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/more_iid_condition_nested.json b/benchmarks/iid_testing/more_iid_condition_nested.json index 539b45e1..ce2829ba 100644 --- a/benchmarks/iid_testing/more_iid_condition_nested.json +++ b/benchmarks/iid_testing/more_iid_condition_nested.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 7836, + "num_executions": 7883, "num_covered_branchings": 9, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/more_iid_condition_nested_big_values.json b/benchmarks/iid_testing/more_iid_condition_nested_big_values.json index 9b3ce7d9..66cb4f21 100644 --- a/benchmarks/iid_testing/more_iid_condition_nested_big_values.json +++ b/benchmarks/iid_testing/more_iid_condition_nested_big_values.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 43225, + "num_executions": 42668, "num_covered_branchings": 12, "covered_branchings": [ 1, From f4915a28c439bbd2a0d17308bbaf2d9b91211df2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 1 Feb 2025 22:28:58 +0100 Subject: [PATCH 106/144] feat: feat: update leaf computation and state management --- .../include/fuzzing/iid_vector_analysis.hpp | 29 ++-- src/fuzzing/src/iid_vector_analysis.cpp | 131 ++++++++++-------- 2 files changed, 94 insertions(+), 66 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index f47761b6..49564752 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -109,6 +109,7 @@ struct equation { bool operator==( const equation& other ) const = default; equation add_to_positive( int value ) const; + equation add_to_values( const equation& other ) const; int get_vector_size() const; int get_one_way_branching_count() const; int get_biggest_value() const; @@ -116,6 +117,7 @@ struct equation { bool same_values() const; bool is_linear_dependent( const equation& other ) const; + friend std::ostream& operator<<( std::ostream& os, const equation& eq ) { for ( int i = 0; i < eq.values.size(); ++i ) { @@ -194,12 +196,14 @@ struct equation_matrix { enum generation_state { STATE_NOT_COVERED, + STATE_GENERATING_ARTIFICIAL_DATA, STATE_GENERATION_MORE, STATE_COVERED, - STATE_GENERATION_DATA_FOR_NEXT_NODE, - STATE_GENERATING_ARTIFICIAL_DATA + STATE_GENERATION_DATA_FOR_NEXT_NODE }; +enum failed_generation_method { METHOD_GENERATE_FROM_OTHER_NODE, METHOD_GENERATE_ARTIFICIAL_DATA }; + struct iid_node_generations_stats { int generation_starts = 0; int successful_generations = 0; @@ -218,7 +222,7 @@ struct iid_node_generations_stats { generation_state state = generation_state::STATE_NOT_COVERED; - generation_state failed_state = generation_state::STATE_GENERATING_ARTIFICIAL_DATA; + failed_generation_method last_failed_method = METHOD_GENERATE_ARTIFICIAL_DATA; }; struct iid_node_dependence_props { @@ -227,8 +231,10 @@ struct iid_node_dependence_props { iid_node_generations_stats& get_generations_stats() { return stats; } bool should_generate() const; - bool needs_data_from_other_node( int max_failed_generations_in_row ) const; + bool too_much_failed_in_row( int max_failed_generations_in_row ) const; void set_as_generating_for_other_node( int minimal_max_generation_for_other_node ); + void set_as_generating_artificial_data( int minimal_max_generation_artificial_data ); + failed_generation_method get_method_for_failed_generation( bool is_first ); bool is_equal_branching_predicate() const; void print_dependencies() const; @@ -297,11 +303,16 @@ struct iid_dependencies { std::map< location_id, iid_node_dependence_props > id_to_equation_map; std::set< location_id > non_iid_nodes; - // Settings - bool generate_more_data_after_coverage = true; - int minimal_max_generation_after_covered = 10; - int max_failed_generations_in_row = 3; - int minimal_max_generation_for_other_node = 10; +public: + // Configurations + inline static bool random_nested_loops = false; + inline static bool random_direction_in_path = true; + inline static bool generate_more_data_after_coverage = true; + inline static int minimal_max_generation_after_covered = 10; + inline static int max_failed_generations_in_row = 2; + inline static int minimal_max_generation_for_other_node = 10; + inline static int minimal_max_generation_artificial_data = 5; + inline static float percentage_to_add_to_path = 0.4; }; std::vector< node_direction > get_path( branching_node* node ); diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 875b5025..c417927d 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -40,7 +40,7 @@ bool path_node_props::get_desired_direction() const INVARIANT( can_go_left || can_go_right ); - if ( can_go_left && can_go_right ) { + if ( can_go_left && can_go_right && iid_dependencies::random_direction_in_path ) { return rand() % 2 == 0; } @@ -210,6 +210,19 @@ equation equation::add_to_positive( int value ) const return { new_values, best_value }; } +// ------------------------------------------------------------------------------------------------ +equation fuzzing::equation::add_to_values( const equation& other ) const +{ + INVARIANT( values.size() == other.values.size() ); + + std::vector< int > new_values; + for ( int i = 0; i < values.size(); ++i ) { + new_values.push_back( values[ i ] + other.values[ i ] ); + } + + return { new_values, best_value }; +} + // ------------------------------------------------------------------------------------------------ int equation::get_vector_size() const { @@ -457,19 +470,19 @@ std::optional< equation > equation_matrix::get_new_leaf_counts_from_vectors( con double counts = std::abs( row.best_value ) / std::abs( vector.best_value ); int rounded_counts = static_cast< int >( std::round( counts ) ); - if ( std::abs( counts - double( rounded_counts ) ) > 1e-6 && !generate_more_data ) { + if ( std::abs( counts - double( rounded_counts ) ) > 1e-8 && !generate_more_data ) { continue; } - if ( generate_more_data ) { - rounded_counts++; - } - equation new_path = vector * rounded_counts + row; + if ( !is_equal_sign ) { - int limit = std::max( new_path.get_biggest_value() / 4, 1 ); - new_path = new_path.add_to_positive( 1 + rand() % limit ); - // new_path = new_path.add_to_positive( 1 + limit ); + new_path = new_path.add_to_values( vector ); + if ( generate_more_data ) { + double best_value = new_path.best_value; + new_path = new_path * ( 1 + iid_dependencies::percentage_to_add_to_path ); + new_path.best_value = best_value; + } } if ( !new_path.is_any_negative() ) { @@ -563,6 +576,7 @@ possible_path iid_node_dependence_props::generate_probabilities() } bool generate_more_data = stats.state == generation_state::STATE_GENERATION_MORE || + stats.state == generation_state::STATE_GENERATION_DATA_FOR_NEXT_NODE || stats.state == generation_state::STATE_GENERATING_ARTIFICIAL_DATA; std::optional< equation > new_leaf_counts = submatrix.get_new_leaf_counts_from_vectors( *best_vectors, stats.generated_after_covered, generate_more_data ); @@ -598,7 +612,7 @@ bool iid_node_dependence_props::should_generate() const } // ------------------------------------------------------------------------------------------------ -bool iid_node_dependence_props::needs_data_from_other_node( int max_failed_generations_in_row ) const +bool iid_node_dependence_props::too_much_failed_in_row( int max_failed_generations_in_row ) const { if ( stats.state != generation_state::STATE_NOT_COVERED ) { return false; @@ -621,6 +635,42 @@ void iid_node_dependence_props::set_as_generating_for_other_node( int minimal_ma stats.generated_for_other_node = 0; } +// ------------------------------------------------------------------------------------------------ +void fuzzing::iid_node_dependence_props::set_as_generating_artificial_data( int minimal_max_generation_artificial_data ) +{ + INVARIANT( stats.state == generation_state::STATE_NOT_COVERED ); + + stats.state = generation_state::STATE_GENERATING_ARTIFICIAL_DATA; + stats.generate_artificial_data_max = minimal_max_generation_artificial_data; + stats.generate_artificial_data = 0; +} + +// ------------------------------------------------------------------------------------------------ +failed_generation_method fuzzing::iid_node_dependence_props::get_method_for_failed_generation( bool is_first ) +{ + failed_generation_method new_method; + + if ( is_first ) { + new_method = failed_generation_method::METHOD_GENERATE_ARTIFICIAL_DATA; + } else { + switch ( stats.last_failed_method ) { + case failed_generation_method::METHOD_GENERATE_ARTIFICIAL_DATA: + new_method = failed_generation_method::METHOD_GENERATE_FROM_OTHER_NODE; + break; + case failed_generation_method::METHOD_GENERATE_FROM_OTHER_NODE: + new_method = failed_generation_method::METHOD_GENERATE_ARTIFICIAL_DATA; + break; + } + } + + if ( new_method == failed_generation_method::METHOD_GENERATE_ARTIFICIAL_DATA ) { + set_as_generating_artificial_data( iid_dependencies::minimal_max_generation_artificial_data ); + } + + stats.last_failed_method = new_method; + return new_method; +} + // ------------------------------------------------------------------------------------------------ bool iid_node_dependence_props::is_equal_branching_predicate() const { @@ -930,7 +980,11 @@ void iid_node_dependence_props::compute_path_counts_loops( nodes_to_counts& path continue; } - compute_path_counts_for_nested_loops( path_counts, child_loop_counts, loop_head, non_loop_child_max_count ); + compute_path_counts_for_nested_loops( path_counts, + child_loop_counts, + loop_head, + non_loop_child_max_count, + iid_dependencies::random_nested_loops ); } } @@ -1055,8 +1109,6 @@ std::vector< equation > iid_node_dependence_props::compute_best_vectors( const s return a.first.get_vector_size() < b.first.get_vector_size(); } ); best_vectors.push_back( it->first ); - // std::cout << "Chosen vector : " << it->first << std::endl; - // std::cout << "Original vector: " << sorted_vectors[ i ].first << std::endl; } else { best_vectors.push_back( sorted_vectors[ i ].first ); } @@ -1291,33 +1343,6 @@ void iid_node_dependence_props::compute_dependencies_by_loading( branching_node* } } } - - - { - return; - for ( const auto& [ loop_head_id, body ] : loop_to_props ) { - auto loading_props = loading_loops.at( loop_head_id ); - natural_32_bit loaded_bits = loading_props.max - loading_props.min; - double per_loop = double( loaded_bits ) / double( loading_props.loop_count ); - - std::cout << "------------------------------" << std::endl; - std::cout << "Loop with id: " << loop_head_id.id << std::endl; - std::cout << "Loaded " << loaded_bits << " bits" << std::endl; - std::cout << "Min: " << loading_props.min << ", Max: " << loading_props.max << std::endl; - std::cout << "Loop count: " << loading_props.loop_count << ", bits per loop: " << per_loop - << std::endl; - - for ( const auto& [ id, props ] : body ) { - std::cout << "Body ID: " << id.id << std::endl; - std::cout << "Min: " << props.min << ", Max: " << props.max << std::endl; - std::cout << "Sensitive stdin bit counts: "; - for ( const auto& count : props.sensitive_stdin_bit_counts ) { - std::cout << count << " "; - } - std::cout << std::endl; - } - } - } } // ------------------------------------------------------------------------------------------------ @@ -1426,9 +1451,9 @@ void iid_dependencies::remove_node_dependence( location_id id ) iid_node_generations_stats& stats = it->second.get_generations_stats(); stats.state = generation_state::STATE_COVERED; - if ( generate_more_data_after_coverage && !it->second.is_equal_branching_predicate() ) { + if ( iid_dependencies::generate_more_data_after_coverage && !it->second.is_equal_branching_predicate() ) { stats.state = generation_state::STATE_GENERATION_MORE; - int max_generation_after_covered = std::max( minimal_max_generation_after_covered, + int max_generation_after_covered = std::max( iid_dependencies::minimal_max_generation_after_covered, stats.successful_generations / 2 ); stats.generated_after_covered_max = max_generation_after_covered; } @@ -1461,26 +1486,18 @@ std::optional< location_id > iid_dependencies::get_next_iid_node() iid_node_dependence_props& props = it->second; if ( previous_needs_more_data ) { - props.set_as_generating_for_other_node( minimal_max_generation_for_other_node ); + props.set_as_generating_for_other_node( iid_dependencies::minimal_max_generation_for_other_node ); previous_needs_more_data = false; } - if ( props.needs_data_from_other_node( max_failed_generations_in_row ) ) { + if ( props.too_much_failed_in_row( iid_dependencies::max_failed_generations_in_row ) ) { props.get_generations_stats().failed_generations_in_row = 0; + bool is_first = std::next( it ) == id_to_equation_map.rend(); - if ( std::next( it ) == id_to_equation_map.rend() ) { - props.get_generations_stats().state = generation_state::STATE_GENERATING_ARTIFICIAL_DATA; - props.get_generations_stats().generate_artificial_data_max = 5; - } else { - if ( props.get_generations_stats().failed_state == - generation_state::STATE_GENERATION_DATA_FOR_NEXT_NODE ) { - props.get_generations_stats().state = generation_state::STATE_GENERATING_ARTIFICIAL_DATA; - props.get_generations_stats().generate_artificial_data_max = 5; - props.get_generations_stats().failed_state = generation_state::STATE_GENERATING_ARTIFICIAL_DATA; - } else { - previous_needs_more_data = true; - props.get_generations_stats().failed_state = generation_state::STATE_GENERATION_DATA_FOR_NEXT_NODE; - } + failed_generation_method method = props.get_method_for_failed_generation( is_first ); + + if ( method == failed_generation_method::METHOD_GENERATE_FROM_OTHER_NODE ) { + previous_needs_more_data = true; } } } From cbb2ae2d627a5fee64e0a35b382d221d4d54156c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 8 Feb 2025 23:41:44 +0100 Subject: [PATCH 107/144] feat: support for multiple loop heads in one loop --- .../include/fuzzing/iid_vector_analysis.hpp | 27 +- src/fuzzing/src/iid_vector_analysis.cpp | 347 +++++++++++++++--- 2 files changed, 317 insertions(+), 57 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index 49564752..f1406ac6 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -161,6 +161,31 @@ struct loading_loops_props : loop_dependencies_props { std::map< location_id, loading_body_props > bit_values; }; +struct loading_loop_head_properties { + int count; +}; + +struct loading_loop_properties { + std::map< node_direction, loading_loop_head_properties > heads; + + std::optional< node_direction > chosen_loop_head; + std::set< node_direction > bodies; + + bool is_same( const std::unordered_set< location_id >& other_ids ) const; + std::unordered_set< location_id > get_all_ids() const; + std::unordered_set< location_id > get_loop_head_ids() const; + std::unordered_set< location_id > get_body_ids() const; + location_id get_smallest_loop_head_id() const; + void set_chosen_loop_head(); +}; + +struct dependencies_by_loops_t { + std::vector< loading_loop_properties > loops; + + loading_loop_properties& get_props( const std::unordered_set< location_id >& ids, location_id loop_head_id ); + void merge_properties(); + loading_loop_properties& get_props_by_loop_head_id( location_id loop_head_id ); +}; using loop_head_to_loaded_bits_props = std::unordered_map< location_id, loaded_bits_props >; using loop_ending_to_bodies = std::map< location_id, loop_dependencies_props >; @@ -285,7 +310,7 @@ struct iid_node_dependence_props { std::set< location_id > get_loop_heads( bool include_loading_loops = true ); equation_matrix matrix; - loop_ending_to_bodies dependencies_by_loops; + dependencies_by_loops_t dependencies_by_loops; std::map< location_id, loading_loops_props > dependencies_by_loading; iid_node_generations_stats stats; diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index c417927d..17cd2411 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -102,10 +102,10 @@ bool path_node_props::get_preferred_direction_loop_head() const auto is_depleted = []( int computed, int taken ) { return computed == taken; }; if ( !loop_head_end_direction ) { - INVARIANT( computed_counts.left_count == 1 ); + // INVARIANT( computed_counts.left_count == 1 ); return !is_depleted( computed_counts.right_count, taken_counts.right_count ); } else { - INVARIANT( computed_counts.right_count == 1 ); + // INVARIANT( computed_counts.right_count == 1 ); return is_depleted( computed_counts.left_count, taken_counts.left_count ); } } @@ -300,7 +300,184 @@ auto node_direction::operator<=>( node_direction const& other ) const return branching_direction <=> other.branching_direction; } -// equation_matrix +// loading_loop_properties +// ------------------------------------------------------------------------------------------------ +bool fuzzing::loading_loop_properties::is_same( const std::unordered_set< location_id >& other_ids ) const +{ + return get_all_ids() == other_ids; +} + +// ------------------------------------------------------------------------------------------------ +std::unordered_set< location_id > fuzzing::loading_loop_properties::get_all_ids() const +{ + std::unordered_set< location_id > all_ids; + + for ( const auto& [ head, props ] : heads ) { + all_ids.insert( head.node_id ); + } + + for ( const auto& body : bodies ) { + all_ids.insert( body.node_id ); + } + + return all_ids; +} + +// ------------------------------------------------------------------------------------------------ +std::unordered_set< location_id > fuzzing::loading_loop_properties::get_loop_head_ids() const +{ + std::unordered_set< location_id > loop_head_ids; + + for ( const auto& [ head, props ] : heads ) { + loop_head_ids.insert( head.node_id ); + } + + return loop_head_ids; +} + +// ------------------------------------------------------------------------------------------------ +std::unordered_set< location_id > fuzzing::loading_loop_properties::get_body_ids() const +{ + std::unordered_set< location_id > body_ids; + + for ( const auto& body : bodies ) { + body_ids.insert( body.node_id ); + } + + return body_ids; +} + +// ------------------------------------------------------------------------------------------------ +location_id fuzzing::loading_loop_properties::get_smallest_loop_head_id() const +{ + location_id smallest_id = heads.begin()->first.node_id; + + for ( const auto& [ head, props ] : heads ) { + if ( head.node_id < smallest_id ) { + smallest_id = head.node_id; + } + } + + return smallest_id; +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::loading_loop_properties::set_chosen_loop_head() +{ + for ( const auto& [ head, props ] : heads ) { + if ( !chosen_loop_head.has_value() ) { + chosen_loop_head = head; + } + + if ( props.count > heads.at( *chosen_loop_head ).count ) { + chosen_loop_head = head; + } + } +} + +// dependencies_by_loops_t +// ------------------------------------------------------------------------------------------------ +loading_loop_properties& fuzzing::dependencies_by_loops_t::get_props( const std::unordered_set< location_id >& ids, + location_id loop_head_id ) +{ + for ( loading_loop_properties& loop : loops ) { + for ( const auto& [ head, props ] : loop.heads ) { + if ( head.node_id == loop_head_id ) { + return loop; + } + } + + if ( loop.is_same( ids ) ) { + return loop; + } + } + + loops.emplace_back(); + return loops.back(); +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::dependencies_by_loops_t::merge_properties() +{ + for ( auto it = loops.begin(); it != loops.end(); it++ ) { + std::set< location_id > head_ids; + for ( const auto& [ head, props ] : it->heads ) { + head_ids.insert( head.node_id ); + } + + for ( auto body_it = it->bodies.begin(); body_it != it->bodies.end(); ) { + if ( head_ids.contains( body_it->node_id ) ) { + body_it = it->bodies.erase( body_it ); + } else { + ++body_it; + } + } + } + + for ( auto it_1 = loops.begin(); it_1 != loops.end(); it_1++ ) { + for ( auto it_2 = loops.begin(); it_2 != loops.end(); ) { + if ( it_1 == it_2 ) { + ++it_2; + continue; + } + + if ( it_1->is_same( it_2->get_all_ids() ) ) { + for ( const auto& [ head, props ] : it_2->heads ) { + it_1->heads[ head ].count += props.count; + } + + it_1->bodies.insert( it_2->bodies.begin(), it_2->bodies.end() ); + it_2 = loops.erase( it_2 ); + + it_1 = loops.begin(); + it_2 = loops.begin(); + } else { + ++it_2; + } + } + } + + for ( auto it = loops.begin(); it != loops.end(); it++ ) { + std::set< location_id > head_ids; + for ( const auto& [ head, props ] : it->heads ) { + head_ids.insert( head.node_id ); + } + + for ( auto body_it = it->bodies.begin(); body_it != it->bodies.end(); ) { + if ( head_ids.contains( body_it->node_id ) ) { + body_it = it->bodies.erase( body_it ); + } else { + ++body_it; + } + } + } + + + // Not sure if deleting loops with only loop heads is a good idea + for ( auto it = loops.begin(); it != loops.end(); ) { + if ( it->bodies.empty() ) { + it = loops.erase( it ); + } else { + ++it; + } + } +} + +// ------------------------------------------------------------------------------------------------ +loading_loop_properties& fuzzing::dependencies_by_loops_t::get_props_by_loop_head_id( location_id loop_head_id ) +{ + for ( loading_loop_properties& loop : loops ) { + for ( const auto& [ head, props ] : loop.heads ) { + if ( head.node_id == loop_head_id ) { + return loop; + } + } + } + + throw std::runtime_error( "Loop head not found" ); +} + +// equation_matrix // ------------------------------------------------------------------------------------------------ equation_matrix equation_matrix::get_submatrix( std::set< node_direction > const& subset, bool unique ) const { @@ -551,10 +728,13 @@ void equation_matrix::recompute_matrix() // ------------------------------------------------------------------------------------------------ possible_path iid_node_dependence_props::generate_probabilities() { - TMPROF_BLOCK(); + std::set< node_direction > all_leafs = get_leaf_subsets(); + if ( all_leafs.empty() || dependencies_by_loops.loops.empty() ) { + return {}; + } + TMPROF_BLOCK(); stats.generation_starts++; - std::set< node_direction > all_leafs = get_leaf_subsets(); equation_matrix submatrix = matrix.get_submatrix( all_leafs, true ); { @@ -682,9 +862,15 @@ void iid_node_dependence_props::print_dependencies() const { std::cout << "# Dependencies:" << std::endl; std::cout << "## Dependencies by loops:" << std::endl; - for ( const auto& [ loop_head, props ] : dependencies_by_loops ) { - for ( const auto& body : props.bodies ) { - std::cout << "- " << "`(" << body << ") → " << loop_head.id << "`" << std::endl; + for ( const auto& loop : dependencies_by_loops.loops ) { + std::cout << "Loop heads:" << std::endl; + for ( const auto& [ head, head_props ] : loop.heads ) { + std::cout << "- " << head << " (" << head_props.count << ")" << std::endl; + } + + std::cout << "Loop bodies:" << std::endl; + for ( const auto& body : loop.bodies ) { + std::cout << "- " << body << std::endl; } } @@ -845,11 +1031,6 @@ void iid_node_dependence_props::compute_path_counts_for_nested_loops( nodes_to_c { INVARIANT( !child_loop_counts.empty() ); - int previous_count = 1; - if ( dependencies_by_loops.at( loop_head_id ).previous_counts.size() > 0 ) { - previous_count = dependencies_by_loops.at( loop_head_id ).previous_counts.back().get_max_count(); - } - int max_child_count = std::max_element( child_loop_counts.begin(), child_loop_counts.end(), []( const auto& a, const auto& b ) { return a.second < b.second; @@ -880,19 +1061,27 @@ void iid_node_dependence_props::compute_path_counts_for_nested_loops( nodes_to_c *possible_counts.rbegin(); for ( auto& [ node_id, count ] : child_loop_counts ) { - auto& [ left_count, right_count ] = path_counts[ node_id ]; + loading_loop_properties& props = dependencies_by_loops.get_props_by_loop_head_id( node_id ); - if ( dependencies_by_loops.at( node_id ).end_direction ) { - left_count = count / highest_count; - } else { - right_count = count / highest_count; + for ( auto& [ head, _ ] : props.heads ) { + auto& [ left_count, right_count ] = path_counts[ head.node_id ]; + + if ( head.branching_direction ) { + left_count = count / highest_count; + } else { + right_count = count / highest_count; + } } } - if ( dependencies_by_loops.at( loop_head_id ).end_direction ) { - path_counts[ loop_head_id ] = { highest_count, 1 }; - } else { - path_counts[ loop_head_id ] = { 1, highest_count }; + + loading_loop_properties& props = dependencies_by_loops.get_props_by_loop_head_id( loop_head_id ); + for ( auto& [ head, _ ] : props.heads ) { + if ( head.branching_direction ) { + path_counts[ head.node_id ] = { highest_count, 1 }; + } else { + path_counts[ head.node_id ] = { 1, highest_count }; + } } } @@ -909,10 +1098,10 @@ int fuzzing::iid_node_dependence_props::compute_loop_count_loading( nodes_to_cou bool is_loop_head = true; if ( !loop_heads.contains( id ) ) { - for ( const auto& [ head_id, props ] : dependencies_by_loops ) { - for ( const auto& body : props.bodies ) { + for ( const auto& loop : dependencies_by_loops.loops ) { + for ( const auto& body : loop.bodies ) { if ( body.node_id == id ) { - id = head_id; + id = ( *loop.chosen_loop_head ).node_id; is_loop_head = false; } } @@ -962,7 +1151,7 @@ void iid_node_dependence_props::compute_path_counts_loops( nodes_to_counts& path const equation& path, const std::set< location_id >& loop_heads ) { - for ( const auto& [ loop_head, props ] : std::ranges::views::reverse( dependencies_by_loops ) ) { + for ( const auto& props : std::ranges::views::reverse( dependencies_by_loops.loops ) ) { std::map< location_id, int > child_loop_counts; int non_loop_child_max_count = 1; @@ -982,7 +1171,7 @@ void iid_node_dependence_props::compute_path_counts_loops( nodes_to_counts& path compute_path_counts_for_nested_loops( path_counts, child_loop_counts, - loop_head, + ( *props.chosen_loop_head ).node_id, non_loop_child_max_count, iid_dependencies::random_nested_loops ); } @@ -1008,10 +1197,14 @@ nodes_to_counts iid_node_dependence_props::compute_path_counts( const equation& } } - for ( const auto& [ loop_head, props ] : std::ranges::views::reverse( dependencies_by_loops ) ) { + for ( auto& loop : dependencies_by_loops.loops ) { + loop.set_chosen_loop_head(); + } + + for ( const auto& loop_props : std::ranges::views::reverse( dependencies_by_loops.loops ) ) { int loop_count = 0; - for ( const auto& body : props.bodies ) { + for ( const auto& body : loop_props.bodies ) { auto& [ left_count, right_count ] = path_counts[ body.node_id ]; if ( loop_heads.contains( body.node_id ) ) { @@ -1021,29 +1214,42 @@ nodes_to_counts iid_node_dependence_props::compute_path_counts( const equation& } } - if ( props.end_direction ) { - path_counts[ loop_head ] = { loop_count, 1 }; - } else { - path_counts[ loop_head ] = { 1, loop_count }; + for ( const auto& [ head, _ ] : loop_props.heads ) { + if ( head == ( *loop_props.chosen_loop_head ) ) { + if ( head.branching_direction ) { + path_counts[ head.node_id ] = { loop_count, 1 }; + } else { + path_counts[ head.node_id ] = { 1, loop_count }; + } + } else { + if ( head.branching_direction ) { + path_counts[ head.node_id ] = { loop_count, 0 }; + // path_counts[ head.node_id ] = { loop_count + 1, 0 }; + } else { + path_counts[ head.node_id ] = { 0, loop_count }; + // path_counts[ head.node_id ] = { 0, loop_count + 1 }; + } + } } } compute_path_counts_loops( path_counts, path, loop_heads ); compute_path_counts_loading( path_counts, path, loop_heads ); - for ( auto& [ loop_head, props ] : dependencies_by_loops ) { - auto it = path_counts.find( loop_head ); - if ( it != path_counts.end() ) { - props.previous_counts.push_back( it->second ); - } - } + // for ( auto& [ loop_head, props ] : dependencies_by_loops ) { + // auto it = path_counts.find( loop_head ); + // if ( it != path_counts.end() ) { + // props.previous_counts.push_back( it->second ); + // } + // } + + // for ( auto& [ loop_head, props ] : dependencies_by_loading ) { + // auto it = path_counts.find( loop_head ); + // if ( it != path_counts.end() ) { + // props.previous_counts.push_back( it->second ); + // } + // } - for ( auto& [ loop_head, props ] : dependencies_by_loading ) { - auto it = path_counts.find( loop_head ); - if ( it != path_counts.end() ) { - props.previous_counts.push_back( it->second ); - } - } return path_counts; } @@ -1171,10 +1377,12 @@ std::vector< equation > iid_node_dependence_props::get_random_vector( const std: std::set< node_direction > iid_node_dependence_props::get_leaf_subsets() { std::set< node_direction > all_leafs; - for ( const auto& [ _, props ] : dependencies_by_loops ) { - for ( const auto& body : props.bodies ) { + auto loop_heads = get_loop_heads( false ); + + for ( const auto& loop : dependencies_by_loops.loops ) { + for ( const auto& body : loop.bodies ) { location_id body_id = body.node_id; - if ( !dependencies_by_loops.contains( body_id ) ) { + if ( !loop_heads.contains( body_id ) ) { all_leafs.insert( body ); } } @@ -1355,18 +1563,39 @@ void iid_node_dependence_props::compute_dependencies_by_loops( const loop_head_t } bool loop_head_end_direction = loop_heads_ending.at( loop_head ); + node_direction loop_head_direction = { loop_head, loop_head_end_direction }; + + std::unordered_set< location_id > all_ids = loop_bodies; + all_ids.insert( loop_head ); + + loading_loop_properties& props = dependencies_by_loops.get_props( all_ids, loop_head ); + + props.heads[ loop_head_direction ].count++; + auto it = props.bodies.find( loop_head_direction ); for ( const auto& body : loop_bodies ) { for ( bool direction : { true, false } ) { node_direction node_id_direction = { body, direction }; + for ( const auto& [ head, _ ] : props.heads ) { + if ( head.node_id == body ) { + continue; + } + } + if ( matrix.contains( node_id_direction ) ) { - dependencies_by_loops[ loop_head ].bodies.insert( node_id_direction ); - dependencies_by_loops[ loop_head ].end_direction = loop_head_end_direction; + props.bodies.insert( node_id_direction ); } } } } + + dependencies_by_loops.merge_properties(); + std::sort( dependencies_by_loops.loops.begin(), + dependencies_by_loops.loops.end(), + []( const auto& a, const auto& b ) { + return a.get_smallest_loop_head_id() < b.get_smallest_loop_head_id(); + } ); } // ------------------------------------------------------------------------------------------------ @@ -1381,10 +1610,13 @@ possible_path iid_node_dependence_props::generate_path_from_node_counts( const n bool loop_head_end_direction = false; bool is_loop_head = false; - auto it_loops = dependencies_by_loops.find( id ); - if ( it_loops != dependencies_by_loops.end() ) { - is_loop_head = true; - loop_head_end_direction = it_loops->second.end_direction; + for ( const auto& loop : dependencies_by_loops.loops ) { + for ( const auto& [ head, _ ] : loop.heads ) { + if ( head.node_id == id ) { + is_loop_head = true; + loop_head_end_direction = head.branching_direction; + } + } } auto it_loading = dependencies_by_loading.find( id ); @@ -1404,8 +1636,10 @@ possible_path iid_node_dependence_props::generate_path_from_node_counts( const n std::set< location_id > iid_node_dependence_props::get_loop_heads( bool include_loading_loops ) { std::set< location_id > loop_heads; - for ( const auto& [ loop_head, _ ] : dependencies_by_loops ) { - loop_heads.insert( loop_head ); + for ( const auto& props : dependencies_by_loops.loops ) { + for ( const auto& [ head, _ ] : props.heads ) { + loop_heads.insert( head.node_id ); + } } if ( include_loading_loops ) { @@ -1440,6 +1674,7 @@ void iid_dependencies::process_node_dependence( branching_node* node ) return; iid_node_dependence_props& props = id_to_equation_map[ node->get_location_id() ]; + // std::cout << "Processing node with id:" << node->get_location_id().id << std::endl; props.process_node( node ); } From 8b05ab40ff38c45cceebb96a7a460f15d851c47b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sun, 9 Feb 2025 22:07:20 +0100 Subject: [PATCH 108/144] feat: add benchmarks for computation in loop and multiple loop heads --- benchmarks/iid_testing/computation_in_loop.c | 36 +++++++++++ .../iid_testing/computation_in_loop.json | 55 ++++++++++++++++ benchmarks/iid_testing/multiple_loop_heads.c | 41 ++++++++++++ .../iid_testing/multiple_loop_heads.json | 64 +++++++++++++++++++ 4 files changed, 196 insertions(+) create mode 100644 benchmarks/iid_testing/computation_in_loop.c create mode 100644 benchmarks/iid_testing/computation_in_loop.json create mode 100644 benchmarks/iid_testing/multiple_loop_heads.c create mode 100644 benchmarks/iid_testing/multiple_loop_heads.json diff --git a/benchmarks/iid_testing/computation_in_loop.c b/benchmarks/iid_testing/computation_in_loop.c new file mode 100644 index 00000000..94b5c0cc --- /dev/null +++ b/benchmarks/iid_testing/computation_in_loop.c @@ -0,0 +1,36 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 7 + --k; + ++k; + ++i; + } + if ( k == 19 ) // ID: 8 + return 1; + return 0; + } +} diff --git a/benchmarks/iid_testing/computation_in_loop.json b/benchmarks/iid_testing/computation_in_loop.json new file mode 100644 index 00000000..5cd1e344 --- /dev/null +++ b/benchmarks/iid_testing/computation_in_loop.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 6789, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 5, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/multiple_loop_heads.c b/benchmarks/iid_testing/multiple_loop_heads.c new file mode 100644 index 00000000..2b10db3c --- /dev/null +++ b/benchmarks/iid_testing/multiple_loop_heads.c @@ -0,0 +1,41 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] > 'K' ) // ID: 6 + break; + if ( s[ i ] == 'X' ) // ID: 7 + break; + if ( s[ i ] > 'C' && s[ i ] < "G" ) // ID: 8 + break; + if ( s[ i ] == 'A' ) // ID: 9 + ++k; + ++i; + } + if ( k == 10 ) // ID: 10 + return 1; + return 0; + } +} diff --git a/benchmarks/iid_testing/multiple_loop_heads.json b/benchmarks/iid_testing/multiple_loop_heads.json new file mode 100644 index 00000000..dc336b80 --- /dev/null +++ b/benchmarks/iid_testing/multiple_loop_heads.json @@ -0,0 +1,64 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 3008, + "num_covered_branchings": 9, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 5, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file From 89da9f1441cbf0e96e2b033e51d350ec15f70d56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sun, 9 Feb 2025 23:10:54 +0100 Subject: [PATCH 109/144] feat: update conditions in multiple loop heads benchmark --- benchmarks/iid_testing/multiple_loop_heads.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/benchmarks/iid_testing/multiple_loop_heads.c b/benchmarks/iid_testing/multiple_loop_heads.c index 2b10db3c..730b17f3 100644 --- a/benchmarks/iid_testing/multiple_loop_heads.c +++ b/benchmarks/iid_testing/multiple_loop_heads.c @@ -24,17 +24,15 @@ int main() while ( true ) { if ( s[ i ] == '\0' ) // ID: 5 break; - if ( s[ i ] > 'K' ) // ID: 6 + if ( s[ i ] > 'R' ) // ID: 6 break; - if ( s[ i ] == 'X' ) // ID: 7 + if ( s[ i ] == 'F' ) // ID: 7 break; - if ( s[ i ] > 'C' && s[ i ] < "G" ) // ID: 8 - break; - if ( s[ i ] == 'A' ) // ID: 9 + if ( s[ i ] == 'A' ) // ID: 8 ++k; ++i; } - if ( k == 10 ) // ID: 10 + if ( k == 10 ) // ID: 9 return 1; return 0; } From db813c7a5e8aba4756ccccdf0f67f87572393585 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sun, 9 Feb 2025 23:11:20 +0100 Subject: [PATCH 110/144] feat: update location_id usage to id_type in loop and dependency structures --- .../include/fuzzing/iid_vector_analysis.hpp | 70 +++-- src/fuzzing/src/iid_vector_analysis.cpp | 278 +++++++++--------- 2 files changed, 180 insertions(+), 168 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index f1406ac6..e0bc301e 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -27,6 +27,7 @@ struct node_counts { int right_count; int get_max_count() const; + int get_total_count() const { return left_count + right_count; } }; @@ -128,14 +129,14 @@ struct equation { }; struct node_direction { - location_id node_id; + location_id::id_type node_id; bool branching_direction; auto operator<=>( node_direction const& other ) const; bool operator==( node_direction const& other ) const = default; friend std::ostream& operator<<( std::ostream& os, const node_direction& nav ) { - return os << nav.node_id.id << " " << ( nav.branching_direction ? "right" : "left" ); + return os << nav.node_id << " " << ( nav.branching_direction ? "right" : "left" ); } }; @@ -150,48 +151,43 @@ struct loading_body_props { natural_32_bit minimal_bit_offset = std::numeric_limits< natural_32_bit >::max(); }; -struct loop_dependencies_props { +struct loading_loops_props { bool end_direction; std::set< node_direction > bodies; - std::vector< node_counts > previous_counts; -}; - -struct loading_loops_props : loop_dependencies_props { mean_counter< float > average_bits_per_loop; - std::map< location_id, loading_body_props > bit_values; + std::map< location_id::id_type, loading_body_props > bit_values; }; -struct loading_loop_head_properties { +struct dependent_loop_head_properties { int count; }; -struct loading_loop_properties { - std::map< node_direction, loading_loop_head_properties > heads; +struct dependent_loop_properties { + std::map< node_direction, dependent_loop_head_properties > heads; std::optional< node_direction > chosen_loop_head; std::set< node_direction > bodies; - bool is_same( const std::unordered_set< location_id >& other_ids ) const; - std::unordered_set< location_id > get_all_ids() const; - std::unordered_set< location_id > get_loop_head_ids() const; - std::unordered_set< location_id > get_body_ids() const; - location_id get_smallest_loop_head_id() const; + bool is_same( const std::unordered_set< location_id::id_type >& other_ids ) const; + std::unordered_set< location_id::id_type > get_all_ids() const; + std::unordered_set< location_id::id_type > get_loop_head_ids() const; + std::unordered_set< location_id::id_type > get_body_ids() const; + location_id::id_type get_smallest_loop_head_id() const; void set_chosen_loop_head(); }; struct dependencies_by_loops_t { - std::vector< loading_loop_properties > loops; + std::vector< dependent_loop_properties > loops; - loading_loop_properties& get_props( const std::unordered_set< location_id >& ids, location_id loop_head_id ); + dependent_loop_properties& get_props( const std::unordered_set< location_id::id_type >& ids, location_id::id_type loop_head_id ); void merge_properties(); - loading_loop_properties& get_props_by_loop_head_id( location_id loop_head_id ); + dependent_loop_properties& get_props_by_loop_head_id( location_id::id_type loop_head_id ); }; -using loop_head_to_loaded_bits_props = std::unordered_map< location_id, loaded_bits_props >; -using loop_ending_to_bodies = std::map< location_id, loop_dependencies_props >; -using loop_endings = std::map< location_id, bool >; +using loop_head_to_loaded_bits_props = std::unordered_map< location_id::id_type, loaded_bits_props >; +using loop_endings = std::map< location_id::id_type, bool >; using loop_head_to_bodies_t = std::unordered_map< location_id, std::unordered_set< location_id > >; -using nodes_to_counts = std::map< location_id, node_counts >; +using nodes_to_counts = std::map< location_id::id_type, node_counts >; struct equation_matrix { equation_matrix get_submatrix( std::set< node_direction > const& subset, bool unique ) const; @@ -272,20 +268,20 @@ struct iid_node_dependence_props { possible_path return_empty_path(); possible_path return_path( const possible_path& path ); void compute_path_counts_for_nested_loops( nodes_to_counts& path_counts, - std::map< location_id, int >& child_loop_counts, - location_id loop_head_id, + std::map< location_id::id_type, int >& child_loop_counts, + location_id::id_type loop_head_id, int minimum_count, bool use_random = false ); int compute_loop_count_loading( nodes_to_counts& path_counts, - location_id id, - const std::set< location_id >& loop_heads, + location_id::id_type id, + const std::set< location_id::id_type >& loop_heads, const loading_loops_props& props ); void compute_path_counts_loading( nodes_to_counts& path_counts, const equation& path, - const std::set< location_id >& loop_heads ); + const std::set< location_id::id_type >& loop_heads ); void compute_path_counts_loops( nodes_to_counts& path_counts, const equation& path, - const std::set< location_id >& loop_heads ); + const std::set< location_id::id_type >& loop_heads ); nodes_to_counts compute_path_counts( const equation& path, std::set< node_direction > const& all_leafs ); std::vector< equation > compute_best_vectors( const std::map< equation, int >& vectors_with_hits, int number_of_vectors, @@ -307,11 +303,11 @@ struct iid_node_dependence_props { void compute_dependencies_by_loops( const loop_head_to_bodies_t& loop_heads_to_bodies, const loop_endings& loop_heads_ending ); possible_path generate_path_from_node_counts( const nodes_to_counts& path_counts ); - std::set< location_id > get_loop_heads( bool include_loading_loops = true ); + std::set< location_id::id_type > get_loop_heads( bool include_loading_loops = true ); equation_matrix matrix; dependencies_by_loops_t dependencies_by_loops; - std::map< location_id, loading_loops_props > dependencies_by_loading; + std::map< location_id::id_type, loading_loops_props > dependencies_by_loading; iid_node_generations_stats stats; }; @@ -319,14 +315,14 @@ struct iid_node_dependence_props { struct iid_dependencies { void update_non_iid_nodes( sensitivity_analysis& sensitivity ); void process_node_dependence( branching_node* node ); - void remove_node_dependence( location_id id ); - iid_node_dependence_props& get_props( location_id id ); - std::vector< location_id > get_iid_nodes(); - std::optional< location_id > get_next_iid_node(); + void remove_node_dependence( location_id::id_type id ); + iid_node_dependence_props& get_props( location_id::id_type id ); + std::vector< location_id::id_type > get_iid_nodes(); + std::optional< location_id::id_type > get_next_iid_node(); private: - std::map< location_id, iid_node_dependence_props > id_to_equation_map; - std::set< location_id > non_iid_nodes; + std::map< location_id::id_type, iid_node_dependence_props > id_to_equation_map; + std::set< location_id::id_type > non_iid_nodes; public: // Configurations diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 17cd2411..1a615b38 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -52,7 +52,7 @@ bool path_node_props::get_desired_direction() const return true; } - assert( false ); + throw std::runtime_error( "No direction to go" ); } // ------------------------------------------------------------------------------------------------ @@ -287,30 +287,34 @@ bool equation::is_linear_dependent( const equation& other ) const return false; } - return std::abs( best_value / other.best_value - ratio ) < 1e-9; + if ( std::isnan( ratio ) ) { + return ( best_value - other.best_value ) < 1e-7; + } + + return std::abs( best_value / other.best_value - ratio ) < 1e-7; } // node_direction // ------------------------------------------------------------------------------------------------ auto node_direction::operator<=>( node_direction const& other ) const { - if ( auto const cmp = node_id.id <=> other.node_id.id; cmp != 0 ) + if ( auto const cmp = node_id <=> other.node_id; cmp != 0 ) return cmp; return branching_direction <=> other.branching_direction; } -// loading_loop_properties +// dependent_loop_properties // ------------------------------------------------------------------------------------------------ -bool fuzzing::loading_loop_properties::is_same( const std::unordered_set< location_id >& other_ids ) const +bool fuzzing::dependent_loop_properties::is_same( const std::unordered_set< location_id::id_type >& other_ids ) const { return get_all_ids() == other_ids; } // ------------------------------------------------------------------------------------------------ -std::unordered_set< location_id > fuzzing::loading_loop_properties::get_all_ids() const +std::unordered_set< location_id::id_type > fuzzing::dependent_loop_properties::get_all_ids() const { - std::unordered_set< location_id > all_ids; + std::unordered_set< location_id::id_type > all_ids; for ( const auto& [ head, props ] : heads ) { all_ids.insert( head.node_id ); @@ -324,9 +328,9 @@ std::unordered_set< location_id > fuzzing::loading_loop_properties::get_all_ids( } // ------------------------------------------------------------------------------------------------ -std::unordered_set< location_id > fuzzing::loading_loop_properties::get_loop_head_ids() const +std::unordered_set< location_id::id_type > fuzzing::dependent_loop_properties::get_loop_head_ids() const { - std::unordered_set< location_id > loop_head_ids; + std::unordered_set< location_id::id_type > loop_head_ids; for ( const auto& [ head, props ] : heads ) { loop_head_ids.insert( head.node_id ); @@ -336,9 +340,9 @@ std::unordered_set< location_id > fuzzing::loading_loop_properties::get_loop_hea } // ------------------------------------------------------------------------------------------------ -std::unordered_set< location_id > fuzzing::loading_loop_properties::get_body_ids() const +std::unordered_set< location_id::id_type > fuzzing::dependent_loop_properties::get_body_ids() const { - std::unordered_set< location_id > body_ids; + std::unordered_set< location_id::id_type > body_ids; for ( const auto& body : bodies ) { body_ids.insert( body.node_id ); @@ -348,9 +352,9 @@ std::unordered_set< location_id > fuzzing::loading_loop_properties::get_body_ids } // ------------------------------------------------------------------------------------------------ -location_id fuzzing::loading_loop_properties::get_smallest_loop_head_id() const +location_id::id_type fuzzing::dependent_loop_properties::get_smallest_loop_head_id() const { - location_id smallest_id = heads.begin()->first.node_id; + location_id::id_type smallest_id = heads.begin()->first.node_id; for ( const auto& [ head, props ] : heads ) { if ( head.node_id < smallest_id ) { @@ -362,7 +366,7 @@ location_id fuzzing::loading_loop_properties::get_smallest_loop_head_id() const } // ------------------------------------------------------------------------------------------------ -void fuzzing::loading_loop_properties::set_chosen_loop_head() +void fuzzing::dependent_loop_properties::set_chosen_loop_head() { for ( const auto& [ head, props ] : heads ) { if ( !chosen_loop_head.has_value() ) { @@ -377,10 +381,11 @@ void fuzzing::loading_loop_properties::set_chosen_loop_head() // dependencies_by_loops_t // ------------------------------------------------------------------------------------------------ -loading_loop_properties& fuzzing::dependencies_by_loops_t::get_props( const std::unordered_set< location_id >& ids, - location_id loop_head_id ) +dependent_loop_properties& +fuzzing::dependencies_by_loops_t::get_props( const std::unordered_set< location_id::id_type >& ids, + location_id::id_type loop_head_id ) { - for ( loading_loop_properties& loop : loops ) { + for ( dependent_loop_properties& loop : loops ) { for ( const auto& [ head, props ] : loop.heads ) { if ( head.node_id == loop_head_id ) { return loop; @@ -400,10 +405,7 @@ loading_loop_properties& fuzzing::dependencies_by_loops_t::get_props( const std: void fuzzing::dependencies_by_loops_t::merge_properties() { for ( auto it = loops.begin(); it != loops.end(); it++ ) { - std::set< location_id > head_ids; - for ( const auto& [ head, props ] : it->heads ) { - head_ids.insert( head.node_id ); - } + std::unordered_set< location_id::id_type > head_ids = it->get_loop_head_ids(); for ( auto body_it = it->bodies.begin(); body_it != it->bodies.end(); ) { if ( head_ids.contains( body_it->node_id ) ) { @@ -426,7 +428,6 @@ void fuzzing::dependencies_by_loops_t::merge_properties() it_1->heads[ head ].count += props.count; } - it_1->bodies.insert( it_2->bodies.begin(), it_2->bodies.end() ); it_2 = loops.erase( it_2 ); it_1 = loops.begin(); @@ -438,10 +439,7 @@ void fuzzing::dependencies_by_loops_t::merge_properties() } for ( auto it = loops.begin(); it != loops.end(); it++ ) { - std::set< location_id > head_ids; - for ( const auto& [ head, props ] : it->heads ) { - head_ids.insert( head.node_id ); - } + std::unordered_set< location_id::id_type > head_ids = it->get_loop_head_ids(); for ( auto body_it = it->bodies.begin(); body_it != it->bodies.end(); ) { if ( head_ids.contains( body_it->node_id ) ) { @@ -452,21 +450,20 @@ void fuzzing::dependencies_by_loops_t::merge_properties() } } - // Not sure if deleting loops with only loop heads is a good idea - for ( auto it = loops.begin(); it != loops.end(); ) { - if ( it->bodies.empty() ) { - it = loops.erase( it ); - } else { - ++it; - } - } + // for ( auto it = loops.begin(); it != loops.end(); ) { + // if ( it->bodies.empty() ) { + // it = loops.erase( it ); + // } else { + // ++it; + // } + // } } // ------------------------------------------------------------------------------------------------ -loading_loop_properties& fuzzing::dependencies_by_loops_t::get_props_by_loop_head_id( location_id loop_head_id ) +dependent_loop_properties& fuzzing::dependencies_by_loops_t::get_props_by_loop_head_id( location_id::id_type loop_head_id ) { - for ( loading_loop_properties& loop : loops ) { + for ( dependent_loop_properties& loop : loops ) { for ( const auto& [ head, props ] : loop.heads ) { if ( head.node_id == loop_head_id ) { return loop; @@ -728,6 +725,8 @@ void equation_matrix::recompute_matrix() // ------------------------------------------------------------------------------------------------ possible_path iid_node_dependence_props::generate_probabilities() { + print_dependencies(); + std::set< node_direction > all_leafs = get_leaf_subsets(); if ( all_leafs.empty() || dependencies_by_loops.loops.empty() ) { return {}; @@ -876,11 +875,11 @@ void iid_node_dependence_props::print_dependencies() const std::cout << "## Dependencies by loading:" << std::endl; for ( const auto& [ loop_head, props ] : dependencies_by_loading ) { - std::cout << "Loop ID: " << loop_head.id << std::endl; + std::cout << "Loop ID: " << loop_head << std::endl; std::cout << "Loaded bits per loop: " << props.average_bits_per_loop.mean << std::endl; for ( const auto& body : props.bodies ) { const auto& bit_props = props.bit_values.at( body.node_id ); - std::cout << "- " << "`(" << body << ") → " << loop_head.id << "`" + std::cout << "- " << "`(" << body << ") → " << loop_head << "`" << ", Bits: " << bit_props.average_bit_size.mean << ", offset: " << bit_props.minimal_bit_offset << std::endl; } @@ -1024,8 +1023,8 @@ possible_path iid_node_dependence_props::return_path( const possible_path& path // ------------------------------------------------------------------------------------------------ void iid_node_dependence_props::compute_path_counts_for_nested_loops( nodes_to_counts& path_counts, - std::map< location_id, int >& child_loop_counts, - location_id loop_head_id, + std::map< location_id::id_type, int >& child_loop_counts, + location_id::id_type loop_head_id, int minimum_count, bool use_random ) { @@ -1061,7 +1060,7 @@ void iid_node_dependence_props::compute_path_counts_for_nested_loops( nodes_to_c *possible_counts.rbegin(); for ( auto& [ node_id, count ] : child_loop_counts ) { - loading_loop_properties& props = dependencies_by_loops.get_props_by_loop_head_id( node_id ); + dependent_loop_properties& props = dependencies_by_loops.get_props_by_loop_head_id( node_id ); for ( auto& [ head, _ ] : props.heads ) { auto& [ left_count, right_count ] = path_counts[ head.node_id ]; @@ -1075,7 +1074,7 @@ void iid_node_dependence_props::compute_path_counts_for_nested_loops( nodes_to_c } - loading_loop_properties& props = dependencies_by_loops.get_props_by_loop_head_id( loop_head_id ); + dependent_loop_properties& props = dependencies_by_loops.get_props_by_loop_head_id( loop_head_id ); for ( auto& [ head, _ ] : props.heads ) { if ( head.branching_direction ) { path_counts[ head.node_id ] = { highest_count, 1 }; @@ -1087,11 +1086,12 @@ void iid_node_dependence_props::compute_path_counts_for_nested_loops( nodes_to_c // ------------------------------------------------------------------------------------------------ int fuzzing::iid_node_dependence_props::compute_loop_count_loading( nodes_to_counts& path_counts, - location_id id, - const std::set< location_id >& loop_heads, + location_id::id_type id, + const std::set< location_id::id_type >& loop_heads, const loading_loops_props& props ) { float loaded_per_loop = props.average_bits_per_loop.mean; + INVARIANT( loaded_per_loop > 0 ); float average_bits = props.bit_values.at( id ).average_bit_size.mean; natural_32_bit offset = props.bit_values.at( id ).minimal_bit_offset; @@ -1108,14 +1108,7 @@ int fuzzing::iid_node_dependence_props::compute_loop_count_loading( nodes_to_cou } } - int total_count = -1; - if ( is_loop_head ) { - total_count = path_counts[ id ].left_count + path_counts[ id ].right_count; - } else { - total_count = path_counts[ id ].get_max_count(); - } - - INVARIANT( total_count > 0 ); + int total_count = is_loop_head ? path_counts[ id ].get_total_count() : path_counts[ id ].get_max_count(); float bits_needed = average_bits * total_count + offset; return std::ceil( bits_needed / loaded_per_loop ); @@ -1124,7 +1117,7 @@ int fuzzing::iid_node_dependence_props::compute_loop_count_loading( nodes_to_cou // ------------------------------------------------------------------------------------------------ void iid_node_dependence_props::compute_path_counts_loading( nodes_to_counts& path_counts, const equation& path, - const std::set< location_id >& loop_heads ) + const std::set< location_id::id_type >& loop_heads ) { for ( const auto& [ loading_head, props ] : dependencies_by_loading ) { int loop_count = 0; @@ -1149,10 +1142,14 @@ void iid_node_dependence_props::compute_path_counts_loading( nodes_to_counts& pa // ------------------------------------------------------------------------------------------------ void iid_node_dependence_props::compute_path_counts_loops( nodes_to_counts& path_counts, const equation& path, - const std::set< location_id >& loop_heads ) + const std::set< location_id::id_type >& loop_heads ) { for ( const auto& props : std::ranges::views::reverse( dependencies_by_loops.loops ) ) { - std::map< location_id, int > child_loop_counts; + if ( props.bodies.empty() ) { + continue; + } + + std::map< location_id::id_type, int > child_loop_counts; int non_loop_child_max_count = 1; for ( const auto& body : props.bodies ) { @@ -1186,7 +1183,11 @@ nodes_to_counts iid_node_dependence_props::compute_path_counts( const equation& std::vector< node_direction > leafs = std::vector< node_direction >( all_leafs.begin(), all_leafs.end() ); INVARIANT( leafs.size() == path.values.size() ); - std::set< location_id > loop_heads = get_loop_heads( false ); + std::set< location_id::id_type > loop_heads = get_loop_heads( false ); + + for ( auto& loop : dependencies_by_loops.loops ) { + loop.set_chosen_loop_head(); + } for ( int i = 0; i < leafs.size(); ++i ) { auto& [ left_count, right_count ] = path_counts[ leafs[ i ].node_id ]; @@ -1197,38 +1198,24 @@ nodes_to_counts iid_node_dependence_props::compute_path_counts( const equation& } } - for ( auto& loop : dependencies_by_loops.loops ) { - loop.set_chosen_loop_head(); - } - for ( const auto& loop_props : std::ranges::views::reverse( dependencies_by_loops.loops ) ) { + if ( loop_props.bodies.empty() ) { + continue; + } + int loop_count = 0; for ( const auto& body : loop_props.bodies ) { - auto& [ left_count, right_count ] = path_counts[ body.node_id ]; - - if ( loop_heads.contains( body.node_id ) ) { - loop_count = std::max( loop_count, 1 ); - } else { - loop_count = std::max( loop_count, left_count + right_count ); - } + loop_count = std::max( loop_count, path_counts[ body.node_id ].get_total_count() ); } for ( const auto& [ head, _ ] : loop_props.heads ) { - if ( head == ( *loop_props.chosen_loop_head ) ) { - if ( head.branching_direction ) { - path_counts[ head.node_id ] = { loop_count, 1 }; - } else { - path_counts[ head.node_id ] = { 1, loop_count }; - } + int end_count = head == ( *loop_props.chosen_loop_head ) ? 1 : 0; + + if ( head.branching_direction ) { + path_counts[ head.node_id ] = { loop_count, end_count }; } else { - if ( head.branching_direction ) { - path_counts[ head.node_id ] = { loop_count, 0 }; - // path_counts[ head.node_id ] = { loop_count + 1, 0 }; - } else { - path_counts[ head.node_id ] = { 0, loop_count }; - // path_counts[ head.node_id ] = { 0, loop_count + 1 }; - } + path_counts[ head.node_id ] = { end_count, loop_count }; } } } @@ -1236,20 +1223,6 @@ nodes_to_counts iid_node_dependence_props::compute_path_counts( const equation& compute_path_counts_loops( path_counts, path, loop_heads ); compute_path_counts_loading( path_counts, path, loop_heads ); - // for ( auto& [ loop_head, props ] : dependencies_by_loops ) { - // auto it = path_counts.find( loop_head ); - // if ( it != path_counts.end() ) { - // props.previous_counts.push_back( it->second ); - // } - // } - - // for ( auto& [ loop_head, props ] : dependencies_by_loading ) { - // auto it = path_counts.find( loop_head ); - // if ( it != path_counts.end() ) { - // props.previous_counts.push_back( it->second ); - // } - // } - return path_counts; } @@ -1380,35 +1353,58 @@ std::set< node_direction > iid_node_dependence_props::get_leaf_subsets() auto loop_heads = get_loop_heads( false ); for ( const auto& loop : dependencies_by_loops.loops ) { + if ( loop.bodies.empty() ) { + continue; + } + for ( const auto& body : loop.bodies ) { - location_id body_id = body.node_id; + location_id::id_type body_id = body.node_id; if ( !loop_heads.contains( body_id ) ) { all_leafs.insert( body ); } } + + for ( const auto& [ head, _ ] : loop.heads ) { + all_leafs.insert( { head.node_id, !head.branching_direction } ); + } } return all_leafs; } // ------------------------------------------------------------------------------------------------ -std::map< location_id, bool > iid_node_dependence_props::get_loop_heads_ending( branching_node* end_node, - loop_head_to_bodies_t& loop_heads_to_bodies ) +std::map< location_id::id_type, bool > +iid_node_dependence_props::get_loop_heads_ending( branching_node* end_node, + loop_head_to_bodies_t& loop_heads_to_bodies ) { std::vector< fuzzer::loop_boundary_props > loops; fuzzer::detect_loops_along_path_to_node( end_node, loop_heads_to_bodies, &loops ); - std::map< location_id, bool > loop_heads_ending; + std::map< location_id::id_type, bool > loop_heads_ending; auto is_outside_loop = [ & ]( branching_node* successor, location_id loop_head_id, const std::unordered_set< location_id >& loop_bodies ) { - return successor != nullptr && successor->get_location_id() != loop_head_id && - !loop_bodies.contains( successor->get_location_id() ); + if ( successor == nullptr ) { + return false; + } + + if ( successor->get_location_id() == loop_head_id ) { + return false; + } + + for ( const auto& bodies : loop_bodies ) { + if ( bodies.id == successor->get_location_id().id ) { + return false; + } + } + + return true; }; for ( const auto& loop : loops ) { location_id loop_head_id = loop.exit->get_location_id(); + const auto& loop_bodies = loop_heads_to_bodies.at( loop_head_id ); branching_node* loop_end_node = loop.exit; @@ -1416,9 +1412,9 @@ std::map< location_id, bool > iid_node_dependence_props::get_loop_heads_ending( branching_node* right_successor = loop_end_node->successor( true ).pointer; if ( is_outside_loop( left_successor, loop_head_id, loop_bodies ) ) { - loop_heads_ending[ loop_end_node->get_location_id() ] = false; + loop_heads_ending[ loop_end_node->get_location_id().id ] = false; } else if ( is_outside_loop( right_successor, loop_head_id, loop_bodies ) ) { - loop_heads_ending[ loop_end_node->get_location_id() ] = true; + loop_heads_ending[ loop_end_node->get_location_id().id ] = true; } } @@ -1431,8 +1427,8 @@ void fuzzing::iid_node_dependence_props::compute_loading_loops( branching_node* loop_head_to_loaded_bits_props& loading_loops ) { for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { - loading_loops[ loop_head ] = { std::numeric_limits< natural_32_bit >::max(), - std::numeric_limits< natural_32_bit >::min() }; + loading_loops[ loop_head.id ] = { std::numeric_limits< natural_32_bit >::max(), + std::numeric_limits< natural_32_bit >::min() }; } branching_node* node = end_node; @@ -1441,7 +1437,7 @@ void fuzzing::iid_node_dependence_props::compute_loading_loops( branching_node* if ( loop_head.id == node->get_location_id().id ) { natural_32_bit bits_count = node->get_num_stdin_bits(); - auto& props = loading_loops[ loop_head ]; + auto& props = loading_loops[ loop_head.id ]; props.min = std::min( props.min, bits_count ); props.max = std::max( props.max, bits_count ); props.loop_count++; @@ -1482,10 +1478,11 @@ void iid_node_dependence_props::compute_dependencies_by_loading( branching_node* natural_32_bit max = std::numeric_limits< natural_32_bit >::min(); std::vector< natural_32_bit > sensitive_stdin_bit_counts; }; - std::map< location_id, std::map< location_id, loading_body_props_tmp > > loop_to_props; + + std::map< location_id::id_type, std::map< location_id::id_type, loading_body_props_tmp > > loop_to_props; while ( node != nullptr ) { - auto node_id = node->get_location_id(); + location_id::id_type node_id = node->get_location_id().id; for ( const auto& [ loop_head, props ] : loading_loops ) { if ( !loop_heads_ending.contains( loop_head ) ) { @@ -1503,7 +1500,7 @@ void iid_node_dependence_props::compute_dependencies_by_loading( branching_node* if ( it == node->sensitive_stdin_bits.end() ) continue; - loading_body_props_tmp& loop_props = loop_to_props[ loop_head.id ][ node_id ]; + loading_body_props_tmp& loop_props = loop_to_props[ loop_head ][ node_id ]; auto min_it = std::min_element( node->sensitive_stdin_bits.begin(), node->sensitive_stdin_bits.end() ); if ( min_it != node->sensitive_stdin_bits.end() ) { @@ -1558,27 +1555,30 @@ void iid_node_dependence_props::compute_dependencies_by_loops( const loop_head_t const loop_endings& loop_heads_ending ) { for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { - if ( !loop_heads_ending.contains( loop_head ) ) { + location_id::id_type loop_head_id = loop_head.id; + if ( !loop_heads_ending.contains( loop_head_id ) || dependencies_by_loading.contains( loop_head_id ) ) { continue; } - bool loop_head_end_direction = loop_heads_ending.at( loop_head ); - node_direction loop_head_direction = { loop_head, loop_head_end_direction }; + bool loop_head_end_direction = loop_heads_ending.at( loop_head_id ); + node_direction loop_head_direction = { loop_head_id, loop_head_end_direction }; - std::unordered_set< location_id > all_ids = loop_bodies; - all_ids.insert( loop_head ); + std::unordered_set< location_id::id_type > all_ids; + all_ids.insert( loop_head_id ); + for ( const auto& loop_body_id : loop_bodies ) { + all_ids.insert( loop_body_id.id ); + } - loading_loop_properties& props = dependencies_by_loops.get_props( all_ids, loop_head ); + dependent_loop_properties& props = dependencies_by_loops.get_props( all_ids, loop_head_id ); props.heads[ loop_head_direction ].count++; - auto it = props.bodies.find( loop_head_direction ); for ( const auto& body : loop_bodies ) { for ( bool direction : { true, false } ) { - node_direction node_id_direction = { body, direction }; + node_direction node_id_direction = { body.id, direction }; for ( const auto& [ head, _ ] : props.heads ) { - if ( head.node_id == body ) { + if ( head.node_id == body.id ) { continue; } } @@ -1591,6 +1591,22 @@ void iid_node_dependence_props::compute_dependencies_by_loops( const loop_head_t } dependencies_by_loops.merge_properties(); + + for ( auto it = dependencies_by_loops.loops.begin(); it != dependencies_by_loops.loops.end(); ) { + bool removed = false; + for ( const auto& [ head, _ ] : it->heads ) { + if ( dependencies_by_loading.contains( head.node_id ) ) { + it = dependencies_by_loops.loops.erase( it ); + removed = true; + break; + } + } + + if ( !removed ) { + ++it; + } + } + std::sort( dependencies_by_loops.loops.begin(), dependencies_by_loops.loops.end(), []( const auto& a, const auto& b ) { @@ -1626,16 +1642,16 @@ possible_path iid_node_dependence_props::generate_path_from_node_counts( const n } path_node_props props = { counts, is_loop_head, loop_head_end_direction }; - path.emplace( id.id, props ); + path.emplace( id, props ); } return possible_path( path ); } // ------------------------------------------------------------------------------------------------ -std::set< location_id > iid_node_dependence_props::get_loop_heads( bool include_loading_loops ) +std::set< location_id::id_type > iid_node_dependence_props::get_loop_heads( bool include_loading_loops ) { - std::set< location_id > loop_heads; + std::set< location_id::id_type > loop_heads; for ( const auto& props : dependencies_by_loops.loops ) { for ( const auto& [ head, _ ] : props.heads ) { loop_heads.insert( head.node_id ); @@ -1657,7 +1673,7 @@ void iid_dependencies::update_non_iid_nodes( sensitivity_analysis& sensitivity ) { for ( branching_node* node : sensitivity.get_changed_nodes() ) { if ( node->is_did_branching() ) { - auto location_id = node->get_location_id(); + location_id::id_type location_id = node->get_location_id().id; if ( non_iid_nodes.insert( location_id ).second ) { id_to_equation_map.erase( location_id ); } @@ -1670,16 +1686,15 @@ void iid_dependencies::process_node_dependence( branching_node* node ) { TMPROF_BLOCK(); - if ( non_iid_nodes.contains( node->get_location_id() ) ) + if ( non_iid_nodes.contains( node->get_location_id().id ) ) return; - iid_node_dependence_props& props = id_to_equation_map[ node->get_location_id() ]; - // std::cout << "Processing node with id:" << node->get_location_id().id << std::endl; + iid_node_dependence_props& props = id_to_equation_map[ node->get_location_id().id ]; props.process_node( node ); } // ------------------------------------------------------------------------------------------------ -void iid_dependencies::remove_node_dependence( location_id id ) +void iid_dependencies::remove_node_dependence( location_id::id_type id ) { auto it = id_to_equation_map.find( id ); if ( it != id_to_equation_map.end() ) { @@ -1696,15 +1711,15 @@ void iid_dependencies::remove_node_dependence( location_id id ) } // ------------------------------------------------------------------------------------------------ -iid_node_dependence_props& iid_dependencies::get_props( location_id id ) +iid_node_dependence_props& iid_dependencies::get_props( location_id::id_type id ) { return id_to_equation_map.at( id ); } // ------------------------------------------------------------------------------------------------ -std::vector< location_id > iid_dependencies::get_iid_nodes() +std::vector< location_id::id_type > iid_dependencies::get_iid_nodes() { - std::vector< location_id > result; + std::vector< location_id::id_type > result; for ( const auto& [ key, _ ] : id_to_equation_map ) { result.push_back( key ); } @@ -1714,7 +1729,7 @@ std::vector< location_id > iid_dependencies::get_iid_nodes() } // ------------------------------------------------------------------------------------------------ -std::optional< location_id > iid_dependencies::get_next_iid_node() +std::optional< location_id::id_type > iid_dependencies::get_next_iid_node() { bool previous_needs_more_data = false; for ( auto it = id_to_equation_map.rbegin(); it != id_to_equation_map.rend(); ++it ) { @@ -1737,7 +1752,7 @@ std::optional< location_id > iid_dependencies::get_next_iid_node() } } - std::optional< location_id > best_id = std::nullopt; + std::optional< location_id::id_type > best_id = std::nullopt; for ( const auto& [ id, props ] : id_to_equation_map ) { if ( props.should_generate() ) { @@ -1758,7 +1773,8 @@ std::vector< node_direction > get_path( branching_node* node ) while ( current != nullptr ) { branching_node* predecessor = current->predecessor; if ( predecessor != nullptr ) { - node_direction nav = { predecessor->get_location_id(), predecessor->successor_direction( current ) }; + node_direction nav = { predecessor->get_location_id().id, + predecessor->successor_direction( current ) }; result.push_back( nav ); } current = predecessor; From a78df71df676c32a4c6f8ba3dec2d36c688ac359 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Wed, 12 Feb 2025 10:37:20 +0100 Subject: [PATCH 111/144] feat: add output of statistics --- .../include/fuzzing/iid_vector_analysis.hpp | 201 ++++++++++-------- src/fuzzing/src/iid_vector_analysis.cpp | 201 +++++++++++++----- 2 files changed, 258 insertions(+), 144 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index e0bc301e..6c101c11 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -22,6 +22,106 @@ struct mean_counter { void add( T value ); }; +struct node_direction { + location_id::id_type node_id; + bool branching_direction; + + auto operator<=>( node_direction const& other ) const; + bool operator==( node_direction const& other ) const = default; + friend std::ostream& operator<<( std::ostream& os, const node_direction& nav ) + { + return os << nav.node_id << " " << ( nav.branching_direction ? "right" : "left" ); + } +}; + +enum generation_state { + STATE_NOT_COVERED, + STATE_GENERATING_ARTIFICIAL_DATA, + STATE_GENERATION_MORE, + STATE_COVERED, + STATE_GENERATION_DATA_FOR_NEXT_NODE +}; + +enum failed_generation_method { METHOD_GENERATE_FROM_OTHER_NODE, METHOD_GENERATE_ARTIFICIAL_DATA }; + +struct iid_node_generations_stats { + int method_calls = 0; + int generation_starts = 0; + int successful_generations = 0; + int failed_generations = 0; + + int generated_for_other_node_count = 0; + int generate_artificial_data_count = 0; + + // Not saved values, because they are cleared multiple times through out the run + int failed_generations_in_row = 0; + + int generated_after_covered_max = 0; + int generated_after_covered = 0; + + int generated_for_other_node = 0; + int generated_for_other_node_max = 0; + + int generate_artificial_data = 0; + int generate_artificial_data_max = 0; + + + generation_state state = generation_state::STATE_NOT_COVERED; + + failed_generation_method last_failed_method = METHOD_GENERATE_ARTIFICIAL_DATA; +}; + +struct loading_body_props { + mean_counter< float > average_bit_size; + natural_32_bit minimal_bit_offset = std::numeric_limits< natural_32_bit >::max(); +}; + +struct loading_loops_props { + bool end_direction; + std::set< node_direction > bodies; + mean_counter< float > average_bits_per_loop; + std::map< location_id::id_type, loading_body_props > bit_values; +}; + +struct dependent_loop_head_properties { + int count; +}; + +struct dependent_loop_properties { + std::map< node_direction, dependent_loop_head_properties > heads; + + std::optional< node_direction > chosen_loop_head; + std::set< node_direction > bodies; + + bool is_same( const std::set< location_id::id_type >& other_ids ) const; + std::set< location_id::id_type > get_all_ids() const; + std::set< location_id::id_type > get_loop_head_ids() const; + std::set< location_id::id_type > get_body_ids() const; + location_id::id_type get_smallest_loop_head_id() const; + void set_chosen_loop_head(); +}; + +struct dependencies_by_loops_t { + std::vector< dependent_loop_properties > loops; + + dependent_loop_properties& get_props( const std::set< location_id::id_type >& ids, + location_id::id_type loop_head_id ); + void merge_properties(); + dependent_loop_properties& get_props_by_loop_head_id( location_id::id_type loop_head_id ); +}; + +struct iid_vector_analysis_statistics_per_node { + iid_node_generations_stats generation_stats; + dependencies_by_loops_t dependencies_by_loops; + std::map< location_id::id_type, loading_loops_props > dependencies_by_loading; +}; + +struct iid_vector_analysis_statistics { + std::map< location_id::id_type, iid_vector_analysis_statistics_per_node > iid_nodes_stats; + std::vector< location_id::id_type > ignored_nodes; +}; + + struct node_counts { int left_count; int right_count; @@ -90,7 +190,6 @@ struct possible_path { std::map< location_id::id_type, path_node_props > path; }; - struct equation { equation( std::vector< int > values, double best_value ) : values( std::move( values ) ) @@ -128,61 +227,12 @@ struct equation { } }; -struct node_direction { - location_id::id_type node_id; - bool branching_direction; - - auto operator<=>( node_direction const& other ) const; - bool operator==( node_direction const& other ) const = default; - friend std::ostream& operator<<( std::ostream& os, const node_direction& nav ) - { - return os << nav.node_id << " " << ( nav.branching_direction ? "right" : "left" ); - } -}; - struct loaded_bits_props { natural_32_bit min; natural_32_bit max; int loop_count; }; -struct loading_body_props { - mean_counter< float > average_bit_size; - natural_32_bit minimal_bit_offset = std::numeric_limits< natural_32_bit >::max(); -}; - -struct loading_loops_props { - bool end_direction; - std::set< node_direction > bodies; - mean_counter< float > average_bits_per_loop; - std::map< location_id::id_type, loading_body_props > bit_values; -}; - -struct dependent_loop_head_properties { - int count; -}; - -struct dependent_loop_properties { - std::map< node_direction, dependent_loop_head_properties > heads; - - std::optional< node_direction > chosen_loop_head; - std::set< node_direction > bodies; - - bool is_same( const std::unordered_set< location_id::id_type >& other_ids ) const; - std::unordered_set< location_id::id_type > get_all_ids() const; - std::unordered_set< location_id::id_type > get_loop_head_ids() const; - std::unordered_set< location_id::id_type > get_body_ids() const; - location_id::id_type get_smallest_loop_head_id() const; - void set_chosen_loop_head(); -}; - -struct dependencies_by_loops_t { - std::vector< dependent_loop_properties > loops; - - dependent_loop_properties& get_props( const std::unordered_set< location_id::id_type >& ids, location_id::id_type loop_head_id ); - void merge_properties(); - dependent_loop_properties& get_props_by_loop_head_id( location_id::id_type loop_head_id ); -}; using loop_head_to_loaded_bits_props = std::unordered_map< location_id::id_type, loaded_bits_props >; using loop_endings = std::map< location_id::id_type, bool >; @@ -197,9 +247,9 @@ struct equation_matrix { std::pair< std::size_t, std::size_t > get_dimensions() const; std::map< equation, int > compute_vectors_with_hits(); std::vector< equation >& get_matrix(); - std::optional< equation > get_new_leaf_counts_from_vectors( const std::vector< equation >& vector, - int generated_after_covered, - bool generate_more_data ); + std::optional< equation > get_new_subset_counts_from_vectors( const std::vector< equation >& vector, + int generated_after_covered, + const iid_node_generations_stats& state ); int get_desired_vector_direction() const; float get_biggest_branching_value() const; @@ -215,41 +265,16 @@ struct equation_matrix { std::set< node_direction > nodes; }; -enum generation_state { - STATE_NOT_COVERED, - STATE_GENERATING_ARTIFICIAL_DATA, - STATE_GENERATION_MORE, - STATE_COVERED, - STATE_GENERATION_DATA_FOR_NEXT_NODE -}; - -enum failed_generation_method { METHOD_GENERATE_FROM_OTHER_NODE, METHOD_GENERATE_ARTIFICIAL_DATA }; - -struct iid_node_generations_stats { - int generation_starts = 0; - int successful_generations = 0; - int failed_generations = 0; - - int failed_generations_in_row = 0; - - int generated_after_covered_max = 0; - int generated_after_covered = 0; - - int generated_for_other_node = 0; - int generated_for_other_node_max = 0; - - int generate_artificial_data = 0; - int generate_artificial_data_max = 0; - - generation_state state = generation_state::STATE_NOT_COVERED; - - failed_generation_method last_failed_method = METHOD_GENERATE_ARTIFICIAL_DATA; -}; - struct iid_node_dependence_props { possible_path generate_probabilities(); void process_node( branching_node* end_node ); iid_node_generations_stats& get_generations_stats() { return stats; } + const iid_node_generations_stats& get_generations_stats() const { return stats; } + const dependencies_by_loops_t& get_dependencies_by_loops() const { return dependencies_by_loops; } + const std::map< location_id::id_type, loading_loops_props >& get_dependencies_by_loading() const + { + return dependencies_by_loading; + } bool should_generate() const; bool too_much_failed_in_row( int max_failed_generations_in_row ) const; @@ -257,6 +282,7 @@ struct iid_node_dependence_props { void set_as_generating_artificial_data( int minimal_max_generation_artificial_data ); failed_generation_method get_method_for_failed_generation( bool is_first ); bool is_equal_branching_predicate() const; + void combine_props( const iid_node_dependence_props& other ); void print_dependencies() const; void print_stats( bool only_state = false ) const; @@ -292,7 +318,7 @@ struct iid_node_dependence_props { equation& best_vector ); std::vector< equation > get_random_vector( const std::map< equation, int >& vectors_with_hits, int number_of_vectors ); - std::set< node_direction > get_leaf_subsets(); + std::set< node_direction > get_node_subsets_for_computation(); loop_endings get_loop_heads_ending( branching_node* end_node, loop_head_to_bodies_t& loop_heads_to_bodies ); void compute_loading_loops( branching_node* end_node, const loop_head_to_bodies_t& loop_heads_to_bodies, @@ -319,10 +345,11 @@ struct iid_dependencies { iid_node_dependence_props& get_props( location_id::id_type id ); std::vector< location_id::id_type > get_iid_nodes(); std::optional< location_id::id_type > get_next_iid_node(); + iid_vector_analysis_statistics get_stats() const; private: std::map< location_id::id_type, iid_node_dependence_props > id_to_equation_map; - std::set< location_id::id_type > non_iid_nodes; + std::set< location_id::id_type > ignored_nodes; public: // Configurations @@ -334,7 +361,9 @@ struct iid_dependencies { inline static int minimal_max_generation_for_other_node = 10; inline static int minimal_max_generation_artificial_data = 5; inline static float percentage_to_add_to_path = 0.4; + inline static bool create_artificial_data = true; }; std::vector< node_direction > get_path( branching_node* node ); +bool should_generate_more_data( const generation_state& state ); } // namespace fuzzing diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 1a615b38..45f2907f 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -306,15 +306,15 @@ auto node_direction::operator<=>( node_direction const& other ) const // dependent_loop_properties // ------------------------------------------------------------------------------------------------ -bool fuzzing::dependent_loop_properties::is_same( const std::unordered_set< location_id::id_type >& other_ids ) const +bool fuzzing::dependent_loop_properties::is_same( const std::set< location_id::id_type >& other_ids ) const { return get_all_ids() == other_ids; } // ------------------------------------------------------------------------------------------------ -std::unordered_set< location_id::id_type > fuzzing::dependent_loop_properties::get_all_ids() const +std::set< location_id::id_type > fuzzing::dependent_loop_properties::get_all_ids() const { - std::unordered_set< location_id::id_type > all_ids; + std::set< location_id::id_type > all_ids; for ( const auto& [ head, props ] : heads ) { all_ids.insert( head.node_id ); @@ -328,9 +328,9 @@ std::unordered_set< location_id::id_type > fuzzing::dependent_loop_properties::g } // ------------------------------------------------------------------------------------------------ -std::unordered_set< location_id::id_type > fuzzing::dependent_loop_properties::get_loop_head_ids() const +std::set< location_id::id_type > fuzzing::dependent_loop_properties::get_loop_head_ids() const { - std::unordered_set< location_id::id_type > loop_head_ids; + std::set< location_id::id_type > loop_head_ids; for ( const auto& [ head, props ] : heads ) { loop_head_ids.insert( head.node_id ); @@ -340,9 +340,9 @@ std::unordered_set< location_id::id_type > fuzzing::dependent_loop_properties::g } // ------------------------------------------------------------------------------------------------ -std::unordered_set< location_id::id_type > fuzzing::dependent_loop_properties::get_body_ids() const +std::set< location_id::id_type > fuzzing::dependent_loop_properties::get_body_ids() const { - std::unordered_set< location_id::id_type > body_ids; + std::set< location_id::id_type > body_ids; for ( const auto& body : bodies ) { body_ids.insert( body.node_id ); @@ -381,9 +381,8 @@ void fuzzing::dependent_loop_properties::set_chosen_loop_head() // dependencies_by_loops_t // ------------------------------------------------------------------------------------------------ -dependent_loop_properties& -fuzzing::dependencies_by_loops_t::get_props( const std::unordered_set< location_id::id_type >& ids, - location_id::id_type loop_head_id ) +dependent_loop_properties& fuzzing::dependencies_by_loops_t::get_props( const std::set< location_id::id_type >& ids, + location_id::id_type loop_head_id ) { for ( dependent_loop_properties& loop : loops ) { for ( const auto& [ head, props ] : loop.heads ) { @@ -405,7 +404,7 @@ fuzzing::dependencies_by_loops_t::get_props( const std::unordered_set< location_ void fuzzing::dependencies_by_loops_t::merge_properties() { for ( auto it = loops.begin(); it != loops.end(); it++ ) { - std::unordered_set< location_id::id_type > head_ids = it->get_loop_head_ids(); + std::set< location_id::id_type > head_ids = it->get_loop_head_ids(); for ( auto body_it = it->bodies.begin(); body_it != it->bodies.end(); ) { if ( head_ids.contains( body_it->node_id ) ) { @@ -439,7 +438,7 @@ void fuzzing::dependencies_by_loops_t::merge_properties() } for ( auto it = loops.begin(); it != loops.end(); it++ ) { - std::unordered_set< location_id::id_type > head_ids = it->get_loop_head_ids(); + std::set< location_id::id_type > head_ids = it->get_loop_head_ids(); for ( auto body_it = it->bodies.begin(); body_it != it->bodies.end(); ) { if ( head_ids.contains( body_it->node_id ) ) { @@ -516,8 +515,10 @@ void equation_matrix::process_node( branching_node* end_node ) std::vector< node_direction > path = get_path( end_node ); bool new_node = false; for ( const node_direction& nav : path ) { - auto [ it, inserted ] = nodes.insert( nav ); - new_node |= inserted; + for ( bool direction : { true, false } ) { + auto [ it, inserted ] = nodes.insert( { nav.node_id, direction } ); + new_node |= inserted; + } } if ( new_node ) { @@ -628,13 +629,16 @@ float equation_matrix::get_biggest_branching_value() const } // ------------------------------------------------------------------------------------------------ -std::optional< equation > equation_matrix::get_new_leaf_counts_from_vectors( const std::vector< equation >& vectors, - int generation_count_after_covered, - bool generate_more_data ) +std::optional< equation > +equation_matrix::get_new_subset_counts_from_vectors( const std::vector< equation >& vectors, + int generation_count_after_covered, + const iid_node_generations_stats& stats ) { INVARIANT( !vectors.empty() ); INVARIANT( vectors[ 0 ].values.size() == nodes.size() ); + bool generate_more_data = should_generate_more_data( stats.state ); + std::vector< equation > paths; bool is_equal_sign = get_branching_predicate() == BRANCHING_PREDICATE::BP_EQUAL; @@ -654,7 +658,15 @@ std::optional< equation > equation_matrix::get_new_leaf_counts_from_vectors( con new_path = new_path.add_to_values( vector ); if ( generate_more_data ) { double best_value = new_path.best_value; - new_path = new_path * ( 1 + iid_dependencies::percentage_to_add_to_path ); + float procents_to_add = 1 + iid_dependencies::percentage_to_add_to_path; + + if ( stats.state == generation_state::STATE_GENERATING_ARTIFICIAL_DATA ) { + // new_path = new_path.add_to_positive( stats.generate_artificial_data_count ); + // new_path = new_path + vector * stats.generate_artificial_data_count; + procents_to_add += stats.generate_artificial_data_count / 3.0; + } + + new_path = new_path * procents_to_add; new_path.best_value = best_value; } } @@ -725,16 +737,16 @@ void equation_matrix::recompute_matrix() // ------------------------------------------------------------------------------------------------ possible_path iid_node_dependence_props::generate_probabilities() { - print_dependencies(); + stats.method_calls++; - std::set< node_direction > all_leafs = get_leaf_subsets(); - if ( all_leafs.empty() || dependencies_by_loops.loops.empty() ) { + std::set< node_direction > computation_subset = get_node_subsets_for_computation(); + if ( computation_subset.empty() || dependencies_by_loops.loops.empty() ) { return {}; } TMPROF_BLOCK(); stats.generation_starts++; - equation_matrix submatrix = matrix.get_submatrix( all_leafs, true ); + equation_matrix submatrix = matrix.get_submatrix( computation_subset, true ); { // print_stats( true ); @@ -754,17 +766,14 @@ possible_path iid_node_dependence_props::generate_probabilities() generate_vectors_if_not_enough_data( *best_vectors, submatrix ); } - bool generate_more_data = stats.state == generation_state::STATE_GENERATION_MORE || - stats.state == generation_state::STATE_GENERATION_DATA_FOR_NEXT_NODE || - stats.state == generation_state::STATE_GENERATING_ARTIFICIAL_DATA; - std::optional< equation > new_leaf_counts = submatrix.get_new_leaf_counts_from_vectors( - *best_vectors, stats.generated_after_covered, generate_more_data ); + std::optional< equation > new_subset_counts = + submatrix.get_new_subset_counts_from_vectors( *best_vectors, stats.generated_after_covered, stats ); - if ( !new_leaf_counts.has_value() ) { + if ( !new_subset_counts.has_value() ) { return return_empty_path(); } - nodes_to_counts node_counts = compute_path_counts( new_leaf_counts.value(), all_leafs ); + nodes_to_counts node_counts = compute_path_counts( new_subset_counts.value(), computation_subset ); possible_path path = generate_path_from_node_counts( node_counts ); // std::cout << "Generated path: " << std::endl << path << std::endl; @@ -812,6 +821,7 @@ void iid_node_dependence_props::set_as_generating_for_other_node( int minimal_ma stats.state = generation_state::STATE_GENERATION_DATA_FOR_NEXT_NODE; stats.generated_for_other_node_max = minimal_max_generation_for_other_node; stats.generated_for_other_node = 0; + stats.generated_for_other_node_count++; } // ------------------------------------------------------------------------------------------------ @@ -822,6 +832,7 @@ void fuzzing::iid_node_dependence_props::set_as_generating_artificial_data( int stats.state = generation_state::STATE_GENERATING_ARTIFICIAL_DATA; stats.generate_artificial_data_max = minimal_max_generation_artificial_data; stats.generate_artificial_data = 0; + stats.generate_artificial_data_count++; } // ------------------------------------------------------------------------------------------------ @@ -829,7 +840,9 @@ failed_generation_method fuzzing::iid_node_dependence_props::get_method_for_fail { failed_generation_method new_method; - if ( is_first ) { + if ( !iid_dependencies::create_artificial_data ) { + return failed_generation_method::METHOD_GENERATE_FROM_OTHER_NODE; + } else if ( is_first ) { new_method = failed_generation_method::METHOD_GENERATE_ARTIFICIAL_DATA; } else { switch ( stats.last_failed_method ) { @@ -856,32 +869,70 @@ bool iid_node_dependence_props::is_equal_branching_predicate() const return matrix.get_branching_predicate() == BRANCHING_PREDICATE::BP_EQUAL; } +// ------------------------------------------------------------------------------------------------ +void fuzzing::iid_node_dependence_props::combine_props( const iid_node_dependence_props& other ) +{ + std::vector< dependent_loop_properties > loops_to_add; + + for ( const auto& other_loop : other.dependencies_by_loops.loops ) { + for ( const auto& loop : dependencies_by_loops.loops ) { + std::set< location_id::id_type > other_ids = other_loop.get_loop_head_ids(); + std::set< location_id::id_type > ids = loop.get_loop_head_ids(); + + std::set< location_id::id_type > intersection; + + std::set_intersection( other_ids.begin(), + other_ids.end(), + ids.begin(), + ids.end(), + std::inserter( intersection, intersection.begin() ) ); + + if ( !intersection.empty() ) { + loops_to_add.push_back( other_loop ); + } + } + } + + for ( const auto& loop : loops_to_add ) { + dependencies_by_loops.loops.push_back( loop ); + } + + dependencies_by_loops.merge_properties(); +} + // ------------------------------------------------------------------------------------------------ void iid_node_dependence_props::print_dependencies() const { + bool print_dependencies_by_loops = true; + bool print_dependencies_by_loading = false; + std::cout << "# Dependencies:" << std::endl; - std::cout << "## Dependencies by loops:" << std::endl; - for ( const auto& loop : dependencies_by_loops.loops ) { - std::cout << "Loop heads:" << std::endl; - for ( const auto& [ head, head_props ] : loop.heads ) { - std::cout << "- " << head << " (" << head_props.count << ")" << std::endl; - } + if ( print_dependencies_by_loops ) { + std::cout << "## Dependencies by loops:" << std::endl; + for ( const auto& loop : dependencies_by_loops.loops ) { + std::cout << "Loop heads:" << std::endl; + for ( const auto& [ head, head_props ] : loop.heads ) { + std::cout << "- " << head << " (" << head_props.count << ")" << std::endl; + } - std::cout << "Loop bodies:" << std::endl; - for ( const auto& body : loop.bodies ) { - std::cout << "- " << body << std::endl; + std::cout << "Loop bodies:" << std::endl; + for ( const auto& body : loop.bodies ) { + std::cout << "- " << body << std::endl; + } } } - std::cout << "## Dependencies by loading:" << std::endl; - for ( const auto& [ loop_head, props ] : dependencies_by_loading ) { - std::cout << "Loop ID: " << loop_head << std::endl; - std::cout << "Loaded bits per loop: " << props.average_bits_per_loop.mean << std::endl; - for ( const auto& body : props.bodies ) { - const auto& bit_props = props.bit_values.at( body.node_id ); - std::cout << "- " << "`(" << body << ") → " << loop_head << "`" - << ", Bits: " << bit_props.average_bit_size.mean - << ", offset: " << bit_props.minimal_bit_offset << std::endl; + if ( print_dependencies_by_loading ) { + std::cout << "## Dependencies by loading:" << std::endl; + for ( const auto& [ loop_head, props ] : dependencies_by_loading ) { + std::cout << "Loop ID: " << loop_head << std::endl; + std::cout << "Loaded bits per loop: " << props.average_bits_per_loop.mean << std::endl; + for ( const auto& body : props.bodies ) { + const auto& bit_props = props.bit_values.at( body.node_id ); + std::cout << "- " << "`(" << body << ") → " << loop_head << "`" + << ", Bits: " << bit_props.average_bit_size.mean + << ", offset: " << bit_props.minimal_bit_offset << std::endl; + } } } } @@ -1176,11 +1227,12 @@ void iid_node_dependence_props::compute_path_counts_loops( nodes_to_counts& path // ------------------------------------------------------------------------------------------------ nodes_to_counts iid_node_dependence_props::compute_path_counts( const equation& path, - std::set< node_direction > const& all_leafs ) + std::set< node_direction > const& computation_subset ) { nodes_to_counts path_counts; - std::vector< node_direction > leafs = std::vector< node_direction >( all_leafs.begin(), all_leafs.end() ); + std::vector< node_direction > leafs = std::vector< node_direction >( computation_subset.begin(), + computation_subset.end() ); INVARIANT( leafs.size() == path.values.size() ); std::set< location_id::id_type > loop_heads = get_loop_heads( false ); @@ -1347,9 +1399,9 @@ std::vector< equation > iid_node_dependence_props::get_random_vector( const std: } // ------------------------------------------------------------------------------------------------ -std::set< node_direction > iid_node_dependence_props::get_leaf_subsets() +std::set< node_direction > iid_node_dependence_props::get_node_subsets_for_computation() { - std::set< node_direction > all_leafs; + std::set< node_direction > computation_subset; auto loop_heads = get_loop_heads( false ); for ( const auto& loop : dependencies_by_loops.loops ) { @@ -1360,16 +1412,16 @@ std::set< node_direction > iid_node_dependence_props::get_leaf_subsets() for ( const auto& body : loop.bodies ) { location_id::id_type body_id = body.node_id; if ( !loop_heads.contains( body_id ) ) { - all_leafs.insert( body ); + computation_subset.insert( body ); } } for ( const auto& [ head, _ ] : loop.heads ) { - all_leafs.insert( { head.node_id, !head.branching_direction } ); + computation_subset.insert( { head.node_id, !head.branching_direction } ); } } - return all_leafs; + return computation_subset; } // ------------------------------------------------------------------------------------------------ @@ -1563,7 +1615,7 @@ void iid_node_dependence_props::compute_dependencies_by_loops( const loop_head_t bool loop_head_end_direction = loop_heads_ending.at( loop_head_id ); node_direction loop_head_direction = { loop_head_id, loop_head_end_direction }; - std::unordered_set< location_id::id_type > all_ids; + std::set< location_id::id_type > all_ids; all_ids.insert( loop_head_id ); for ( const auto& loop_body_id : loop_bodies ) { all_ids.insert( loop_body_id.id ); @@ -1674,7 +1726,7 @@ void iid_dependencies::update_non_iid_nodes( sensitivity_analysis& sensitivity ) for ( branching_node* node : sensitivity.get_changed_nodes() ) { if ( node->is_did_branching() ) { location_id::id_type location_id = node->get_location_id().id; - if ( non_iid_nodes.insert( location_id ).second ) { + if ( ignored_nodes.insert( location_id ).second ) { id_to_equation_map.erase( location_id ); } } @@ -1686,7 +1738,7 @@ void iid_dependencies::process_node_dependence( branching_node* node ) { TMPROF_BLOCK(); - if ( non_iid_nodes.contains( node->get_location_id().id ) ) + if ( ignored_nodes.contains( node->get_location_id().id ) ) return; iid_node_dependence_props& props = id_to_equation_map[ node->get_location_id().id ]; @@ -1700,6 +1752,12 @@ void iid_dependencies::remove_node_dependence( location_id::id_type id ) if ( it != id_to_equation_map.end() ) { iid_node_generations_stats& stats = it->second.get_generations_stats(); stats.state = generation_state::STATE_COVERED; + ignored_nodes.insert( id ); + + if ( it != id_to_equation_map.end() ) { + auto next_it = std::next( it ); + next_it->second.combine_props( it->second ); + } if ( iid_dependencies::generate_more_data_after_coverage && !it->second.is_equal_branching_predicate() ) { stats.state = generation_state::STATE_GENERATION_MORE; @@ -1763,6 +1821,26 @@ std::optional< location_id::id_type > iid_dependencies::get_next_iid_node() return best_id; } + +// ------------------------------------------------------------------------------------------------ +iid_vector_analysis_statistics fuzzing::iid_dependencies::get_stats() const +{ + iid_vector_analysis_statistics stats; + + iid_vector_analysis_statistics_per_node node_stats; + for ( const auto& [ id, props ] : id_to_equation_map ) { + node_stats.generation_stats = props.get_generations_stats(); + node_stats.dependencies_by_loops = props.get_dependencies_by_loops(); + node_stats.dependencies_by_loading = props.get_dependencies_by_loading(); + + stats.iid_nodes_stats[ id ] = node_stats; + } + + stats.ignored_nodes = std::vector< location_id::id_type >( ignored_nodes.begin(), ignored_nodes.end() ); + + return stats; +} + // non member functions // ------------------------------------------------------------------------------------------------ std::vector< node_direction > get_path( branching_node* node ) @@ -1783,4 +1861,11 @@ std::vector< node_direction > get_path( branching_node* node ) return result; } +bool should_generate_more_data( const generation_state& state ) +{ + return state == generation_state::STATE_GENERATION_MORE || + state == generation_state::STATE_GENERATION_DATA_FOR_NEXT_NODE || + state == generation_state::STATE_GENERATING_ARTIFICIAL_DATA; +} + } // namespace fuzzing From 7730c4147becb051b8b0c8f8c9911e8de4a5faba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Wed, 12 Feb 2025 15:15:29 +0100 Subject: [PATCH 112/144] feat: remove two definitions of loop properties --- .../include/fuzzing/iid_vector_analysis.hpp | 20 +- src/fuzzing/src/iid_vector_analysis.cpp | 190 +++++++++++------- 2 files changed, 131 insertions(+), 79 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index 6c101c11..a6c8672a 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -93,6 +93,11 @@ struct dependent_loop_properties { std::optional< node_direction > chosen_loop_head; std::set< node_direction > bodies; + bool is_loading_loop; + std::set< node_direction > bit_dependent_nodes; + mean_counter< float > average_bits_per_loop; + std::map< location_id::id_type, loading_body_props > bit_values_per_node; + bool is_same( const std::set< location_id::id_type >& other_ids ) const; std::set< location_id::id_type > get_all_ids() const; std::set< location_id::id_type > get_loop_head_ids() const; @@ -113,7 +118,6 @@ struct dependencies_by_loops_t { struct iid_vector_analysis_statistics_per_node { iid_node_generations_stats generation_stats; dependencies_by_loops_t dependencies_by_loops; - std::map< location_id::id_type, loading_loops_props > dependencies_by_loading; }; struct iid_vector_analysis_statistics { @@ -271,10 +275,6 @@ struct iid_node_dependence_props { iid_node_generations_stats& get_generations_stats() { return stats; } const iid_node_generations_stats& get_generations_stats() const { return stats; } const dependencies_by_loops_t& get_dependencies_by_loops() const { return dependencies_by_loops; } - const std::map< location_id::id_type, loading_loops_props >& get_dependencies_by_loading() const - { - return dependencies_by_loading; - } bool should_generate() const; bool too_much_failed_in_row( int max_failed_generations_in_row ) const; @@ -298,11 +298,12 @@ struct iid_node_dependence_props { location_id::id_type loop_head_id, int minimum_count, bool use_random = false ); - int compute_loop_count_loading( nodes_to_counts& path_counts, + int compute_loop_count_loading_new( nodes_to_counts& path_counts, location_id::id_type id, const std::set< location_id::id_type >& loop_heads, - const loading_loops_props& props ); - void compute_path_counts_loading( nodes_to_counts& path_counts, + const dependent_loop_properties & props, + float loaded_bits_per_loop ); + void compute_path_counts_loading_new( nodes_to_counts& path_counts, const equation& path, const std::set< location_id::id_type >& loop_heads ); void compute_path_counts_loops( nodes_to_counts& path_counts, @@ -323,7 +324,7 @@ struct iid_node_dependence_props { void compute_loading_loops( branching_node* end_node, const loop_head_to_bodies_t& loop_heads_to_bodies, loop_head_to_loaded_bits_props& loading_loops ); - void compute_dependencies_by_loading( branching_node* end_node, + void compute_dependencies_by_loading_new( branching_node* end_node, const loop_head_to_bodies_t& loop_heads_to_bodies, const loop_endings& loop_heads_ending ); void compute_dependencies_by_loops( const loop_head_to_bodies_t& loop_heads_to_bodies, @@ -333,7 +334,6 @@ struct iid_node_dependence_props { equation_matrix matrix; dependencies_by_loops_t dependencies_by_loops; - std::map< location_id::id_type, loading_loops_props > dependencies_by_loading; iid_node_generations_stats stats; }; diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 45f2907f..9ab33edb 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -470,7 +470,7 @@ dependent_loop_properties& fuzzing::dependencies_by_loops_t::get_props_by_loop_h } } - throw std::runtime_error( "Loop head not found" ); + throw std::runtime_error( "Loop head not found: " + std::to_string(loop_head_id) ); } // equation_matrix @@ -789,8 +789,9 @@ void iid_node_dependence_props::process_node( branching_node* end_node ) matrix.process_node( end_node ); - compute_dependencies_by_loading( end_node, loop_heads_to_bodies, loop_heads_ending ); + // compute_dependencies_by_loading( end_node, loop_heads_to_bodies, loop_heads_ending ); compute_dependencies_by_loops( loop_heads_to_bodies, loop_heads_ending ); + compute_dependencies_by_loading_new( end_node, loop_heads_to_bodies, loop_heads_ending ); } // ------------------------------------------------------------------------------------------------ @@ -904,12 +905,20 @@ void fuzzing::iid_node_dependence_props::combine_props( const iid_node_dependenc void iid_node_dependence_props::print_dependencies() const { bool print_dependencies_by_loops = true; - bool print_dependencies_by_loading = false; + bool print_dependencies_by_loading = true; + + if ( !print_dependencies_by_loops && !print_dependencies_by_loading ) { + return; + } std::cout << "# Dependencies:" << std::endl; if ( print_dependencies_by_loops ) { std::cout << "## Dependencies by loops:" << std::endl; for ( const auto& loop : dependencies_by_loops.loops ) { + if ( loop.is_loading_loop ) { + continue; + } + std::cout << "Loop heads:" << std::endl; for ( const auto& [ head, head_props ] : loop.heads ) { std::cout << "- " << head << " (" << head_props.count << ")" << std::endl; @@ -919,20 +928,42 @@ void iid_node_dependence_props::print_dependencies() const for ( const auto& body : loop.bodies ) { std::cout << "- " << body << std::endl; } + + if ( loop.is_loading_loop ) { + std::cout << "Loading loop" << std::endl; + std::cout << "Average bits per loop: " << loop.average_bits_per_loop.mean << std::endl; + for ( const auto& [ body, body_props ] : loop.bit_values_per_node ) { + std::cout << "- " << body << ", Bits: " << body_props.average_bit_size.mean + << ", offset: " << body_props.minimal_bit_offset << std::endl; + } + } } } if ( print_dependencies_by_loading ) { std::cout << "## Dependencies by loading:" << std::endl; - for ( const auto& [ loop_head, props ] : dependencies_by_loading ) { - std::cout << "Loop ID: " << loop_head << std::endl; - std::cout << "Loaded bits per loop: " << props.average_bits_per_loop.mean << std::endl; - for ( const auto& body : props.bodies ) { - const auto& bit_props = props.bit_values.at( body.node_id ); - std::cout << "- " << "`(" << body << ") → " << loop_head << "`" - << ", Bits: " << bit_props.average_bit_size.mean - << ", offset: " << bit_props.minimal_bit_offset << std::endl; + for ( const auto& loop : dependencies_by_loops.loops ) { + if ( !loop.is_loading_loop ) { + continue; } + + std::cout << "Loop heads:" << std::endl; + for ( const auto& [ head, head_props ] : loop.heads ) { + std::cout << "- " << head << " (" << head_props.count << ")" << std::endl; + } + + std::cout << "Loop bodies:" << std::endl; + for ( const auto& body : loop.bodies ) { + std::cout << "- " << body << std::endl; + } + + std::cout << "Dependent nodes:" << std::endl; + for ( const auto& [ body, body_props ] : loop.bit_values_per_node ) { + std::cout << "- " << body << ", Bits: " << body_props.average_bit_size.mean + << ", offset: " << body_props.minimal_bit_offset << std::endl; + } + + std::cout << "Loaded bits per loop: " << loop.average_bits_per_loop.mean << std::endl; } } } @@ -1136,16 +1167,19 @@ void iid_node_dependence_props::compute_path_counts_for_nested_loops( nodes_to_c } // ------------------------------------------------------------------------------------------------ -int fuzzing::iid_node_dependence_props::compute_loop_count_loading( nodes_to_counts& path_counts, - location_id::id_type id, - const std::set< location_id::id_type >& loop_heads, - const loading_loops_props& props ) +int fuzzing::iid_node_dependence_props::compute_loop_count_loading_new( nodes_to_counts& path_counts, + location_id::id_type id, + const std::set< location_id::id_type >& loop_heads, + const dependent_loop_properties& props, + float loaded_bits_per_loop ) { - float loaded_per_loop = props.average_bits_per_loop.mean; - INVARIANT( loaded_per_loop > 0 ); + float loaded_per_loop = props.average_bits_per_loop.mean - loaded_bits_per_loop; + if ( loaded_per_loop <= 0 ) { + loaded_per_loop = 8; + } - float average_bits = props.bit_values.at( id ).average_bit_size.mean; - natural_32_bit offset = props.bit_values.at( id ).minimal_bit_offset; + float average_bits = props.bit_values_per_node.at( id ).average_bit_size.mean; + natural_32_bit offset = props.bit_values_per_node.at( id ).minimal_bit_offset; bool is_loop_head = true; if ( !loop_heads.contains( id ) ) { @@ -1166,26 +1200,67 @@ int fuzzing::iid_node_dependence_props::compute_loop_count_loading( nodes_to_cou } // ------------------------------------------------------------------------------------------------ -void iid_node_dependence_props::compute_path_counts_loading( nodes_to_counts& path_counts, - const equation& path, - const std::set< location_id::id_type >& loop_heads ) +void iid_node_dependence_props::compute_path_counts_loading_new( nodes_to_counts& path_counts, + const equation& path, + const std::set< location_id::id_type >& loop_heads ) { - for ( const auto& [ loading_head, props ] : dependencies_by_loading ) { + for ( const auto& loop_props : std::ranges::views::reverse( dependencies_by_loops.loops ) ) { + if ( !loop_props.is_loading_loop ) { + continue; + } + int loop_count = 0; + std::map< location_id::id_type, int > child_loop_counts; + std::map< location_id::id_type, float > loading_loops_count_bits; - for ( const auto& body : props.bodies ) { - if ( !path_counts.contains( body.node_id ) ) { + float loaded_bits_per_loop = 0.0f; + + for ( const auto& body : loop_props.bodies ) { + if ( !loop_heads.contains( body.node_id ) ) { continue; } + + const dependent_loop_properties& body_props = dependencies_by_loops.get_props_by_loop_head_id( body.node_id ); + if ( body_props.is_loading_loop ) { + loaded_bits_per_loop += body_props.average_bits_per_loop.mean; + } + } - int minimal_count = compute_loop_count_loading( path_counts, body.node_id, loop_heads, props ); + for ( const auto& body : loop_props.bit_dependent_nodes ) { + if ( !path_counts.contains( body.node_id ) ) { + continue; + } + + int minimal_count = compute_loop_count_loading_new( path_counts, body.node_id, loop_heads, loop_props, loaded_bits_per_loop ); loop_count = std::max( loop_count, minimal_count ); } - if ( props.end_direction ) { - path_counts[ loading_head ] = { loop_count, 1 }; + for ( const auto& body : loop_props.bodies ) { + auto& [ left_count, right_count ] = path_counts[ body.node_id ]; + + if ( loop_heads.contains( body.node_id ) ) { + child_loop_counts[ body.node_id ] = std::max( left_count, right_count ); + } else { + loop_count = std::max( loop_count, left_count + right_count ); + } + } + + if ( child_loop_counts.empty() ) { + for ( const auto& [ head, _ ] : loop_props.heads ) { + int end_count = head == ( *loop_props.chosen_loop_head ) ? 1 : 0; + + if ( head.branching_direction ) { + path_counts[ head.node_id ] = { loop_count, end_count }; + } else { + path_counts[ head.node_id ] = { end_count, loop_count }; + } + } } else { - path_counts[ loading_head ] = { 1, loop_count }; + compute_path_counts_for_nested_loops( path_counts, + child_loop_counts, + ( *loop_props.chosen_loop_head ).node_id, + loop_count, + iid_dependencies::random_nested_loops ); } } } @@ -1196,7 +1271,7 @@ void iid_node_dependence_props::compute_path_counts_loops( nodes_to_counts& path const std::set< location_id::id_type >& loop_heads ) { for ( const auto& props : std::ranges::views::reverse( dependencies_by_loops.loops ) ) { - if ( props.bodies.empty() ) { + if ( props.bodies.empty() || props.is_loading_loop ) { continue; } @@ -1273,7 +1348,7 @@ nodes_to_counts iid_node_dependence_props::compute_path_counts( const equation& } compute_path_counts_loops( path_counts, path, loop_heads ); - compute_path_counts_loading( path_counts, path, loop_heads ); + compute_path_counts_loading_new( path_counts, path, loop_heads ); return path_counts; } @@ -1405,7 +1480,7 @@ std::set< node_direction > iid_node_dependence_props::get_node_subsets_for_compu auto loop_heads = get_loop_heads( false ); for ( const auto& loop : dependencies_by_loops.loops ) { - if ( loop.bodies.empty() ) { + if ( loop.bodies.empty() || loop.is_loading_loop ) { continue; } @@ -1516,9 +1591,9 @@ void fuzzing::iid_node_dependence_props::compute_loading_loops( branching_node* } // ------------------------------------------------------------------------------------------------ -void iid_node_dependence_props::compute_dependencies_by_loading( branching_node* end_node, - const loop_head_to_bodies_t& loop_heads_to_bodies, - const loop_endings& loop_heads_ending ) +void iid_node_dependence_props::compute_dependencies_by_loading_new( branching_node* end_node, + const loop_head_to_bodies_t& loop_heads_to_bodies, + const loop_endings& loop_heads_ending ) { loop_head_to_loaded_bits_props loading_loops; compute_loading_loops( end_node, loop_heads_to_bodies, loading_loops ); @@ -1572,7 +1647,8 @@ void iid_node_dependence_props::compute_dependencies_by_loading( branching_node* for ( const auto& [ loop_head_id, body ] : loop_to_props ) { auto loading_props = loading_loops.at( loop_head_id ); - auto& dependencies = dependencies_by_loading[ loop_head_id ]; + dependent_loop_properties& dependencies = dependencies_by_loops.get_props_by_loop_head_id( loop_head_id ); + dependencies.is_loading_loop = true; natural_32_bit loaded_bits = loading_props.max - loading_props.min; double per_loop = double( loaded_bits ) / double( loading_props.loop_count ); @@ -1581,7 +1657,7 @@ void iid_node_dependence_props::compute_dependencies_by_loading( branching_node* bool loop_head_end_direction = loop_heads_ending.at( loop_head_id ); for ( const auto& [ body_id, props ] : body ) { - auto& body_props = dependencies.bit_values[ body_id ]; + auto& body_props = dependencies.bit_values_per_node[ body_id ]; natural_32_bit minimal_offset = props.min - loading_props.min; INVARIANT( minimal_offset >= 0 ); body_props.minimal_bit_offset = std::min( body_props.minimal_bit_offset, minimal_offset ); @@ -1594,21 +1670,21 @@ void iid_node_dependence_props::compute_dependencies_by_loading( branching_node* node_direction node_id_direction = { body_id, direction }; if ( matrix.contains( node_id_direction ) ) { - dependencies.bodies.insert( node_id_direction ); - dependencies.end_direction = loop_head_end_direction; + dependencies.bit_dependent_nodes.insert( node_id_direction ); } } } } } + // ------------------------------------------------------------------------------------------------ void iid_node_dependence_props::compute_dependencies_by_loops( const loop_head_to_bodies_t& loop_heads_to_bodies, const loop_endings& loop_heads_ending ) { for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { location_id::id_type loop_head_id = loop_head.id; - if ( !loop_heads_ending.contains( loop_head_id ) || dependencies_by_loading.contains( loop_head_id ) ) { + if ( !loop_heads_ending.contains( loop_head_id ) ) { continue; } @@ -1644,21 +1720,6 @@ void iid_node_dependence_props::compute_dependencies_by_loops( const loop_head_t dependencies_by_loops.merge_properties(); - for ( auto it = dependencies_by_loops.loops.begin(); it != dependencies_by_loops.loops.end(); ) { - bool removed = false; - for ( const auto& [ head, _ ] : it->heads ) { - if ( dependencies_by_loading.contains( head.node_id ) ) { - it = dependencies_by_loops.loops.erase( it ); - removed = true; - break; - } - } - - if ( !removed ) { - ++it; - } - } - std::sort( dependencies_by_loops.loops.begin(), dependencies_by_loops.loops.end(), []( const auto& a, const auto& b ) { @@ -1687,12 +1748,6 @@ possible_path iid_node_dependence_props::generate_path_from_node_counts( const n } } - auto it_loading = dependencies_by_loading.find( id ); - if ( it_loading != dependencies_by_loading.end() ) { - is_loop_head = true; - loop_head_end_direction = it_loading->second.end_direction; - } - path_node_props props = { counts, is_loop_head, loop_head_end_direction }; path.emplace( id, props ); } @@ -1705,14 +1760,12 @@ std::set< location_id::id_type > iid_node_dependence_props::get_loop_heads( bool { std::set< location_id::id_type > loop_heads; for ( const auto& props : dependencies_by_loops.loops ) { - for ( const auto& [ head, _ ] : props.heads ) { - loop_heads.insert( head.node_id ); + if ( props.is_loading_loop && !include_loading_loops ) { + continue; } - } - if ( include_loading_loops ) { - for ( const auto& [ loading_head, _ ] : dependencies_by_loading ) { - loop_heads.insert( loading_head ); + for ( const auto& [ head, _ ] : props.heads ) { + loop_heads.insert( head.node_id ); } } @@ -1826,12 +1879,11 @@ std::optional< location_id::id_type > iid_dependencies::get_next_iid_node() iid_vector_analysis_statistics fuzzing::iid_dependencies::get_stats() const { iid_vector_analysis_statistics stats; - + iid_vector_analysis_statistics_per_node node_stats; for ( const auto& [ id, props ] : id_to_equation_map ) { node_stats.generation_stats = props.get_generations_stats(); node_stats.dependencies_by_loops = props.get_dependencies_by_loops(); - node_stats.dependencies_by_loading = props.get_dependencies_by_loading(); stats.iid_nodes_stats[ id ] = node_stats; } From d117ccbe35ce6bdbbf28c60f698102f29a857725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Wed, 12 Feb 2025 17:26:16 +0100 Subject: [PATCH 113/144] feat: renamed methods and types --- .../include/fuzzing/iid_vector_analysis.hpp | 137 ++++---- src/fuzzing/src/iid_vector_analysis.cpp | 310 +++++++++--------- 2 files changed, 214 insertions(+), 233 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index a6c8672a..3419359d 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -22,15 +22,15 @@ struct mean_counter { void add( T value ); }; -struct node_direction { +struct node_id_with_direction { location_id::id_type node_id; bool branching_direction; - auto operator<=>( node_direction const& other ) const; - bool operator==( node_direction const& other ) const = default; - friend std::ostream& operator<<( std::ostream& os, const node_direction& nav ) + auto operator<=>( node_id_with_direction const& other ) const; + bool operator==( node_id_with_direction const& other ) const = default; + friend std::ostream& operator<<( std::ostream& os, const node_id_with_direction& nav ) { - return os << nav.node_id << " " << ( nav.branching_direction ? "right" : "left" ); + return os << nav.node_id << " " << ( nav.branching_direction ? "True" : "False" ); } }; @@ -71,32 +71,25 @@ struct iid_node_generations_stats { failed_generation_method last_failed_method = METHOD_GENERATE_ARTIFICIAL_DATA; }; -struct loading_body_props { - mean_counter< float > average_bit_size; +struct loaded_bits_props { + mean_counter< float > average_bits_read; natural_32_bit minimal_bit_offset = std::numeric_limits< natural_32_bit >::max(); }; -struct loading_loops_props { - bool end_direction; - std::set< node_direction > bodies; - mean_counter< float > average_bits_per_loop; - std::map< location_id::id_type, loading_body_props > bit_values; -}; - -struct dependent_loop_head_properties { +struct loop_head_properties { int count; }; -struct dependent_loop_properties { - std::map< node_direction, dependent_loop_head_properties > heads; +struct loop_properties { + std::optional< node_id_with_direction > chosen_loop_head; + std::map< node_id_with_direction, loop_head_properties > heads; - std::optional< node_direction > chosen_loop_head; - std::set< node_direction > bodies; + std::set< node_id_with_direction > bodies; bool is_loading_loop; - std::set< node_direction > bit_dependent_nodes; - mean_counter< float > average_bits_per_loop; - std::map< location_id::id_type, loading_body_props > bit_values_per_node; + std::set< node_id_with_direction > nodes_dependent_by_loading; + mean_counter< float > loaded_bits_per_loop; + std::map< location_id::id_type, loaded_bits_props > bits_read_by_node; bool is_same( const std::set< location_id::id_type >& other_ids ) const; std::set< location_id::id_type > get_all_ids() const; @@ -106,23 +99,22 @@ struct dependent_loop_properties { void set_chosen_loop_head(); }; -struct dependencies_by_loops_t { - std::vector< dependent_loop_properties > loops; +struct loop_dependencies { + std::vector< loop_properties > loops; - dependent_loop_properties& get_props( const std::set< location_id::id_type >& ids, - location_id::id_type loop_head_id ); + loop_properties& get_props( const std::set< location_id::id_type >& ids, location_id::id_type loop_head_id ); void merge_properties(); - dependent_loop_properties& get_props_by_loop_head_id( location_id::id_type loop_head_id ); + loop_properties& get_props_by_loop_head_id( location_id::id_type loop_head_id ); }; struct iid_vector_analysis_statistics_per_node { iid_node_generations_stats generation_stats; - dependencies_by_loops_t dependencies_by_loops; + loop_dependencies loop_to_properties; }; struct iid_vector_analysis_statistics { std::map< location_id::id_type, iid_vector_analysis_statistics_per_node > iid_nodes_stats; - std::vector< location_id::id_type > ignored_nodes; + std::vector< location_id::id_type > ignored_node_ids; }; @@ -135,8 +127,8 @@ struct node_counts { }; -struct path_node_props { - path_node_props( node_counts computed_counts, bool is_loop_head, bool loop_head_end_direction ) +struct node_props_in_path { + node_props_in_path( node_counts computed_counts, bool is_loop_head, bool loop_head_end_direction ) : computed_counts( computed_counts ) , taken_counts( { 0, 0 } ) , is_loop_head( is_loop_head ) @@ -150,7 +142,7 @@ struct path_node_props { float_32_bit get_false_direction_probability() const; - friend std::ostream& operator<<( std::ostream& os, const path_node_props& eq ) + friend std::ostream& operator<<( std::ostream& os, const node_props_in_path& eq ) { os << "L-" << eq.computed_counts.left_count << " R-" << eq.computed_counts.right_count; if ( eq.is_loop_head ) { @@ -170,18 +162,18 @@ struct path_node_props { }; -struct possible_path { - possible_path( std::map< location_id::id_type, path_node_props > path ) +struct generated_path { + generated_path( std::map< location_id::id_type, node_props_in_path > path ) : path( std::move( path ) ) {} - possible_path() = default; + generated_path() = default; bool contains( location_id::id_type id ) const; - std::map< location_id::id_type, path_node_props > get_path() const; - path_node_props& get_props( location_id::id_type id ) { return path.at( id ); } + std::map< location_id::id_type, node_props_in_path > get_path() const; + node_props_in_path& get_props( location_id::id_type id ) { return path.at( id ); } - friend std::ostream& operator<<( std::ostream& os, const possible_path& eq ) + friend std::ostream& operator<<( std::ostream& os, const generated_path& eq ) { for ( const auto& [ id, props ] : eq.path ) { os << id << ": " << props << std::endl; @@ -191,7 +183,7 @@ struct possible_path { } private: - std::map< location_id::id_type, path_node_props > path; + std::map< location_id::id_type, node_props_in_path > path; }; struct equation { @@ -231,23 +223,23 @@ struct equation { } }; -struct loaded_bits_props { +struct loaded_bits_counter { natural_32_bit min; natural_32_bit max; int loop_count; }; -using loop_head_to_loaded_bits_props = std::unordered_map< location_id::id_type, loaded_bits_props >; +using loop_head_to_loaded_bits_counter = std::unordered_map< location_id::id_type, loaded_bits_counter >; using loop_endings = std::map< location_id::id_type, bool >; using loop_head_to_bodies_t = std::unordered_map< location_id, std::unordered_set< location_id > >; using nodes_to_counts = std::map< location_id::id_type, node_counts >; struct equation_matrix { - equation_matrix get_submatrix( std::set< node_direction > const& subset, bool unique ) const; + equation_matrix get_submatrix( std::set< node_id_with_direction > const& subset, bool unique ) const; void process_node( branching_node* end_node ); void add_equation( branching_node* end_node ); - bool contains( node_direction const& node ) const; + bool contains( node_id_with_direction const& node ) const; std::pair< std::size_t, std::size_t > get_dimensions() const; std::map< equation, int > compute_vectors_with_hits(); std::vector< equation >& get_matrix(); @@ -266,15 +258,15 @@ struct equation_matrix { std::vector< equation > matrix; std::vector< branching_node* > all_paths; - std::set< node_direction > nodes; + std::set< node_id_with_direction > nodes; }; struct iid_node_dependence_props { - possible_path generate_probabilities(); + generated_path generate_probabilities(); void process_node( branching_node* end_node ); iid_node_generations_stats& get_generations_stats() { return stats; } const iid_node_generations_stats& get_generations_stats() const { return stats; } - const dependencies_by_loops_t& get_dependencies_by_loops() const { return dependencies_by_loops; } + const loop_dependencies& get_dependencies_by_loops() const { return loop_to_properties; } bool should_generate() const; bool too_much_failed_in_row( int max_failed_generations_in_row ) const; @@ -291,25 +283,26 @@ struct iid_node_dependence_props { private: void generate_vectors_if_not_enough_data( std::vector< equation >& best_vectors, equation_matrix& submatrix ); std::optional< std::vector< equation > > get_best_vectors( equation_matrix& submatrix, int number_of_vectors ); - possible_path return_empty_path(); - possible_path return_path( const possible_path& path ); - void compute_path_counts_for_nested_loops( nodes_to_counts& path_counts, + generated_path return_empty_path(); + generated_path return_path( const generated_path& path ); + void compute_node_counts_for_nested_loops( nodes_to_counts& path_counts, std::map< location_id::id_type, int >& child_loop_counts, location_id::id_type loop_head_id, int minimum_count, bool use_random = false ); - int compute_loop_count_loading_new( nodes_to_counts& path_counts, - location_id::id_type id, - const std::set< location_id::id_type >& loop_heads, - const dependent_loop_properties & props, - float loaded_bits_per_loop ); - void compute_path_counts_loading_new( nodes_to_counts& path_counts, - const equation& path, - const std::set< location_id::id_type >& loop_heads ); - void compute_path_counts_loops( nodes_to_counts& path_counts, - const equation& path, - const std::set< location_id::id_type >& loop_heads ); - nodes_to_counts compute_path_counts( const equation& path, std::set< node_direction > const& all_leafs ); + int compute_loading_loop_interation( nodes_to_counts& path_counts, + location_id::id_type id, + const std::set< location_id::id_type >& loop_heads, + const loop_properties& props, + float loaded_bits_per_loop ); + void compute_node_counts_for_loading_loops( nodes_to_counts& path_counts, + const equation& path, + const std::set< location_id::id_type >& loop_heads ); + void compute_node_counts_for_loops( nodes_to_counts& path_counts, + const equation& path, + const std::set< location_id::id_type >& loop_heads ); + nodes_to_counts compute_node_counts( const equation& path, + std::set< node_id_with_direction > const& all_leafs ); std::vector< equation > compute_best_vectors( const std::map< equation, int >& vectors_with_hits, int number_of_vectors, bool use_random, @@ -319,27 +312,27 @@ struct iid_node_dependence_props { equation& best_vector ); std::vector< equation > get_random_vector( const std::map< equation, int >& vectors_with_hits, int number_of_vectors ); - std::set< node_direction > get_node_subsets_for_computation(); + std::set< node_id_with_direction > get_node_subsets_for_computation(); loop_endings get_loop_heads_ending( branching_node* end_node, loop_head_to_bodies_t& loop_heads_to_bodies ); void compute_loading_loops( branching_node* end_node, const loop_head_to_bodies_t& loop_heads_to_bodies, - loop_head_to_loaded_bits_props& loading_loops ); - void compute_dependencies_by_loading_new( branching_node* end_node, + loop_head_to_loaded_bits_counter& loading_loops ); + void compute_dependencies_by_loading( branching_node* end_node, const loop_head_to_bodies_t& loop_heads_to_bodies, const loop_endings& loop_heads_ending ); void compute_dependencies_by_loops( const loop_head_to_bodies_t& loop_heads_to_bodies, const loop_endings& loop_heads_ending ); - possible_path generate_path_from_node_counts( const nodes_to_counts& path_counts ); + generated_path generate_path_from_node_counts( const nodes_to_counts& path_counts ); std::set< location_id::id_type > get_loop_heads( bool include_loading_loops = true ); equation_matrix matrix; - dependencies_by_loops_t dependencies_by_loops; + loop_dependencies loop_to_properties; iid_node_generations_stats stats; }; struct iid_dependencies { - void update_non_iid_nodes( sensitivity_analysis& sensitivity ); + void update_ignored_nodes( sensitivity_analysis& sensitivity ); void process_node_dependence( branching_node* node ); void remove_node_dependence( location_id::id_type id ); iid_node_dependence_props& get_props( location_id::id_type id ); @@ -348,22 +341,22 @@ struct iid_dependencies { iid_vector_analysis_statistics get_stats() const; private: - std::map< location_id::id_type, iid_node_dependence_props > id_to_equation_map; - std::set< location_id::id_type > ignored_nodes; + std::map< location_id::id_type, iid_node_dependence_props > node_id_to_equation_map; + std::set< location_id::id_type > ignored_node_ids; public: // Configurations - inline static bool random_nested_loops = false; + inline static bool random_nested_loop_counts = false; inline static bool random_direction_in_path = true; inline static bool generate_more_data_after_coverage = true; - inline static int minimal_max_generation_after_covered = 10; inline static int max_failed_generations_in_row = 2; + inline static int minimal_max_generation_after_covered = 10; inline static int minimal_max_generation_for_other_node = 10; inline static int minimal_max_generation_artificial_data = 5; inline static float percentage_to_add_to_path = 0.4; inline static bool create_artificial_data = true; }; -std::vector< node_direction > get_path( branching_node* node ); +std::vector< node_id_with_direction > get_path( branching_node* node ); bool should_generate_more_data( const generation_state& state ); } // namespace fuzzing diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 9ab33edb..12307565 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -25,9 +25,9 @@ inline void fuzzing::mean_counter< T >::add( T value ) // ------------------------------------------------------------------------------------------------ int node_counts::get_max_count() const { return std::max( left_count, right_count ); } -// path_node_props +// node_props_in_path // ------------------------------------------------------------------------------------------------ -bool path_node_props::get_desired_direction() const +bool node_props_in_path::get_desired_direction() const { INVARIANT( computed_counts.left_count + computed_counts.right_count > 0 ); @@ -56,7 +56,7 @@ bool path_node_props::get_desired_direction() const } // ------------------------------------------------------------------------------------------------ -bool path_node_props::can_go_direction( bool direction ) const +bool node_props_in_path::can_go_direction( bool direction ) const { if ( direction ) { return taken_counts.right_count < computed_counts.right_count; @@ -66,7 +66,7 @@ bool path_node_props::can_go_direction( bool direction ) const } // ------------------------------------------------------------------------------------------------ -void path_node_props::go_direction( bool direction ) +void node_props_in_path::go_direction( bool direction ) { if ( direction ) { taken_counts.right_count++; @@ -81,14 +81,14 @@ void path_node_props::go_direction( bool direction ) } // ------------------------------------------------------------------------------------------------ -bool path_node_props::can_take_next_direction() const +bool node_props_in_path::can_take_next_direction() const { return taken_counts.left_count < computed_counts.left_count || taken_counts.right_count < computed_counts.right_count; } // ------------------------------------------------------------------------------------------------ -float_32_bit path_node_props::get_false_direction_probability() const +float_32_bit node_props_in_path::get_false_direction_probability() const { INVARIANT( computed_counts.left_count + computed_counts.right_count > 0 ); @@ -97,25 +97,23 @@ float_32_bit path_node_props::get_false_direction_probability() const } // ------------------------------------------------------------------------------------------------ -bool path_node_props::get_preferred_direction_loop_head() const +bool node_props_in_path::get_preferred_direction_loop_head() const { auto is_depleted = []( int computed, int taken ) { return computed == taken; }; if ( !loop_head_end_direction ) { - // INVARIANT( computed_counts.left_count == 1 ); return !is_depleted( computed_counts.right_count, taken_counts.right_count ); } else { - // INVARIANT( computed_counts.right_count == 1 ); return is_depleted( computed_counts.left_count, taken_counts.left_count ); } } -// possible_path +// generated_path // ------------------------------------------------------------------------------------------------ -bool possible_path::contains( location_id::id_type id ) const { return path.contains( id ); } +bool generated_path::contains( location_id::id_type id ) const { return path.contains( id ); } // ------------------------------------------------------------------------------------------------ -std::map< location_id::id_type, path_node_props > possible_path::get_path() const { return path; } +std::map< location_id::id_type, node_props_in_path > generated_path::get_path() const { return path; } // equation // ------------------------------------------------------------------------------------------------ @@ -294,9 +292,9 @@ bool equation::is_linear_dependent( const equation& other ) const return std::abs( best_value / other.best_value - ratio ) < 1e-7; } -// node_direction +// node_id_with_direction // ------------------------------------------------------------------------------------------------ -auto node_direction::operator<=>( node_direction const& other ) const +auto node_id_with_direction::operator<=>( node_id_with_direction const& other ) const { if ( auto const cmp = node_id <=> other.node_id; cmp != 0 ) return cmp; @@ -304,15 +302,15 @@ auto node_direction::operator<=>( node_direction const& other ) const return branching_direction <=> other.branching_direction; } -// dependent_loop_properties +// loop_properties // ------------------------------------------------------------------------------------------------ -bool fuzzing::dependent_loop_properties::is_same( const std::set< location_id::id_type >& other_ids ) const +bool fuzzing::loop_properties::is_same( const std::set< location_id::id_type >& other_ids ) const { return get_all_ids() == other_ids; } // ------------------------------------------------------------------------------------------------ -std::set< location_id::id_type > fuzzing::dependent_loop_properties::get_all_ids() const +std::set< location_id::id_type > fuzzing::loop_properties::get_all_ids() const { std::set< location_id::id_type > all_ids; @@ -328,7 +326,7 @@ std::set< location_id::id_type > fuzzing::dependent_loop_properties::get_all_ids } // ------------------------------------------------------------------------------------------------ -std::set< location_id::id_type > fuzzing::dependent_loop_properties::get_loop_head_ids() const +std::set< location_id::id_type > fuzzing::loop_properties::get_loop_head_ids() const { std::set< location_id::id_type > loop_head_ids; @@ -340,7 +338,7 @@ std::set< location_id::id_type > fuzzing::dependent_loop_properties::get_loop_he } // ------------------------------------------------------------------------------------------------ -std::set< location_id::id_type > fuzzing::dependent_loop_properties::get_body_ids() const +std::set< location_id::id_type > fuzzing::loop_properties::get_body_ids() const { std::set< location_id::id_type > body_ids; @@ -352,7 +350,7 @@ std::set< location_id::id_type > fuzzing::dependent_loop_properties::get_body_id } // ------------------------------------------------------------------------------------------------ -location_id::id_type fuzzing::dependent_loop_properties::get_smallest_loop_head_id() const +location_id::id_type fuzzing::loop_properties::get_smallest_loop_head_id() const { location_id::id_type smallest_id = heads.begin()->first.node_id; @@ -366,7 +364,7 @@ location_id::id_type fuzzing::dependent_loop_properties::get_smallest_loop_head_ } // ------------------------------------------------------------------------------------------------ -void fuzzing::dependent_loop_properties::set_chosen_loop_head() +void fuzzing::loop_properties::set_chosen_loop_head() { for ( const auto& [ head, props ] : heads ) { if ( !chosen_loop_head.has_value() ) { @@ -379,12 +377,12 @@ void fuzzing::dependent_loop_properties::set_chosen_loop_head() } } -// dependencies_by_loops_t +// loop_dependencies // ------------------------------------------------------------------------------------------------ -dependent_loop_properties& fuzzing::dependencies_by_loops_t::get_props( const std::set< location_id::id_type >& ids, - location_id::id_type loop_head_id ) +loop_properties& fuzzing::loop_dependencies::get_props( const std::set< location_id::id_type >& ids, + location_id::id_type loop_head_id ) { - for ( dependent_loop_properties& loop : loops ) { + for ( loop_properties& loop : loops ) { for ( const auto& [ head, props ] : loop.heads ) { if ( head.node_id == loop_head_id ) { return loop; @@ -401,7 +399,7 @@ dependent_loop_properties& fuzzing::dependencies_by_loops_t::get_props( const st } // ------------------------------------------------------------------------------------------------ -void fuzzing::dependencies_by_loops_t::merge_properties() +void fuzzing::loop_dependencies::merge_properties() { for ( auto it = loops.begin(); it != loops.end(); it++ ) { std::set< location_id::id_type > head_ids = it->get_loop_head_ids(); @@ -448,21 +446,12 @@ void fuzzing::dependencies_by_loops_t::merge_properties() } } } - - // Not sure if deleting loops with only loop heads is a good idea - // for ( auto it = loops.begin(); it != loops.end(); ) { - // if ( it->bodies.empty() ) { - // it = loops.erase( it ); - // } else { - // ++it; - // } - // } } // ------------------------------------------------------------------------------------------------ -dependent_loop_properties& fuzzing::dependencies_by_loops_t::get_props_by_loop_head_id( location_id::id_type loop_head_id ) +loop_properties& fuzzing::loop_dependencies::get_props_by_loop_head_id( location_id::id_type loop_head_id ) { - for ( dependent_loop_properties& loop : loops ) { + for ( loop_properties& loop : loops ) { for ( const auto& [ head, props ] : loop.heads ) { if ( head.node_id == loop_head_id ) { return loop; @@ -470,12 +459,12 @@ dependent_loop_properties& fuzzing::dependencies_by_loops_t::get_props_by_loop_h } } - throw std::runtime_error( "Loop head not found: " + std::to_string(loop_head_id) ); + throw std::runtime_error( "Loop head not found: " + std::to_string( loop_head_id ) ); } // equation_matrix // ------------------------------------------------------------------------------------------------ -equation_matrix equation_matrix::get_submatrix( std::set< node_direction > const& subset, bool unique ) const +equation_matrix equation_matrix::get_submatrix( std::set< node_id_with_direction > const& subset, bool unique ) const { equation_matrix result; result.nodes = subset; @@ -484,7 +473,7 @@ equation_matrix equation_matrix::get_submatrix( std::set< node_direction > const const equation& row = matrix[ i ]; std::vector< int > new_row_values; - for ( const node_direction& nav : subset ) { + for ( const node_id_with_direction& nav : subset ) { auto it = std::find( nodes.begin(), nodes.end(), nav ); if ( it != nodes.end() ) { new_row_values.push_back( row.values[ std::distance( nodes.begin(), it ) ] ); @@ -512,9 +501,9 @@ void equation_matrix::process_node( branching_node* end_node ) { all_paths.push_back( end_node ); - std::vector< node_direction > path = get_path( end_node ); + std::vector< node_id_with_direction > path = get_path( end_node ); bool new_node = false; - for ( const node_direction& nav : path ) { + for ( const node_id_with_direction& nav : path ) { for ( bool direction : { true, false } ) { auto [ it, inserted ] = nodes.insert( { nav.node_id, direction } ); new_node |= inserted; @@ -533,14 +522,14 @@ void equation_matrix::add_equation( branching_node* end_node ) { TMPROF_BLOCK(); - std::map< node_direction, int > directions_in_path; - for ( const node_direction& navigation : nodes ) { + std::map< node_id_with_direction, int > directions_in_path; + for ( const node_id_with_direction& navigation : nodes ) { directions_in_path[ navigation ] = 0; } - std::vector< node_direction > path_nodes = get_path( end_node ); + std::vector< node_id_with_direction > path_nodes = get_path( end_node ); - for ( const node_direction& nav : path_nodes ) { + for ( const node_id_with_direction& nav : path_nodes ) { if ( nodes.contains( nav ) ) { directions_in_path[ nav ]++; } @@ -556,7 +545,7 @@ void equation_matrix::add_equation( branching_node* end_node ) } // ------------------------------------------------------------------------------------------------ -bool equation_matrix::contains( node_direction const& node ) const { return nodes.contains( node ); } +bool equation_matrix::contains( node_id_with_direction const& node ) const { return nodes.contains( node ); } // ------------------------------------------------------------------------------------------------ std::pair< std::size_t, std::size_t > equation_matrix::get_dimensions() const @@ -701,7 +690,7 @@ equation_matrix::get_new_subset_counts_from_vectors( const std::vector< equation void equation_matrix::print_matrix() { std::cout << "# Matrix:" << std::endl; - for ( const node_direction& nav : nodes ) { + for ( const node_id_with_direction& nav : nodes ) { std::cout << nav << " "; } std::cout << std::endl; @@ -735,12 +724,12 @@ void equation_matrix::recompute_matrix() // iid_node_dependence_props // ------------------------------------------------------------------------------------------------ -possible_path iid_node_dependence_props::generate_probabilities() +generated_path iid_node_dependence_props::generate_probabilities() { stats.method_calls++; - std::set< node_direction > computation_subset = get_node_subsets_for_computation(); - if ( computation_subset.empty() || dependencies_by_loops.loops.empty() ) { + std::set< node_id_with_direction > computation_subset = get_node_subsets_for_computation(); + if ( computation_subset.empty() || loop_to_properties.loops.empty() ) { return {}; } @@ -773,10 +762,10 @@ possible_path iid_node_dependence_props::generate_probabilities() return return_empty_path(); } - nodes_to_counts node_counts = compute_path_counts( new_subset_counts.value(), computation_subset ); - possible_path path = generate_path_from_node_counts( node_counts ); - - // std::cout << "Generated path: " << std::endl << path << std::endl; + nodes_to_counts node_counts = compute_node_counts( new_subset_counts.value(), computation_subset ); + generated_path path = generate_path_from_node_counts( node_counts ); + + std::cout << "Generated path: " << std::endl << path << std::endl; return return_path( path ); } @@ -789,9 +778,8 @@ void iid_node_dependence_props::process_node( branching_node* end_node ) matrix.process_node( end_node ); - // compute_dependencies_by_loading( end_node, loop_heads_to_bodies, loop_heads_ending ); compute_dependencies_by_loops( loop_heads_to_bodies, loop_heads_ending ); - compute_dependencies_by_loading_new( end_node, loop_heads_to_bodies, loop_heads_ending ); + compute_dependencies_by_loading( end_node, loop_heads_to_bodies, loop_heads_ending ); } // ------------------------------------------------------------------------------------------------ @@ -873,10 +861,10 @@ bool iid_node_dependence_props::is_equal_branching_predicate() const // ------------------------------------------------------------------------------------------------ void fuzzing::iid_node_dependence_props::combine_props( const iid_node_dependence_props& other ) { - std::vector< dependent_loop_properties > loops_to_add; + std::vector< loop_properties > loops_to_add; - for ( const auto& other_loop : other.dependencies_by_loops.loops ) { - for ( const auto& loop : dependencies_by_loops.loops ) { + for ( const auto& other_loop : other.loop_to_properties.loops ) { + for ( const auto& loop : loop_to_properties.loops ) { std::set< location_id::id_type > other_ids = other_loop.get_loop_head_ids(); std::set< location_id::id_type > ids = loop.get_loop_head_ids(); @@ -895,10 +883,10 @@ void fuzzing::iid_node_dependence_props::combine_props( const iid_node_dependenc } for ( const auto& loop : loops_to_add ) { - dependencies_by_loops.loops.push_back( loop ); + loop_to_properties.loops.push_back( loop ); } - dependencies_by_loops.merge_properties(); + loop_to_properties.merge_properties(); } // ------------------------------------------------------------------------------------------------ @@ -914,7 +902,7 @@ void iid_node_dependence_props::print_dependencies() const std::cout << "# Dependencies:" << std::endl; if ( print_dependencies_by_loops ) { std::cout << "## Dependencies by loops:" << std::endl; - for ( const auto& loop : dependencies_by_loops.loops ) { + for ( const auto& loop : loop_to_properties.loops ) { if ( loop.is_loading_loop ) { continue; } @@ -931,9 +919,9 @@ void iid_node_dependence_props::print_dependencies() const if ( loop.is_loading_loop ) { std::cout << "Loading loop" << std::endl; - std::cout << "Average bits per loop: " << loop.average_bits_per_loop.mean << std::endl; - for ( const auto& [ body, body_props ] : loop.bit_values_per_node ) { - std::cout << "- " << body << ", Bits: " << body_props.average_bit_size.mean + std::cout << "Average bits per loop: " << loop.loaded_bits_per_loop.mean << std::endl; + for ( const auto& [ body, body_props ] : loop.bits_read_by_node ) { + std::cout << "- " << body << ", Bits: " << body_props.average_bits_read.mean << ", offset: " << body_props.minimal_bit_offset << std::endl; } } @@ -942,7 +930,7 @@ void iid_node_dependence_props::print_dependencies() const if ( print_dependencies_by_loading ) { std::cout << "## Dependencies by loading:" << std::endl; - for ( const auto& loop : dependencies_by_loops.loops ) { + for ( const auto& loop : loop_to_properties.loops ) { if ( !loop.is_loading_loop ) { continue; } @@ -958,12 +946,12 @@ void iid_node_dependence_props::print_dependencies() const } std::cout << "Dependent nodes:" << std::endl; - for ( const auto& [ body, body_props ] : loop.bit_values_per_node ) { - std::cout << "- " << body << ", Bits: " << body_props.average_bit_size.mean + for ( const auto& [ body, body_props ] : loop.bits_read_by_node ) { + std::cout << "- " << body << ", Bits: " << body_props.average_bits_read.mean << ", offset: " << body_props.minimal_bit_offset << std::endl; } - std::cout << "Loaded bits per loop: " << loop.average_bits_per_loop.mean << std::endl; + std::cout << "Loaded bits per loop: " << loop.loaded_bits_per_loop.mean << std::endl; } } } @@ -1061,15 +1049,15 @@ fuzzing::iid_node_dependence_props::get_best_vectors( equation_matrix& submatrix } // ------------------------------------------------------------------------------------------------ -possible_path iid_node_dependence_props::return_empty_path() +generated_path iid_node_dependence_props::return_empty_path() { stats.failed_generations++; stats.failed_generations_in_row++; - return possible_path(); + return generated_path(); } // ------------------------------------------------------------------------------------------------ -possible_path iid_node_dependence_props::return_path( const possible_path& path ) +generated_path iid_node_dependence_props::return_path( const generated_path& path ) { stats.failed_generations_in_row = 0; stats.successful_generations++; @@ -1104,7 +1092,7 @@ possible_path iid_node_dependence_props::return_path( const possible_path& path } // ------------------------------------------------------------------------------------------------ -void iid_node_dependence_props::compute_path_counts_for_nested_loops( nodes_to_counts& path_counts, +void iid_node_dependence_props::compute_node_counts_for_nested_loops( nodes_to_counts& path_counts, std::map< location_id::id_type, int >& child_loop_counts, location_id::id_type loop_head_id, int minimum_count, @@ -1142,7 +1130,7 @@ void iid_node_dependence_props::compute_path_counts_for_nested_loops( nodes_to_c *possible_counts.rbegin(); for ( auto& [ node_id, count ] : child_loop_counts ) { - dependent_loop_properties& props = dependencies_by_loops.get_props_by_loop_head_id( node_id ); + loop_properties& props = loop_to_properties.get_props_by_loop_head_id( node_id ); for ( auto& [ head, _ ] : props.heads ) { auto& [ left_count, right_count ] = path_counts[ head.node_id ]; @@ -1156,7 +1144,7 @@ void iid_node_dependence_props::compute_path_counts_for_nested_loops( nodes_to_c } - dependent_loop_properties& props = dependencies_by_loops.get_props_by_loop_head_id( loop_head_id ); + loop_properties& props = loop_to_properties.get_props_by_loop_head_id( loop_head_id ); for ( auto& [ head, _ ] : props.heads ) { if ( head.branching_direction ) { path_counts[ head.node_id ] = { highest_count, 1 }; @@ -1167,23 +1155,23 @@ void iid_node_dependence_props::compute_path_counts_for_nested_loops( nodes_to_c } // ------------------------------------------------------------------------------------------------ -int fuzzing::iid_node_dependence_props::compute_loop_count_loading_new( nodes_to_counts& path_counts, - location_id::id_type id, - const std::set< location_id::id_type >& loop_heads, - const dependent_loop_properties& props, - float loaded_bits_per_loop ) +int fuzzing::iid_node_dependence_props::compute_loading_loop_interation( nodes_to_counts& path_counts, + location_id::id_type id, + const std::set< location_id::id_type >& loop_heads, + const loop_properties& props, + float loaded_bits_per_loop ) { - float loaded_per_loop = props.average_bits_per_loop.mean - loaded_bits_per_loop; + float loaded_per_loop = props.loaded_bits_per_loop.mean - loaded_bits_per_loop; if ( loaded_per_loop <= 0 ) { loaded_per_loop = 8; } - float average_bits = props.bit_values_per_node.at( id ).average_bit_size.mean; - natural_32_bit offset = props.bit_values_per_node.at( id ).minimal_bit_offset; + float average_bits = props.bits_read_by_node.at( id ).average_bits_read.mean; + natural_32_bit offset = props.bits_read_by_node.at( id ).minimal_bit_offset; bool is_loop_head = true; if ( !loop_heads.contains( id ) ) { - for ( const auto& loop : dependencies_by_loops.loops ) { + for ( const auto& loop : loop_to_properties.loops ) { for ( const auto& body : loop.bodies ) { if ( body.node_id == id ) { id = ( *loop.chosen_loop_head ).node_id; @@ -1200,11 +1188,11 @@ int fuzzing::iid_node_dependence_props::compute_loop_count_loading_new( nodes_to } // ------------------------------------------------------------------------------------------------ -void iid_node_dependence_props::compute_path_counts_loading_new( nodes_to_counts& path_counts, - const equation& path, - const std::set< location_id::id_type >& loop_heads ) +void iid_node_dependence_props::compute_node_counts_for_loading_loops( nodes_to_counts& path_counts, + const equation& path, + const std::set< location_id::id_type >& loop_heads ) { - for ( const auto& loop_props : std::ranges::views::reverse( dependencies_by_loops.loops ) ) { + for ( const auto& loop_props : std::ranges::views::reverse( loop_to_properties.loops ) ) { if ( !loop_props.is_loading_loop ) { continue; } @@ -1219,19 +1207,20 @@ void iid_node_dependence_props::compute_path_counts_loading_new( nodes_to_counts if ( !loop_heads.contains( body.node_id ) ) { continue; } - - const dependent_loop_properties& body_props = dependencies_by_loops.get_props_by_loop_head_id( body.node_id ); + + const loop_properties& body_props = loop_to_properties.get_props_by_loop_head_id( body.node_id ); if ( body_props.is_loading_loop ) { - loaded_bits_per_loop += body_props.average_bits_per_loop.mean; + loaded_bits_per_loop += body_props.loaded_bits_per_loop.mean; } } - for ( const auto& body : loop_props.bit_dependent_nodes ) { + for ( const auto& body : loop_props.nodes_dependent_by_loading ) { if ( !path_counts.contains( body.node_id ) ) { continue; } - - int minimal_count = compute_loop_count_loading_new( path_counts, body.node_id, loop_heads, loop_props, loaded_bits_per_loop ); + + int minimal_count = compute_loading_loop_interation( + path_counts, body.node_id, loop_heads, loop_props, loaded_bits_per_loop ); loop_count = std::max( loop_count, minimal_count ); } @@ -1256,21 +1245,21 @@ void iid_node_dependence_props::compute_path_counts_loading_new( nodes_to_counts } } } else { - compute_path_counts_for_nested_loops( path_counts, + compute_node_counts_for_nested_loops( path_counts, child_loop_counts, ( *loop_props.chosen_loop_head ).node_id, loop_count, - iid_dependencies::random_nested_loops ); + iid_dependencies::random_nested_loop_counts ); } } } // ------------------------------------------------------------------------------------------------ -void iid_node_dependence_props::compute_path_counts_loops( nodes_to_counts& path_counts, - const equation& path, - const std::set< location_id::id_type >& loop_heads ) +void iid_node_dependence_props::compute_node_counts_for_loops( nodes_to_counts& path_counts, + const equation& path, + const std::set< location_id::id_type >& loop_heads ) { - for ( const auto& props : std::ranges::views::reverse( dependencies_by_loops.loops ) ) { + for ( const auto& props : std::ranges::views::reverse( loop_to_properties.loops ) ) { if ( props.bodies.empty() || props.is_loading_loop ) { continue; } @@ -1292,27 +1281,27 @@ void iid_node_dependence_props::compute_path_counts_loops( nodes_to_counts& path continue; } - compute_path_counts_for_nested_loops( path_counts, + compute_node_counts_for_nested_loops( path_counts, child_loop_counts, ( *props.chosen_loop_head ).node_id, non_loop_child_max_count, - iid_dependencies::random_nested_loops ); + iid_dependencies::random_nested_loop_counts ); } } // ------------------------------------------------------------------------------------------------ -nodes_to_counts iid_node_dependence_props::compute_path_counts( const equation& path, - std::set< node_direction > const& computation_subset ) +nodes_to_counts iid_node_dependence_props::compute_node_counts( const equation& path, + std::set< node_id_with_direction > const& computation_subset ) { nodes_to_counts path_counts; - std::vector< node_direction > leafs = std::vector< node_direction >( computation_subset.begin(), - computation_subset.end() ); + std::vector< node_id_with_direction > leafs = + std::vector< node_id_with_direction >( computation_subset.begin(), computation_subset.end() ); INVARIANT( leafs.size() == path.values.size() ); std::set< location_id::id_type > loop_heads = get_loop_heads( false ); - for ( auto& loop : dependencies_by_loops.loops ) { + for ( auto& loop : loop_to_properties.loops ) { loop.set_chosen_loop_head(); } @@ -1325,7 +1314,7 @@ nodes_to_counts iid_node_dependence_props::compute_path_counts( const equation& } } - for ( const auto& loop_props : std::ranges::views::reverse( dependencies_by_loops.loops ) ) { + for ( const auto& loop_props : std::ranges::views::reverse( loop_to_properties.loops ) ) { if ( loop_props.bodies.empty() ) { continue; } @@ -1347,8 +1336,8 @@ nodes_to_counts iid_node_dependence_props::compute_path_counts( const equation& } } - compute_path_counts_loops( path_counts, path, loop_heads ); - compute_path_counts_loading_new( path_counts, path, loop_heads ); + compute_node_counts_for_loops( path_counts, path, loop_heads ); + compute_node_counts_for_loading_loops( path_counts, path, loop_heads ); return path_counts; } @@ -1474,12 +1463,12 @@ std::vector< equation > iid_node_dependence_props::get_random_vector( const std: } // ------------------------------------------------------------------------------------------------ -std::set< node_direction > iid_node_dependence_props::get_node_subsets_for_computation() +std::set< node_id_with_direction > iid_node_dependence_props::get_node_subsets_for_computation() { - std::set< node_direction > computation_subset; + std::set< node_id_with_direction > computation_subset; auto loop_heads = get_loop_heads( false ); - for ( const auto& loop : dependencies_by_loops.loops ) { + for ( const auto& loop : loop_to_properties.loops ) { if ( loop.bodies.empty() || loop.is_loading_loop ) { continue; } @@ -1551,7 +1540,7 @@ iid_node_dependence_props::get_loop_heads_ending( branching_node* end_node, // ------------------------------------------------------------------------------------------------ void fuzzing::iid_node_dependence_props::compute_loading_loops( branching_node* end_node, const loop_head_to_bodies_t& loop_heads_to_bodies, - loop_head_to_loaded_bits_props& loading_loops ) + loop_head_to_loaded_bits_counter& loading_loops ) { for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { loading_loops[ loop_head.id ] = { std::numeric_limits< natural_32_bit >::max(), @@ -1591,11 +1580,11 @@ void fuzzing::iid_node_dependence_props::compute_loading_loops( branching_node* } // ------------------------------------------------------------------------------------------------ -void iid_node_dependence_props::compute_dependencies_by_loading_new( branching_node* end_node, - const loop_head_to_bodies_t& loop_heads_to_bodies, - const loop_endings& loop_heads_ending ) +void iid_node_dependence_props::compute_dependencies_by_loading( branching_node* end_node, + const loop_head_to_bodies_t& loop_heads_to_bodies, + const loop_endings& loop_heads_ending ) { - loop_head_to_loaded_bits_props loading_loops; + loop_head_to_loaded_bits_counter loading_loops; compute_loading_loops( end_node, loop_heads_to_bodies, loading_loops ); branching_node* node = end_node; @@ -1647,30 +1636,30 @@ void iid_node_dependence_props::compute_dependencies_by_loading_new( branching_n for ( const auto& [ loop_head_id, body ] : loop_to_props ) { auto loading_props = loading_loops.at( loop_head_id ); - dependent_loop_properties& dependencies = dependencies_by_loops.get_props_by_loop_head_id( loop_head_id ); + loop_properties& dependencies = loop_to_properties.get_props_by_loop_head_id( loop_head_id ); dependencies.is_loading_loop = true; natural_32_bit loaded_bits = loading_props.max - loading_props.min; double per_loop = double( loaded_bits ) / double( loading_props.loop_count ); - dependencies.average_bits_per_loop.add( per_loop ); + dependencies.loaded_bits_per_loop.add( per_loop ); bool loop_head_end_direction = loop_heads_ending.at( loop_head_id ); for ( const auto& [ body_id, props ] : body ) { - auto& body_props = dependencies.bit_values_per_node[ body_id ]; + auto& body_props = dependencies.bits_read_by_node[ body_id ]; natural_32_bit minimal_offset = props.min - loading_props.min; INVARIANT( minimal_offset >= 0 ); body_props.minimal_bit_offset = std::min( body_props.minimal_bit_offset, minimal_offset ); for ( const auto& count : props.sensitive_stdin_bit_counts ) { - body_props.average_bit_size.add( count ); + body_props.average_bits_read.add( count ); } for ( bool direction : { true, false } ) { - node_direction node_id_direction = { body_id, direction }; + node_id_with_direction node_id_direction = { body_id, direction }; if ( matrix.contains( node_id_direction ) ) { - dependencies.bit_dependent_nodes.insert( node_id_direction ); + dependencies.nodes_dependent_by_loading.insert( node_id_direction ); } } } @@ -1689,7 +1678,7 @@ void iid_node_dependence_props::compute_dependencies_by_loops( const loop_head_t } bool loop_head_end_direction = loop_heads_ending.at( loop_head_id ); - node_direction loop_head_direction = { loop_head_id, loop_head_end_direction }; + node_id_with_direction loop_head_direction = { loop_head_id, loop_head_end_direction }; std::set< location_id::id_type > all_ids; all_ids.insert( loop_head_id ); @@ -1697,13 +1686,13 @@ void iid_node_dependence_props::compute_dependencies_by_loops( const loop_head_t all_ids.insert( loop_body_id.id ); } - dependent_loop_properties& props = dependencies_by_loops.get_props( all_ids, loop_head_id ); + loop_properties& props = loop_to_properties.get_props( all_ids, loop_head_id ); props.heads[ loop_head_direction ].count++; for ( const auto& body : loop_bodies ) { for ( bool direction : { true, false } ) { - node_direction node_id_direction = { body.id, direction }; + node_id_with_direction node_id_direction = { body.id, direction }; for ( const auto& [ head, _ ] : props.heads ) { if ( head.node_id == body.id ) { @@ -1718,19 +1707,17 @@ void iid_node_dependence_props::compute_dependencies_by_loops( const loop_head_t } } - dependencies_by_loops.merge_properties(); + loop_to_properties.merge_properties(); - std::sort( dependencies_by_loops.loops.begin(), - dependencies_by_loops.loops.end(), - []( const auto& a, const auto& b ) { - return a.get_smallest_loop_head_id() < b.get_smallest_loop_head_id(); - } ); + std::sort( loop_to_properties.loops.begin(), loop_to_properties.loops.end(), []( const auto& a, const auto& b ) { + return a.get_smallest_loop_head_id() < b.get_smallest_loop_head_id(); + } ); } // ------------------------------------------------------------------------------------------------ -possible_path iid_node_dependence_props::generate_path_from_node_counts( const nodes_to_counts& path_counts ) +generated_path iid_node_dependence_props::generate_path_from_node_counts( const nodes_to_counts& path_counts ) { - std::map< location_id::id_type, path_node_props > path; + std::map< location_id::id_type, node_props_in_path > path; for ( const auto& [ id, counts ] : path_counts ) { if ( counts.left_count == 0 && counts.right_count == 0 ) { continue; @@ -1739,7 +1726,7 @@ possible_path iid_node_dependence_props::generate_path_from_node_counts( const n bool loop_head_end_direction = false; bool is_loop_head = false; - for ( const auto& loop : dependencies_by_loops.loops ) { + for ( const auto& loop : loop_to_properties.loops ) { for ( const auto& [ head, _ ] : loop.heads ) { if ( head.node_id == id ) { is_loop_head = true; @@ -1748,18 +1735,18 @@ possible_path iid_node_dependence_props::generate_path_from_node_counts( const n } } - path_node_props props = { counts, is_loop_head, loop_head_end_direction }; + node_props_in_path props = { counts, is_loop_head, loop_head_end_direction }; path.emplace( id, props ); } - return possible_path( path ); + return generated_path( path ); } // ------------------------------------------------------------------------------------------------ std::set< location_id::id_type > iid_node_dependence_props::get_loop_heads( bool include_loading_loops ) { std::set< location_id::id_type > loop_heads; - for ( const auto& props : dependencies_by_loops.loops ) { + for ( const auto& props : loop_to_properties.loops ) { if ( props.is_loading_loop && !include_loading_loops ) { continue; } @@ -1774,13 +1761,13 @@ std::set< location_id::id_type > iid_node_dependence_props::get_loop_heads( bool // iid_dependencies // ------------------------------------------------------------------------------------------------ -void iid_dependencies::update_non_iid_nodes( sensitivity_analysis& sensitivity ) +void iid_dependencies::update_ignored_nodes( sensitivity_analysis& sensitivity ) { for ( branching_node* node : sensitivity.get_changed_nodes() ) { if ( node->is_did_branching() ) { location_id::id_type location_id = node->get_location_id().id; - if ( ignored_nodes.insert( location_id ).second ) { - id_to_equation_map.erase( location_id ); + if ( ignored_node_ids.insert( location_id ).second ) { + node_id_to_equation_map.erase( location_id ); } } } @@ -1791,23 +1778,23 @@ void iid_dependencies::process_node_dependence( branching_node* node ) { TMPROF_BLOCK(); - if ( ignored_nodes.contains( node->get_location_id().id ) ) + if ( ignored_node_ids.contains( node->get_location_id().id ) ) return; - iid_node_dependence_props& props = id_to_equation_map[ node->get_location_id().id ]; + iid_node_dependence_props& props = node_id_to_equation_map[ node->get_location_id().id ]; props.process_node( node ); } // ------------------------------------------------------------------------------------------------ void iid_dependencies::remove_node_dependence( location_id::id_type id ) { - auto it = id_to_equation_map.find( id ); - if ( it != id_to_equation_map.end() ) { + auto it = node_id_to_equation_map.find( id ); + if ( it != node_id_to_equation_map.end() ) { iid_node_generations_stats& stats = it->second.get_generations_stats(); stats.state = generation_state::STATE_COVERED; - ignored_nodes.insert( id ); + ignored_node_ids.insert( id ); - if ( it != id_to_equation_map.end() ) { + if ( it != node_id_to_equation_map.end() ) { auto next_it = std::next( it ); next_it->second.combine_props( it->second ); } @@ -1824,14 +1811,14 @@ void iid_dependencies::remove_node_dependence( location_id::id_type id ) // ------------------------------------------------------------------------------------------------ iid_node_dependence_props& iid_dependencies::get_props( location_id::id_type id ) { - return id_to_equation_map.at( id ); + return node_id_to_equation_map.at( id ); } // ------------------------------------------------------------------------------------------------ std::vector< location_id::id_type > iid_dependencies::get_iid_nodes() { std::vector< location_id::id_type > result; - for ( const auto& [ key, _ ] : id_to_equation_map ) { + for ( const auto& [ key, _ ] : node_id_to_equation_map ) { result.push_back( key ); } @@ -1843,7 +1830,7 @@ std::vector< location_id::id_type > iid_dependencies::get_iid_nodes() std::optional< location_id::id_type > iid_dependencies::get_next_iid_node() { bool previous_needs_more_data = false; - for ( auto it = id_to_equation_map.rbegin(); it != id_to_equation_map.rend(); ++it ) { + for ( auto it = node_id_to_equation_map.rbegin(); it != node_id_to_equation_map.rend(); ++it ) { iid_node_dependence_props& props = it->second; if ( previous_needs_more_data ) { @@ -1853,7 +1840,7 @@ std::optional< location_id::id_type > iid_dependencies::get_next_iid_node() if ( props.too_much_failed_in_row( iid_dependencies::max_failed_generations_in_row ) ) { props.get_generations_stats().failed_generations_in_row = 0; - bool is_first = std::next( it ) == id_to_equation_map.rend(); + bool is_first = std::next( it ) == node_id_to_equation_map.rend(); failed_generation_method method = props.get_method_for_failed_generation( is_first ); @@ -1865,7 +1852,7 @@ std::optional< location_id::id_type > iid_dependencies::get_next_iid_node() std::optional< location_id::id_type > best_id = std::nullopt; - for ( const auto& [ id, props ] : id_to_equation_map ) { + for ( const auto& [ id, props ] : node_id_to_equation_map ) { if ( props.should_generate() ) { return id; } @@ -1881,30 +1868,31 @@ iid_vector_analysis_statistics fuzzing::iid_dependencies::get_stats() const iid_vector_analysis_statistics stats; iid_vector_analysis_statistics_per_node node_stats; - for ( const auto& [ id, props ] : id_to_equation_map ) { + for ( const auto& [ id, props ] : node_id_to_equation_map ) { node_stats.generation_stats = props.get_generations_stats(); - node_stats.dependencies_by_loops = props.get_dependencies_by_loops(); + node_stats.loop_to_properties = props.get_dependencies_by_loops(); stats.iid_nodes_stats[ id ] = node_stats; } - stats.ignored_nodes = std::vector< location_id::id_type >( ignored_nodes.begin(), ignored_nodes.end() ); + stats.ignored_node_ids = std::vector< location_id::id_type >( ignored_node_ids.begin(), + ignored_node_ids.end() ); return stats; } // non member functions // ------------------------------------------------------------------------------------------------ -std::vector< node_direction > get_path( branching_node* node ) +std::vector< node_id_with_direction > get_path( branching_node* node ) { - std::vector< node_direction > result; + std::vector< node_id_with_direction > result; branching_node* current = node; while ( current != nullptr ) { branching_node* predecessor = current->predecessor; if ( predecessor != nullptr ) { - node_direction nav = { predecessor->get_location_id().id, - predecessor->successor_direction( current ) }; + node_id_with_direction nav = { predecessor->get_location_id().id, + predecessor->successor_direction( current ) }; result.push_back( nav ); } current = predecessor; From b98c6278f4ac4fa648357d019bd5a4aac14224cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Wed, 12 Feb 2025 21:14:56 +0100 Subject: [PATCH 114/144] feat: update num_executions in benchmark input files --- benchmarks/iid_testing/input_cycle_2_loops.json | 2 +- benchmarks/iid_testing/input_cycle_dec_b.json | 2 +- .../iid_testing/more_iid_condition_nested_big_values.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/iid_testing/input_cycle_2_loops.json b/benchmarks/iid_testing/input_cycle_2_loops.json index 8d080249..0fc5542c 100644 --- a/benchmarks/iid_testing/input_cycle_2_loops.json +++ b/benchmarks/iid_testing/input_cycle_2_loops.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 1912, + "num_executions": 2633, "num_covered_branchings": 11, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/input_cycle_dec_b.json b/benchmarks/iid_testing/input_cycle_dec_b.json index be4d51d9..da0a3aa3 100644 --- a/benchmarks/iid_testing/input_cycle_dec_b.json +++ b/benchmarks/iid_testing/input_cycle_dec_b.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 1100, + "num_executions": 1388, "num_covered_branchings": 10, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/more_iid_condition_nested_big_values.json b/benchmarks/iid_testing/more_iid_condition_nested_big_values.json index 66cb4f21..03c36964 100644 --- a/benchmarks/iid_testing/more_iid_condition_nested_big_values.json +++ b/benchmarks/iid_testing/more_iid_condition_nested_big_values.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 42668, + "num_executions": 47736, "num_covered_branchings": 12, "covered_branchings": [ 1, From 0389c4640fa875b8a8aef436315cd420a5ce8c96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Wed, 12 Feb 2025 21:15:32 +0100 Subject: [PATCH 115/144] feat: add saving data to json file --- .../include/fuzzing/analysis_outcomes.hpp | 1 + src/fuzzing/include/fuzzing/fuzzer.hpp | 7 +- src/fuzzing/src/dump.cpp | 128 ++++++++++++++++++ src/fuzzing/src/fuzzer.cpp | 22 +-- src/fuzzing/src/fuzzing_loop.cpp | 1 + 5 files changed, 145 insertions(+), 14 deletions(-) diff --git a/src/fuzzing/include/fuzzing/analysis_outcomes.hpp b/src/fuzzing/include/fuzzing/analysis_outcomes.hpp index cb946b08..f2c3f6f5 100644 --- a/src/fuzzing/include/fuzzing/analysis_outcomes.hpp +++ b/src/fuzzing/include/fuzzing/analysis_outcomes.hpp @@ -44,6 +44,7 @@ struct analysis_outcomes bitshare_analysis::performance_statistics bitshare_statistics{}; fuzzer::performance_statistics fuzzer_statistics{}; std::unordered_map output_statistics{}; + iid_vector_analysis_statistics iid_vector_analysis_statistics{}; }; diff --git a/src/fuzzing/include/fuzzing/fuzzer.hpp b/src/fuzzing/include/fuzzing/fuzzer.hpp index c122ec61..2fa79dd5 100644 --- a/src/fuzzing/include/fuzzing/fuzzer.hpp +++ b/src/fuzzing/include/fuzzing/fuzzer.hpp @@ -84,6 +84,7 @@ struct fuzzer final minimization_analysis::performance_statistics const& get_minimization_statistics() const { return minimization.get_statistics(); } bitshare_analysis::performance_statistics const& get_bitshare_statistics() const { return bitshare.get_statistics(); } performance_statistics const& get_fuzzer_statistics() const { return statistics; } + iid_vector_analysis_statistics get_iid_vector_analysis_statistics() const { return iid_dependences.get_stats(); } private: @@ -257,7 +258,7 @@ struct fuzzer final ); static branching_node* select_start_node_for_monte_carlo_search_with_vector( - possible_path const& path, + generated_path const& path, std::vector const& loop_boundaries, branching_node* fallback_node ); @@ -282,7 +283,7 @@ struct fuzzer final histogram_of_false_direction_probabilities const& histogram, probability_generators_for_locations const& generators, probability_generator_random_uniform& location_miss_generator, - possible_path& path + generated_path& path ); static std::pair monte_carlo_backward_search( branching_node* const start_node, @@ -302,7 +303,7 @@ struct fuzzer final histogram_of_false_direction_probabilities const& histogram, probability_generators_for_locations const& generators, probability_generator_random_uniform& location_miss_generator, - possible_path& path + generated_path& path ); bool generate_next_input(vecb& stdin_bits, TERMINATION_REASON& termination_reason); diff --git a/src/fuzzing/src/dump.cpp b/src/fuzzing/src/dump.cpp index ad1f6ecb..ff6d5720 100644 --- a/src/fuzzing/src/dump.cpp +++ b/src/fuzzing/src/dump.cpp @@ -244,6 +244,128 @@ void log_analysis_outcomes(analysis_outcomes const& results) } +void save_iid_vector_analysis( std::ostream& ostr, analysis_outcomes const& result ) +{ + bool print_dependencies = true; + + std::string const shift = " "; + auto indent = [ & ]( int level ) -> std::string { + std::string result; + for ( int i = 0; i < level; ++i ) { + result += shift; + } + return result; + }; + + + ostr << "{\n"; + ostr << indent( 1 ) << "\"Analyzed node\": {\n"; + + for ( auto it = result.iid_vector_analysis_statistics.iid_nodes_stats.begin(); + it != result.iid_vector_analysis_statistics.iid_nodes_stats.end(); + ++it ) { + ostr << indent( 2 ) << std::dec << "\"" << it->first << "\"" << ": {\n"; + + ostr << indent( 3 ) << "\"Generation state\": \""; + switch ( it->second.generation_stats.state ) { + case generation_state::STATE_NOT_COVERED: ostr << "STATE_NOT_COVERED"; break; + case generation_state::STATE_GENERATING_ARTIFICIAL_DATA: + ostr << "STATE_GENERATING_ARTIFICIAL_DATA"; + break; + case generation_state::STATE_GENERATION_MORE: ostr << "STATE_GENERATION_MORE"; break; + case generation_state::STATE_COVERED: ostr << "STATE_COVERED"; break; + case generation_state::STATE_GENERATION_DATA_FOR_NEXT_NODE: + ostr << "STATE_GENERATION_DATA_FOR_NEXT_NODE"; + break; + default: ostr << "UNKNOWN"; break; + } + ostr << "\",\n"; + + ostr << indent( 3 ) << "\"Method calls\": " << it->second.generation_stats.method_calls << ",\n"; + ostr << indent( 3 ) << "\"Generation starts\": " << it->second.generation_stats.generation_starts << ",\n"; + ostr << indent( 3 ) + << "\"Successful generations\": " << it->second.generation_stats.successful_generations << ",\n"; + ostr << indent( 3 ) << "\"Failed generations\": " << it->second.generation_stats.failed_generations + << ",\n"; + ostr << indent( 3 ) + << "\"Artificial generations\": " << it->second.generation_stats.generate_artificial_data_count + << ",\n"; + ostr << indent( 3 ) + << "\"For other node generations\": " << it->second.generation_stats.generated_for_other_node_count + << ( print_dependencies ? "," : "" ) << "\n"; + + if ( print_dependencies ) { + ostr << indent( 3 ) << "\"Dependencies\": [\n"; + for ( auto loop_it = it->second.loop_to_properties.loops.begin(); + loop_it != it->second.loop_to_properties.loops.end(); + ++loop_it ) { + + ostr << indent( 4 ) << "{\n"; + ostr << indent( 5 ) << "\"Loop heads\": [\n"; + for ( auto head_it = loop_it->heads.begin(); head_it != loop_it->heads.end(); ++head_it ) { + ostr << indent( 6 ) << std::dec << "\"" << head_it->first << "\"" + << ( std::next( head_it ) != loop_it->heads.end() ? "," : "" ) << '\n'; + } + ostr << indent( 5 ) << "],\n"; + + ostr << indent( 5 ) << "\"Loop bodies\": [\n"; + for ( auto body_it = loop_it->bodies.begin(); body_it != loop_it->bodies.end(); ++body_it ) { + ostr << indent( 6 ) << std::dec << "\"" << *body_it << "\"" + << ( std::next( body_it ) != loop_it->bodies.end() ? "," : "" ) << '\n'; + } + ostr << indent( 5 ) << "],\n"; + + ostr << indent( 5 ) + << "\"Is loading loop\": " << ( loop_it->is_loading_loop ? "true," : "false" ) << "\n"; + + if ( !loop_it->is_loading_loop ) { + ostr << indent( 4 ) << "}" + << ( std::next( loop_it ) != it->second.loop_to_properties.loops.end() ? "," : "" ) + << '\n'; + continue; + } + + ostr << indent( 5 ) << "\"Loaded bits per loop\": " << loop_it->loaded_bits_per_loop.mean << ",\n"; + ostr << indent( 5 ) << "\"Dependent nodes\": [\n"; + for ( auto body_it = loop_it->bits_read_by_node.begin(); + body_it != loop_it->bits_read_by_node.end(); + ++body_it ) { + ostr << indent( 6 ) << "{\n"; + ostr << indent( 7 ) << "\"Node\": " << std::dec << body_it->first << ",\n"; + ostr << indent( 7 ) << "\"Bits read\": " << body_it->second.average_bits_read.mean << ",\n"; + ostr << indent( 7 ) << "\"Minimal bit offset\": " << body_it->second.minimal_bit_offset + << '\n'; + ostr << indent( 6 ) << "}" + << ( std::next( body_it ) != loop_it->bits_read_by_node.end() ? "," : "" ) << '\n'; + } + ostr << indent( 5 ) << "]\n"; + + ostr << indent( 4 ) << "}" + << ( std::next( loop_it ) != it->second.loop_to_properties.loops.end() ? "," : "" ) << '\n'; + } + + ostr << indent( 3 ) << "]\n"; + } + + ostr << indent( 2 ) << "}" + << ( std::next( it ) != result.iid_vector_analysis_statistics.iid_nodes_stats.end() ? "," : "" ) + << '\n'; + } + + ostr << indent( 1 ) << "},\n"; + + ostr << indent( 1 ) << "\"Ignored nodes\": [\n"; + for ( std::size_t i = 0, n = result.iid_vector_analysis_statistics.ignored_node_ids.size(); i < n; ++i ) { + ostr << indent( 2 ) << std::dec << result.iid_vector_analysis_statistics.ignored_node_ids.at( i ); + if ( i + 1 < n ) + ostr << ','; + ostr << '\n'; + } + ostr << indent( 1 ) << "]\n"; + + ostr << "}"; +} + void save_analysis_outcomes( std::filesystem::path const& output_dir, std::string const& benchmark, @@ -253,6 +375,12 @@ void save_analysis_outcomes( std::filesystem::path const test_file_path = output_dir / (benchmark + "_outcomes.json"); std::ofstream ostr(test_file_path.c_str(), std::ios::binary); print_analysis_outcomes(ostr, results); + + if ( true ) { + std::filesystem::path const iid_vector_analysis_path = output_dir / (benchmark + "_iid_vector_analysis.json"); + std::ofstream iid_ostr( iid_vector_analysis_path.c_str(), std::ios::binary ); + save_iid_vector_analysis( iid_ostr, results ); + } } diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index fecc2c93..696c1e78 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -635,7 +635,7 @@ void fuzzer::compute_histogram_of_false_direction_probabilities( } branching_node* fuzzer::select_start_node_for_monte_carlo_search_with_vector( - const possible_path& path, + const generated_path& path, std::vector< branching_node* > const& loop_boundaries, branching_node* fallback_node ) { @@ -733,7 +733,7 @@ branching_node* fuzzer::monte_carlo_search( histogram_of_false_direction_probabilities const& histogram, probability_generators_for_locations const& generators, probability_generator_random_uniform& location_miss_generator, - possible_path& path + generated_path& path ) { TMPROF_BLOCK(); @@ -796,7 +796,7 @@ fuzzing::fuzzer::monte_carlo_step_with_path( branching_node* const pivot, histogram_of_false_direction_probabilities const& histogram, probability_generators_for_locations const& generators, probability_generator_random_uniform& location_miss_generator, - possible_path& path ) + generated_path& path ) { INVARIANT( pivot != nullptr && !pivot->is_closed() ); @@ -825,9 +825,10 @@ fuzzing::fuzzer::monte_carlo_step_with_path( branching_node* const pivot, location_id::id_type pivot_id = pivot->get_location_id().id; if ( path.contains( pivot_id ) ) { - path_node_props& props = path.get_props( pivot_id ); + node_props_in_path& props = path.get_props( pivot_id ); if ( props.can_take_next_direction() ) { desired_direction = props.get_desired_direction(); + // std::cout << "Desired direction: " << desired_direction << ", for node id:" << pivot_id << std::endl; } } @@ -835,7 +836,7 @@ fuzzing::fuzzer::monte_carlo_step_with_path( branching_node* const pivot, ( desired_direction == true && can_go_right ); if ( path.contains( pivot_id ) && can_go_desired_direction ) { - path_node_props& props = path.get_props( pivot_id ); + node_props_in_path& props = path.get_props( pivot_id ); if ( props.can_take_next_direction() ) { props.go_direction( desired_direction ); } @@ -846,7 +847,6 @@ fuzzing::fuzzer::monte_carlo_step_with_path( branching_node* const pivot, else if ( !pivot->is_open_branching() ) successor = can_go_left ? left : right; - // std::cout << path << std::endl; return successor; } @@ -1394,7 +1394,7 @@ void fuzzer::do_cleanup() update_close_flags_from(node); break; } - iid_dependences.update_non_iid_nodes(sensitivity); + iid_dependences.update_ignored_nodes(sensitivity); collect_iid_pivots_from_sensitivity_results(); break; case BITSHARE: @@ -1429,7 +1429,7 @@ void fuzzer::do_cleanup() for (auto it = iid_pivots.begin(); it != iid_pivots.end(); ) if (covered_branchings.contains(it->first)) { - iid_dependences.remove_node_dependence(it->first); + iid_dependences.remove_node_dependence(it->first.id); it = iid_pivots.erase(it); } else @@ -1687,12 +1687,12 @@ branching_node* fuzzer::select_iid_coverage_target() histogram_of_hit_counts_per_direction::hit_counts_map hit_counts; it_pivot->second.histogram_ptr->merge(hit_counts); - possible_path path; - std::optional< location_id > iid_location = iid_dependences.get_next_iid_node(); + generated_path path; + std::optional< location_id::id_type > iid_location = iid_dependences.get_next_iid_node(); if ( use_vector_analysis && iid_location.has_value() ) { iid_node_dependence_props& node_props = iid_dependences.get_props( *iid_location ); - // std::cout << "Computing probabilities for location " << ( *iid_location ).id << std::endl; + // std::cout << "Computing probabilities for location " << ( *iid_location ) << std::endl; path = node_props.generate_probabilities(); for ( const auto& path_props : path.get_path() ) { diff --git a/src/fuzzing/src/fuzzing_loop.cpp b/src/fuzzing/src/fuzzing_loop.cpp index 2459c902..22028894 100644 --- a/src/fuzzing/src/fuzzing_loop.cpp +++ b/src/fuzzing/src/fuzzing_loop.cpp @@ -126,6 +126,7 @@ analysis_outcomes run( results.minimization_statistics = f.get_minimization_statistics(); results.bitshare_statistics = f.get_bitshare_statistics(); results.fuzzer_statistics = f.get_fuzzer_statistics(); + results.iid_vector_analysis_statistics = f.get_iid_vector_analysis_statistics(); return results; } From 62f2e34c5a119c232eb1357085bbd8f0dac0d2bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Wed, 12 Feb 2025 21:15:47 +0100 Subject: [PATCH 116/144] feat: update compute_loading_loops to include loop_endings parameter --- .../include/fuzzing/iid_vector_analysis.hpp | 3 +- src/fuzzing/src/iid_vector_analysis.cpp | 54 ++++++++++++++----- 2 files changed, 42 insertions(+), 15 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index 3419359d..6c1f6d9b 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -316,7 +316,8 @@ struct iid_node_dependence_props { loop_endings get_loop_heads_ending( branching_node* end_node, loop_head_to_bodies_t& loop_heads_to_bodies ); void compute_loading_loops( branching_node* end_node, const loop_head_to_bodies_t& loop_heads_to_bodies, - loop_head_to_loaded_bits_counter& loading_loops ); + loop_head_to_loaded_bits_counter& loading_loops, + const loop_endings& loop_heads_ending ); void compute_dependencies_by_loading( branching_node* end_node, const loop_head_to_bodies_t& loop_heads_to_bodies, const loop_endings& loop_heads_ending ); diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 12307565..c174f6d0 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -764,8 +764,8 @@ generated_path iid_node_dependence_props::generate_probabilities() nodes_to_counts node_counts = compute_node_counts( new_subset_counts.value(), computation_subset ); generated_path path = generate_path_from_node_counts( node_counts ); - - std::cout << "Generated path: " << std::endl << path << std::endl; + + // std::cout << "Generated path: " << std::endl << path << std::endl; return return_path( path ); } @@ -1197,9 +1197,8 @@ void iid_node_dependence_props::compute_node_counts_for_loading_loops( nodes_to_ continue; } - int loop_count = 0; + int loop_count = 1; std::map< location_id::id_type, int > child_loop_counts; - std::map< location_id::id_type, float > loading_loops_count_bits; float loaded_bits_per_loop = 0.0f; @@ -1208,9 +1207,10 @@ void iid_node_dependence_props::compute_node_counts_for_loading_loops( nodes_to_ continue; } + const loop_properties& body_props = loop_to_properties.get_props_by_loop_head_id( body.node_id ); if ( body_props.is_loading_loop ) { - loaded_bits_per_loop += body_props.loaded_bits_per_loop.mean; + // loaded_bits_per_loop += body_props.loaded_bits_per_loop.mean; } } @@ -1224,6 +1224,15 @@ void iid_node_dependence_props::compute_node_counts_for_loading_loops( nodes_to_ loop_count = std::max( loop_count, minimal_count ); } + for ( const auto& body : loop_props.bodies ) { + if ( loop_heads.contains( body.node_id ) ) { + const auto& inner_loop_props = loop_to_properties.get_props_by_loop_head_id( body.node_id ); + if ( inner_loop_props.is_loading_loop ) { + loop_count = 1; + } + } + } + for ( const auto& body : loop_props.bodies ) { auto& [ left_count, right_count ] = path_counts[ body.node_id ]; @@ -1337,6 +1346,7 @@ nodes_to_counts iid_node_dependence_props::compute_node_counts( const equation& } compute_node_counts_for_loops( path_counts, path, loop_heads ); + loop_heads = get_loop_heads( true ); compute_node_counts_for_loading_loops( path_counts, path, loop_heads ); return path_counts; @@ -1540,7 +1550,8 @@ iid_node_dependence_props::get_loop_heads_ending( branching_node* end_node, // ------------------------------------------------------------------------------------------------ void fuzzing::iid_node_dependence_props::compute_loading_loops( branching_node* end_node, const loop_head_to_bodies_t& loop_heads_to_bodies, - loop_head_to_loaded_bits_counter& loading_loops ) + loop_head_to_loaded_bits_counter& loading_loops, + const loop_endings& loop_heads_ending ) { for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { loading_loops[ loop_head.id ] = { std::numeric_limits< natural_32_bit >::max(), @@ -1548,6 +1559,8 @@ void fuzzing::iid_node_dependence_props::compute_loading_loops( branching_node* } branching_node* node = end_node; + branching_node* prev_node = nullptr; + while ( node != nullptr ) { for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { if ( loop_head.id == node->get_location_id().id ) { @@ -1560,7 +1573,26 @@ void fuzzing::iid_node_dependence_props::compute_loading_loops( branching_node* } } - node = node->predecessor; + // Remove one loop count for branching that ends the loop + prev_node = node->predecessor; + if ( prev_node != nullptr ) { + bool node_direction = prev_node->successor( true ).pointer == node; + + for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { + auto it = loop_heads_ending.find( loop_head.id ); + if ( it == loop_heads_ending.end() ) { + continue; + } + + if ( loop_head.id == prev_node->get_location_id().id && it->second == node_direction ) { + auto& props = loading_loops[ loop_head.id ]; + props.loop_count--; + + } + } + } + + node = prev_node; } // Remove all loops that did not load any data inside @@ -1571,12 +1603,6 @@ void fuzzing::iid_node_dependence_props::compute_loading_loops( branching_node* ++it; } } - - // Remove one loop count for branching that ends the loop - for ( auto& [ id, props ] : loading_loops ) { - if ( props.loop_count != 0 ) - props.loop_count--; - } } // ------------------------------------------------------------------------------------------------ @@ -1585,7 +1611,7 @@ void iid_node_dependence_props::compute_dependencies_by_loading( branching_node* const loop_endings& loop_heads_ending ) { loop_head_to_loaded_bits_counter loading_loops; - compute_loading_loops( end_node, loop_heads_to_bodies, loading_loops ); + compute_loading_loops( end_node, loop_heads_to_bodies, loading_loops, loop_heads_ending ); branching_node* node = end_node; From 350ef518159bbafb6ff5462974ed5c3c05ec0b74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Wed, 26 Feb 2025 12:44:30 +0100 Subject: [PATCH 117/144] feat: correctly update ignored_nodes --- .../include/fuzzing/iid_vector_analysis.hpp | 7 +++- src/fuzzing/src/iid_vector_analysis.cpp | 33 +++++++++++++++++-- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index 6c1f6d9b..ab6343e7 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -115,6 +115,7 @@ struct iid_vector_analysis_statistics_per_node { struct iid_vector_analysis_statistics { std::map< location_id::id_type, iid_vector_analysis_statistics_per_node > iid_nodes_stats; std::vector< location_id::id_type > ignored_node_ids; + int processed_nodes; }; @@ -335,7 +336,9 @@ struct iid_node_dependence_props { struct iid_dependencies { void update_ignored_nodes( sensitivity_analysis& sensitivity ); void process_node_dependence( branching_node* node ); + void process_node_dependence_from_full_path( branching_node* node ); void remove_node_dependence( location_id::id_type id ); + void remove_all_covered( const std::unordered_set& covered_branchings ); iid_node_dependence_props& get_props( location_id::id_type id ); std::vector< location_id::id_type > get_iid_nodes(); std::optional< location_id::id_type > get_next_iid_node(); @@ -344,6 +347,7 @@ struct iid_dependencies { private: std::map< location_id::id_type, iid_node_dependence_props > node_id_to_equation_map; std::set< location_id::id_type > ignored_node_ids; + int processed_nodes = 0; public: // Configurations @@ -355,9 +359,10 @@ struct iid_dependencies { inline static int minimal_max_generation_for_other_node = 10; inline static int minimal_max_generation_artificial_data = 5; inline static float percentage_to_add_to_path = 0.4; - inline static bool create_artificial_data = true; + inline static bool create_artificial_data = false; }; + std::vector< node_id_with_direction > get_path( branching_node* node ); bool should_generate_more_data( const generation_state& state ); } // namespace fuzzing diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index c174f6d0..b1de7e0b 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -742,6 +742,7 @@ generated_path iid_node_dependence_props::generate_probabilities() // print_dependencies(); // matrix.print_matrix(); // submatrix.print_matrix(); + } std::optional< std::vector< equation > > best_vectors = get_best_vectors( submatrix, 1 ); @@ -1235,6 +1236,7 @@ void iid_node_dependence_props::compute_node_counts_for_loading_loops( nodes_to_ for ( const auto& body : loop_props.bodies ) { auto& [ left_count, right_count ] = path_counts[ body.node_id ]; + // loop_count = std::max( loop_count, left_count + right_count ); if ( loop_heads.contains( body.node_id ) ) { child_loop_counts[ body.node_id ] = std::max( left_count, right_count ); @@ -1587,7 +1589,6 @@ void fuzzing::iid_node_dependence_props::compute_loading_loops( branching_node* if ( loop_head.id == prev_node->get_location_id().id && it->second == node_direction ) { auto& props = loading_loops[ loop_head.id ]; props.loop_count--; - } } } @@ -1662,11 +1663,17 @@ void iid_node_dependence_props::compute_dependencies_by_loading( branching_node* for ( const auto& [ loop_head_id, body ] : loop_to_props ) { auto loading_props = loading_loops.at( loop_head_id ); - loop_properties& dependencies = loop_to_properties.get_props_by_loop_head_id( loop_head_id ); - dependencies.is_loading_loop = true; natural_32_bit loaded_bits = loading_props.max - loading_props.min; + + if ( loading_props.loop_count == 0 || loaded_bits == 0 ) { + continue; + } + double per_loop = double( loaded_bits ) / double( loading_props.loop_count ); + + loop_properties& dependencies = loop_to_properties.get_props_by_loop_head_id( loop_head_id ); + dependencies.is_loading_loop = true; dependencies.loaded_bits_per_loop.add( per_loop ); bool loop_head_end_direction = loop_heads_ending.at( loop_head_id ); @@ -1809,6 +1816,7 @@ void iid_dependencies::process_node_dependence( branching_node* node ) iid_node_dependence_props& props = node_id_to_equation_map[ node->get_location_id().id ]; props.process_node( node ); + processed_nodes++; } // ------------------------------------------------------------------------------------------------ @@ -1834,6 +1842,24 @@ void iid_dependencies::remove_node_dependence( location_id::id_type id ) } } +// ------------------------------------------------------------------------------------------------ +void fuzzing::iid_dependencies::remove_all_covered( const std::unordered_set< location_id >& covered_branchings ) +{ + std::set< location_id::id_type > covered_ids; + for ( const auto& branching : covered_branchings ) { + covered_ids.insert( branching.id ); + } + + for ( auto it = node_id_to_equation_map.begin(); it != node_id_to_equation_map.end(); ) { + if ( covered_ids.contains( it->first ) && it->second.get_generations_stats().state == generation_state::STATE_NOT_COVERED ) { + ignored_node_ids.insert( it->first ); + it = node_id_to_equation_map.erase( it ); + } else { + ++it; + } + } +} + // ------------------------------------------------------------------------------------------------ iid_node_dependence_props& iid_dependencies::get_props( location_id::id_type id ) { @@ -1904,6 +1930,7 @@ iid_vector_analysis_statistics fuzzing::iid_dependencies::get_stats() const stats.ignored_node_ids = std::vector< location_id::id_type >( ignored_node_ids.begin(), ignored_node_ids.end() ); + stats.processed_nodes = processed_nodes; return stats; } From 39b9f853caf7e8c46c60e9b703ad68b98cedcb9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 1 Mar 2025 16:44:38 +0100 Subject: [PATCH 118/144] feat: change the code to be more efficient - collect on whole trace, dependencies only once, compute matrix on demand --- .../include/fuzzing/iid_vector_analysis.hpp | 119 +- src/fuzzing/src/fuzzer.cpp | 22 +- src/fuzzing/src/iid_vector_analysis.cpp | 1252 ++++++++++------- 3 files changed, 815 insertions(+), 578 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index ab6343e7..7dff9b31 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -83,7 +83,6 @@ struct loop_head_properties { struct loop_properties { std::optional< node_id_with_direction > chosen_loop_head; std::map< node_id_with_direction, loop_head_properties > heads; - std::set< node_id_with_direction > bodies; bool is_loading_loop; @@ -91,20 +90,31 @@ struct loop_properties { mean_counter< float > loaded_bits_per_loop; std::map< location_id::id_type, loaded_bits_props > bits_read_by_node; - bool is_same( const std::set< location_id::id_type >& other_ids ) const; - std::set< location_id::id_type > get_all_ids() const; - std::set< location_id::id_type > get_loop_head_ids() const; - std::set< location_id::id_type > get_body_ids() const; + bool is_same( const std::unordered_set< location_id::id_type >& other_ids ) const; + const std::unordered_set< location_id::id_type >& get_all_ids() const; + const std::unordered_set< location_id::id_type >& get_loop_head_ids() const; location_id::id_type get_smallest_loop_head_id() const; + location_id::id_type get_smallest_id() const; void set_chosen_loop_head(); + void update_stored_ids(); + +private: + std::unordered_set< location_id::id_type > all_ids = {}; + std::unordered_set< location_id::id_type > loop_head_ids = {}; }; struct loop_dependencies { std::vector< loop_properties > loops; - loop_properties& get_props( const std::set< location_id::id_type >& ids, location_id::id_type loop_head_id ); + loop_properties& get_props( const std::unordered_set< location_id::id_type >& ids, + location_id::id_type loop_head_id ); void merge_properties(); + const loop_properties& get_props_by_loop_head_id( location_id::id_type loop_head_id ) const; loop_properties& get_props_by_loop_head_id( location_id::id_type loop_head_id ); + std::set< location_id::id_type > get_loop_heads( bool include_loading_loops ) const; + std::set< node_id_with_direction > + get_node_subsets_for_computation( const std::unordered_set< location_id::id_type >& matrix_ids ) const; + void print_dependencies() const; }; struct iid_vector_analysis_statistics_per_node { @@ -232,14 +242,18 @@ struct loaded_bits_counter { using loop_head_to_loaded_bits_counter = std::unordered_map< location_id::id_type, loaded_bits_counter >; -using loop_endings = std::map< location_id::id_type, bool >; +using loop_endings = std::unordered_map< location_id::id_type, bool >; using loop_head_to_bodies_t = std::unordered_map< location_id, std::unordered_set< location_id > >; using nodes_to_counts = std::map< location_id::id_type, node_counts >; struct equation_matrix { equation_matrix get_submatrix( std::set< node_id_with_direction > const& subset, bool unique ) const; - void process_node( branching_node* end_node ); - void add_equation( branching_node* end_node ); + void process_node( branching_node* end_node, const std::vector< node_id_with_direction >& path ); + void process_node_effective( branching_node* end_node, + bool compute_matrix, + const std::unordered_map< node_id_with_direction, int >& directions_in_path ); + void start_compute_matrix(); + void add_equation( branching_node* end_node, const std::vector< node_id_with_direction >& path ); bool contains( node_id_with_direction const& node ) const; std::pair< std::size_t, std::size_t > get_dimensions() const; std::map< equation, int > compute_vectors_with_hits(); @@ -249,25 +263,33 @@ struct equation_matrix { const iid_node_generations_stats& state ); int get_desired_vector_direction() const; float get_biggest_branching_value() const; + const std::unordered_set< location_id::id_type >& get_node_ids() const; void print_matrix(); BRANCHING_PREDICATE get_branching_predicate() const; private: - void recompute_matrix(); + void add_path( branching_node* end_node, + const std::unordered_map< node_id_with_direction, int >& directions_in_path ); std::vector< equation > matrix; std::vector< branching_node* > all_paths; - std::set< node_id_with_direction > nodes; + std::vector< node_id_with_direction > nodes; + std::unordered_set< location_id::id_type > node_ids; }; struct iid_node_dependence_props { - generated_path generate_probabilities(); + generated_path generate_probabilities( const loop_dependencies& loop_to_properties ); void process_node( branching_node* end_node ); + void process_path( branching_node* end_node, + const std::set< location_id::id_type >& seen_ids, + const std::vector< node_id_with_direction >& path ); + void process_path_effective( branching_node* end_node, + const std::unordered_map< node_id_with_direction, int >& directions_in_path ); iid_node_generations_stats& get_generations_stats() { return stats; } const iid_node_generations_stats& get_generations_stats() const { return stats; } - const loop_dependencies& get_dependencies_by_loops() const { return loop_to_properties; } + // const loop_dependencies& get_dependencies_by_loops() const { return loop_to_properties; } bool should_generate() const; bool too_much_failed_in_row( int max_failed_generations_in_row ) const; @@ -275,9 +297,7 @@ struct iid_node_dependence_props { void set_as_generating_artificial_data( int minimal_max_generation_artificial_data ); failed_generation_method get_method_for_failed_generation( bool is_first ); bool is_equal_branching_predicate() const; - void combine_props( const iid_node_dependence_props& other ); - void print_dependencies() const; void print_stats( bool only_state = false ) const; @@ -290,20 +310,26 @@ struct iid_node_dependence_props { std::map< location_id::id_type, int >& child_loop_counts, location_id::id_type loop_head_id, int minimum_count, + const loop_dependencies& loop_to_properties, bool use_random = false ); int compute_loading_loop_interation( nodes_to_counts& path_counts, location_id::id_type id, const std::set< location_id::id_type >& loop_heads, const loop_properties& props, - float loaded_bits_per_loop ); + const loop_dependencies& loop_to_properties ); void compute_node_counts_for_loading_loops( nodes_to_counts& path_counts, const equation& path, - const std::set< location_id::id_type >& loop_heads ); + const std::set< location_id::id_type >& loop_heads, + const loop_dependencies& loop_to_properties ); void compute_node_counts_for_loops( nodes_to_counts& path_counts, const equation& path, - const std::set< location_id::id_type >& loop_heads ); + const std::set< location_id::id_type >& loop_heads, + const loop_dependencies& loop_to_properties, + const std::unordered_set< location_id::id_type >& subset_ids ); nodes_to_counts compute_node_counts( const equation& path, - std::set< node_id_with_direction > const& all_leafs ); + std::set< node_id_with_direction > const& all_leafs, + const loop_dependencies& loop_to_properties, + const std::unordered_set< location_id::id_type >& subset_ids ); std::vector< equation > compute_best_vectors( const std::map< equation, int >& vectors_with_hits, int number_of_vectors, bool use_random, @@ -313,40 +339,49 @@ struct iid_node_dependence_props { equation& best_vector ); std::vector< equation > get_random_vector( const std::map< equation, int >& vectors_with_hits, int number_of_vectors ); - std::set< node_id_with_direction > get_node_subsets_for_computation(); - loop_endings get_loop_heads_ending( branching_node* end_node, loop_head_to_bodies_t& loop_heads_to_bodies ); void compute_loading_loops( branching_node* end_node, const loop_head_to_bodies_t& loop_heads_to_bodies, loop_head_to_loaded_bits_counter& loading_loops, const loop_endings& loop_heads_ending ); - void compute_dependencies_by_loading( branching_node* end_node, - const loop_head_to_bodies_t& loop_heads_to_bodies, - const loop_endings& loop_heads_ending ); - void compute_dependencies_by_loops( const loop_head_to_bodies_t& loop_heads_to_bodies, - const loop_endings& loop_heads_ending ); - generated_path generate_path_from_node_counts( const nodes_to_counts& path_counts ); - std::set< location_id::id_type > get_loop_heads( bool include_loading_loops = true ); + generated_path generate_path_from_node_counts( const nodes_to_counts& path_counts, + const loop_dependencies& loop_to_properties ); equation_matrix matrix; - loop_dependencies loop_to_properties; - iid_node_generations_stats stats; + + bool continuously_gathering_data = false; }; struct iid_dependencies { void update_ignored_nodes( sensitivity_analysis& sensitivity ); void process_node_dependence( branching_node* node ); - void process_node_dependence_from_full_path( branching_node* node ); + void process_node_dependence_from_full_path( branching_node* end_node ); void remove_node_dependence( location_id::id_type id ); - void remove_all_covered( const std::unordered_set& covered_branchings ); + void remove_all_covered( const std::unordered_set< location_id >& covered_branchings ); iid_node_dependence_props& get_props( location_id::id_type id ); std::vector< location_id::id_type > get_iid_nodes(); std::optional< location_id::id_type > get_next_iid_node(); + + generated_path generate_probabilities(); + iid_vector_analysis_statistics get_stats() const; private: + loop_endings get_loop_heads_ending( branching_node* end_node, loop_head_to_bodies_t& loop_heads_to_bodies ); + void compute_dependencies_by_loading( branching_node* end_node, + const loop_head_to_bodies_t& loop_heads_to_bodies, + const loop_endings& loop_heads_ending ); + void compute_dependencies_by_loops( const loop_head_to_bodies_t& loop_heads_to_bodies, + const loop_endings& loop_heads_ending ); + void compute_loading_loops( branching_node* end_node, + const loop_head_to_bodies_t& loop_heads_to_bodies, + loop_head_to_loaded_bits_counter& loading_loops, + const loop_endings& loop_heads_ending ); + void compute_paths( branching_node* end_node ); + std::map< location_id::id_type, iid_node_dependence_props > node_id_to_equation_map; - std::set< location_id::id_type > ignored_node_ids; + std::unordered_set< location_id::id_type > ignored_node_ids; + loop_dependencies loop_to_properties; int processed_nodes = 0; public: @@ -359,10 +394,26 @@ struct iid_dependencies { inline static int minimal_max_generation_for_other_node = 10; inline static int minimal_max_generation_artificial_data = 5; inline static float percentage_to_add_to_path = 0.4; - inline static bool create_artificial_data = false; + inline static bool create_artificial_data = true; + + inline static bool verbose = false; }; std::vector< node_id_with_direction > get_path( branching_node* node ); +std::unordered_map< node_id_with_direction, int > get_directions_in_path( branching_node* node ); bool should_generate_more_data( const generation_state& state ); } // namespace fuzzing + +namespace std +{ +template <> +struct hash< fuzzing::node_id_with_direction > { + std::size_t operator()( const fuzzing::node_id_with_direction& key ) const noexcept + { + std::size_t h1 = std::hash< location_id::id_type >{}( key.node_id ); + std::size_t h2 = std::hash< bool >{}( key.branching_direction ); + return h1 ^ ( h2 << 1 ); + } +}; +} // namespace std \ No newline at end of file diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index 696c1e78..c1598134 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -434,6 +434,7 @@ void fuzzer::detect_loops_along_path_to_node( std::vector* const loops ) { + TMPROF_BLOCK(); struct loop_exit_props { branching_node* exit; @@ -1114,8 +1115,6 @@ execution_record::execution_flags fuzzer::process_execution_results() ); construction_props.diverging_node = entry_branching; - iid_dependences.process_node_dependence(entry_branching); - ++statistics.nodes_created; } @@ -1227,8 +1226,6 @@ execution_record::execution_flags fuzzer::process_execution_results() new_node }); - iid_dependences.process_node_dependence(new_node); - ++statistics.nodes_created; if (construction_props.diverging_node == nullptr) @@ -1248,6 +1245,9 @@ execution_record::execution_flags fuzzer::process_execution_results() construction_props.leaf->successor(trace->back().direction).pointer }); + if ( use_vector_analysis ) + iid_dependences.process_node_dependence_from_full_path( construction_props.leaf ); + if (construction_props.diverging_node != nullptr) { auto const it_and_state = leaf_branchings.insert(construction_props.leaf); @@ -1429,7 +1429,9 @@ void fuzzer::do_cleanup() for (auto it = iid_pivots.begin(); it != iid_pivots.end(); ) if (covered_branchings.contains(it->first)) { - iid_dependences.remove_node_dependence(it->first.id); + if ( use_vector_analysis ) + iid_dependences.remove_node_dependence(it->first.id); + it = iid_pivots.erase(it); } else @@ -1440,6 +1442,9 @@ void fuzzer::do_cleanup() it = coverage_failures_with_hope.erase(it); else ++it; + + if ( use_vector_analysis ) + iid_dependences.remove_all_covered( covered_branchings ); } @@ -1688,12 +1693,9 @@ branching_node* fuzzer::select_iid_coverage_target() it_pivot->second.histogram_ptr->merge(hit_counts); generated_path path; - std::optional< location_id::id_type > iid_location = iid_dependences.get_next_iid_node(); - if ( use_vector_analysis && iid_location.has_value() ) { - iid_node_dependence_props& node_props = iid_dependences.get_props( *iid_location ); - // std::cout << "Computing probabilities for location " << ( *iid_location ) << std::endl; - path = node_props.generate_probabilities(); + if ( use_vector_analysis ) { + path = iid_dependences.generate_probabilities(); for ( const auto& path_props : path.get_path() ) { auto it = histogram.find( path_props.first ); diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index b1de7e0b..8c87a05e 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -236,6 +236,9 @@ int equation::get_one_way_branching_count() const // ------------------------------------------------------------------------------------------------ int equation::get_biggest_value() const { return *std::max_element( values.begin(), values.end() ); } +// ------------------------------------------------------------------------------------------------ +const std::unordered_set< location_id::id_type >& fuzzing::equation_matrix::get_node_ids() const { return node_ids; } + // ------------------------------------------------------------------------------------------------ bool equation::is_any_negative() const { @@ -304,68 +307,84 @@ auto node_id_with_direction::operator<=>( node_id_with_direction const& other ) // loop_properties // ------------------------------------------------------------------------------------------------ -bool fuzzing::loop_properties::is_same( const std::set< location_id::id_type >& other_ids ) const +bool fuzzing::loop_properties::is_same( const std::unordered_set< location_id::id_type >& other_ids ) const { - return get_all_ids() == other_ids; + // TMPROF_BLOCK(); + return all_ids == other_ids; } // ------------------------------------------------------------------------------------------------ -std::set< location_id::id_type > fuzzing::loop_properties::get_all_ids() const +const std::unordered_set< location_id::id_type >& fuzzing::loop_properties::get_all_ids() const { - std::set< location_id::id_type > all_ids; - - for ( const auto& [ head, props ] : heads ) { - all_ids.insert( head.node_id ); - } - - for ( const auto& body : bodies ) { - all_ids.insert( body.node_id ); - } - + // TMPROF_BLOCK(); return all_ids; } // ------------------------------------------------------------------------------------------------ -std::set< location_id::id_type > fuzzing::loop_properties::get_loop_head_ids() const +const std::unordered_set< location_id::id_type >& fuzzing::loop_properties::get_loop_head_ids() const { - std::set< location_id::id_type > loop_head_ids; + // TMPROF_BLOCK(); + return loop_head_ids; +} - for ( const auto& [ head, props ] : heads ) { - loop_head_ids.insert( head.node_id ); +// ------------------------------------------------------------------------------------------------ +location_id::id_type fuzzing::loop_properties::get_smallest_loop_head_id() const +{ + // TMPROF_BLOCK(); + + location_id::id_type smallest_id = *loop_head_ids.begin(); + + for ( const auto& id : loop_head_ids ) { + if ( id < smallest_id ) { + smallest_id = id; + } } - return loop_head_ids; + return smallest_id; } // ------------------------------------------------------------------------------------------------ -std::set< location_id::id_type > fuzzing::loop_properties::get_body_ids() const +location_id::id_type fuzzing::loop_properties::get_smallest_id() const { - std::set< location_id::id_type > body_ids; + // TMPROF_BLOCK(); + location_id::id_type smallest_id = *all_ids.begin(); - for ( const auto& body : bodies ) { - body_ids.insert( body.node_id ); + for ( const auto& id : all_ids ) { + if ( id < smallest_id ) { + smallest_id = id; + } } - return body_ids; + return smallest_id; } // ------------------------------------------------------------------------------------------------ -location_id::id_type fuzzing::loop_properties::get_smallest_loop_head_id() const +void fuzzing::loop_properties::update_stored_ids() { - location_id::id_type smallest_id = heads.begin()->first.node_id; + // TMPROF_BLOCK(); for ( const auto& [ head, props ] : heads ) { - if ( head.node_id < smallest_id ) { - smallest_id = head.node_id; - } + all_ids.insert( head.node_id ); } - return smallest_id; + for ( const auto& body : bodies ) { + all_ids.insert( body.node_id ); + } + + if ( loop_head_ids.size() == heads.size() ) { + return; + } + + for ( const auto& [ head, props ] : heads ) { + loop_head_ids.insert( head.node_id ); + } } // ------------------------------------------------------------------------------------------------ void fuzzing::loop_properties::set_chosen_loop_head() { + // TMPROF_BLOCK(); + for ( const auto& [ head, props ] : heads ) { if ( !chosen_loop_head.has_value() ) { chosen_loop_head = head; @@ -379,9 +398,11 @@ void fuzzing::loop_properties::set_chosen_loop_head() // loop_dependencies // ------------------------------------------------------------------------------------------------ -loop_properties& fuzzing::loop_dependencies::get_props( const std::set< location_id::id_type >& ids, +loop_properties& fuzzing::loop_dependencies::get_props( const std::unordered_set< location_id::id_type >& ids, location_id::id_type loop_head_id ) { + // TMPROF_BLOCK(); + for ( loop_properties& loop : loops ) { for ( const auto& [ head, props ] : loop.heads ) { if ( head.node_id == loop_head_id ) { @@ -401,57 +422,52 @@ loop_properties& fuzzing::loop_dependencies::get_props( const std::set< location // ------------------------------------------------------------------------------------------------ void fuzzing::loop_dependencies::merge_properties() { - for ( auto it = loops.begin(); it != loops.end(); it++ ) { - std::set< location_id::id_type > head_ids = it->get_loop_head_ids(); + // TMPROF_BLOCK(); - for ( auto body_it = it->bodies.begin(); body_it != it->bodies.end(); ) { - if ( head_ids.contains( body_it->node_id ) ) { - body_it = it->bodies.erase( body_it ); - } else { - ++body_it; - } - } + for ( loop_properties& loop : loops ) { + loop.update_stored_ids(); } - for ( auto it_1 = loops.begin(); it_1 != loops.end(); it_1++ ) { - for ( auto it_2 = loops.begin(); it_2 != loops.end(); ) { - if ( it_1 == it_2 ) { - ++it_2; - continue; - } + std::unordered_map< location_id::id_type, loop_properties > merged_loops; - if ( it_1->is_same( it_2->get_all_ids() ) ) { - for ( const auto& [ head, props ] : it_2->heads ) { - it_1->heads[ head ].count += props.count; - } + for ( const auto& loop : loops ) { + location_id::id_type smallest_id = loop.get_smallest_id(); - it_2 = loops.erase( it_2 ); + auto [ it, inserted ] = merged_loops.try_emplace( smallest_id, std::move( loop ) ); - it_1 = loops.begin(); - it_2 = loops.begin(); - } else { - ++it_2; + if ( !inserted ) { + for ( const auto& [ head, props ] : loop.heads ) { + it->second.heads[ head ].count += props.count; } + it->second.bodies.insert( loop.bodies.begin(), loop.bodies.end() ); } } - for ( auto it = loops.begin(); it != loops.end(); it++ ) { - std::set< location_id::id_type > head_ids = it->get_loop_head_ids(); + for ( auto& [ id, loop ] : merged_loops ) { + const std::unordered_set< location_id::id_type >& head_ids = loop.get_loop_head_ids(); - for ( auto body_it = it->bodies.begin(); body_it != it->bodies.end(); ) { + for ( auto body_it = loop.bodies.begin(); body_it != loop.bodies.end(); ) { if ( head_ids.contains( body_it->node_id ) ) { - body_it = it->bodies.erase( body_it ); + body_it = loop.bodies.erase( body_it ); } else { ++body_it; } } } + + loops.clear(); + loops.reserve( merged_loops.size() ); + + for ( auto& [ id, loop ] : merged_loops ) { + loop.update_stored_ids(); + loops.push_back( std::move( loop ) ); + } } // ------------------------------------------------------------------------------------------------ -loop_properties& fuzzing::loop_dependencies::get_props_by_loop_head_id( location_id::id_type loop_head_id ) +const loop_properties& fuzzing::loop_dependencies::get_props_by_loop_head_id( location_id::id_type loop_head_id ) const { - for ( loop_properties& loop : loops ) { + for ( const loop_properties& loop : loops ) { for ( const auto& [ head, props ] : loop.heads ) { if ( head.node_id == loop_head_id ) { return loop; @@ -462,12 +478,132 @@ loop_properties& fuzzing::loop_dependencies::get_props_by_loop_head_id( location throw std::runtime_error( "Loop head not found: " + std::to_string( loop_head_id ) ); } +// ------------------------------------------------------------------------------------------------ +loop_properties& fuzzing::loop_dependencies::get_props_by_loop_head_id( location_id::id_type loop_head_id ) +{ + return const_cast< loop_properties& >( + static_cast< const fuzzing::loop_dependencies& >( *this ).get_props_by_loop_head_id( loop_head_id ) ); +} + +// ------------------------------------------------------------------------------------------------ +std::set< location_id::id_type > fuzzing::loop_dependencies::get_loop_heads( bool include_loading_loops ) const +{ + std::set< location_id::id_type > loop_heads; + for ( const auto& props : loops ) { + if ( props.is_loading_loop && !include_loading_loops ) { + continue; + } + + for ( const auto& [ head, _ ] : props.heads ) { + loop_heads.insert( head.node_id ); + } + } + + return loop_heads; +} + +// ------------------------------------------------------------------------------------------------ +std::set< node_id_with_direction > fuzzing::loop_dependencies::get_node_subsets_for_computation( + const std::unordered_set< location_id::id_type >& matrix_ids ) const +{ + std::set< node_id_with_direction > computation_subset; + + for ( const auto& loop : loops ) { + if ( loop.is_loading_loop || !matrix_ids.contains( loop.get_smallest_loop_head_id() ) ) { + continue; + } + + for ( const auto& body : loop.bodies ) { + computation_subset.insert( body ); + } + + for ( const auto& [ head, _ ] : loop.heads ) { + computation_subset.insert( { head.node_id, !head.branching_direction } ); + } + } + + return computation_subset; +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::loop_dependencies::print_dependencies() const +{ + bool print_dependencies_by_loops = true; + bool print_dependencies_by_loading = true; + + if ( !print_dependencies_by_loops && !print_dependencies_by_loading ) { + return; + } + + std::cout << "# Dependencies:" << std::endl; + if ( print_dependencies_by_loops ) { + std::cout << "## Dependencies by loops:" << std::endl; + for ( const auto& loop : loops ) { + if ( loop.is_loading_loop ) { + continue; + } + + std::cout << "Loop heads:" << std::endl; + for ( const auto& [ head, head_props ] : loop.heads ) { + std::cout << "- " << head << " (" << head_props.count << ")" << std::endl; + } + + std::cout << "Loop bodies:" << std::endl; + for ( const auto& body : loop.bodies ) { + std::cout << "- " << body << std::endl; + } + + if ( loop.is_loading_loop ) { + std::cout << "Loading loop" << std::endl; + std::cout << "Average bits per loop: " << loop.loaded_bits_per_loop.mean << std::endl; + for ( const auto& [ body, body_props ] : loop.bits_read_by_node ) { + std::cout << "- " << body << ", Bits: " << body_props.average_bits_read.mean + << ", offset: " << body_props.minimal_bit_offset << std::endl; + } + } + } + } + + if ( print_dependencies_by_loading ) { + std::cout << "## Dependencies by loading:" << std::endl; + for ( const auto& loop : loops ) { + if ( !loop.is_loading_loop ) { + continue; + } + + std::cout << "Loop heads:" << std::endl; + for ( const auto& [ head, head_props ] : loop.heads ) { + std::cout << "- " << head << " (" << head_props.count << ")" << std::endl; + } + + std::cout << "Loop bodies:" << std::endl; + for ( const auto& body : loop.bodies ) { + std::cout << "- " << body << std::endl; + } + + std::cout << "Dependent nodes:" << std::endl; + for ( const auto& [ body, body_props ] : loop.bits_read_by_node ) { + std::cout << "- " << body << ", Bits: " << body_props.average_bits_read.mean + << ", offset: " << body_props.minimal_bit_offset << std::endl; + } + + std::cout << "Loaded bits per loop: " << loop.loaded_bits_per_loop.mean << std::endl; + } + } +} + // equation_matrix // ------------------------------------------------------------------------------------------------ equation_matrix equation_matrix::get_submatrix( std::set< node_id_with_direction > const& subset, bool unique ) const { + // TMPROF_BLOCK(); + equation_matrix result; - result.nodes = subset; + result.nodes = std::vector< node_id_with_direction >( subset.begin(), subset.end() ); + + for ( const node_id_with_direction& nav : result.nodes ) { + result.node_ids.insert( nav.node_id ); + } for ( int i = 0; i < matrix.size(); ++i ) { const equation& row = matrix[ i ]; @@ -477,6 +613,8 @@ equation_matrix equation_matrix::get_submatrix( std::set< node_id_with_direction auto it = std::find( nodes.begin(), nodes.end(), nav ); if ( it != nodes.end() ) { new_row_values.push_back( row.values[ std::distance( nodes.begin(), it ) ] ); + } else { + new_row_values.push_back( 0 ); } } @@ -497,55 +635,41 @@ equation_matrix equation_matrix::get_submatrix( std::set< node_id_with_direction } // ------------------------------------------------------------------------------------------------ -void equation_matrix::process_node( branching_node* end_node ) +void fuzzing::equation_matrix::process_node_effective( + branching_node* end_node, + bool compute_matrix, + const std::unordered_map< node_id_with_direction, int >& directions_in_path ) { + // TMPROF_BLOCK(); + all_paths.push_back( end_node ); - std::vector< node_id_with_direction > path = get_path( end_node ); - bool new_node = false; - for ( const node_id_with_direction& nav : path ) { - for ( bool direction : { true, false } ) { - auto [ it, inserted ] = nodes.insert( { nav.node_id, direction } ); - new_node |= inserted; - } + if ( !compute_matrix ) { + return; } - if ( new_node ) { - recompute_matrix(); - } else { - add_equation( end_node ); - } + add_path( end_node, directions_in_path ); } // ------------------------------------------------------------------------------------------------ -void equation_matrix::add_equation( branching_node* end_node ) +void fuzzing::equation_matrix::start_compute_matrix() { - TMPROF_BLOCK(); - - std::map< node_id_with_direction, int > directions_in_path; - for ( const node_id_with_direction& navigation : nodes ) { - directions_in_path[ navigation ] = 0; - } - - std::vector< node_id_with_direction > path_nodes = get_path( end_node ); - - for ( const node_id_with_direction& nav : path_nodes ) { - if ( nodes.contains( nav ) ) { - directions_in_path[ nav ]++; - } - } + // TMPROF_BLOCK(); - std::vector< int > values_in_path; - for ( const auto& [ direction, count ] : directions_in_path ) { - values_in_path.push_back( count ); + for ( branching_node* node : all_paths ) { + std::unordered_map< node_id_with_direction, int > directions_in_path = get_directions_in_path( node ); + add_path( node, directions_in_path ); } - - equation row = { values_in_path, end_node->best_coverage_value }; - matrix.push_back( row ); } // ------------------------------------------------------------------------------------------------ -bool equation_matrix::contains( node_id_with_direction const& node ) const { return nodes.contains( node ); } +bool equation_matrix::contains( node_id_with_direction const& node ) const +{ + // TMPROF_BLOCK(); + + auto it = std::find( nodes.begin(), nodes.end(), node ); + return it != nodes.end(); +} // ------------------------------------------------------------------------------------------------ std::pair< std::size_t, std::size_t > equation_matrix::get_dimensions() const @@ -556,6 +680,8 @@ std::pair< std::size_t, std::size_t > equation_matrix::get_dimensions() const // ------------------------------------------------------------------------------------------------ std::map< equation, int > equation_matrix::compute_vectors_with_hits() { + // TMPROF_BLOCK(); + std::map< equation, int > vectors_with_hits; for ( int i = 0; i < matrix.size(); ++i ) { @@ -623,6 +749,8 @@ equation_matrix::get_new_subset_counts_from_vectors( const std::vector< equation int generation_count_after_covered, const iid_node_generations_stats& stats ) { + // TMPROF_BLOCK(); + INVARIANT( !vectors.empty() ); INVARIANT( vectors[ 0 ].values.size() == nodes.size() ); @@ -711,38 +839,78 @@ BRANCHING_PREDICATE equation_matrix::get_branching_predicate() const } // ------------------------------------------------------------------------------------------------ -void equation_matrix::recompute_matrix() +void fuzzing::equation_matrix::add_path( branching_node* end_node, + const std::unordered_map< node_id_with_direction, int >& directions_in_path ) { - TMPROF_BLOCK(); + // TMPROF_BLOCK(); + + int added_nodes = 0; + + for ( const auto& [ nav, count ] : directions_in_path ) { + auto [ it, inserted ] = node_ids.insert( nav.node_id ); + if ( !inserted ) { + continue; + } - matrix.clear(); + for ( bool direction : { true, false } ) { + nodes.push_back( { nav.node_id, direction } ); + added_nodes++; + } + } + + for ( int i = 0; i < added_nodes; ++i ) { + for ( int j = 0; j < matrix.size(); ++j ) { + matrix[ j ].values.push_back( 0 ); + } + } - for ( branching_node* path : all_paths ) { - add_equation( path ); + std::vector< int > values_in_path; + for ( const auto& node : nodes ) { + auto it = directions_in_path.find( node ); + if ( it == directions_in_path.end() ) { + values_in_path.push_back( 0 ); + } else { + values_in_path.push_back( it->second ); + } } + + equation row = { values_in_path, end_node->best_coverage_value }; + matrix.push_back( row ); } // iid_node_dependence_props // ------------------------------------------------------------------------------------------------ -generated_path iid_node_dependence_props::generate_probabilities() +generated_path iid_node_dependence_props::generate_probabilities( const loop_dependencies& loop_to_properties ) { + // TMPROF_BLOCK(); + stats.method_calls++; - std::set< node_id_with_direction > computation_subset = get_node_subsets_for_computation(); + if ( !continuously_gathering_data ) { + continuously_gathering_data = true; + matrix.start_compute_matrix(); + } + + std::set< node_id_with_direction > computation_subset = + loop_to_properties.get_node_subsets_for_computation( matrix.get_node_ids() ); if ( computation_subset.empty() || loop_to_properties.loops.empty() ) { return {}; } - TMPROF_BLOCK(); + std::unordered_set< location_id::id_type > subset_ids; + for ( const auto& nav : computation_subset ) { + subset_ids.insert( nav.node_id ); + } + stats.generation_starts++; equation_matrix submatrix = matrix.get_submatrix( computation_subset, true ); + { // print_stats( true ); - // print_dependencies(); + // loop_to_properties.print_dependencies(); // matrix.print_matrix(); // submatrix.print_matrix(); - } std::optional< std::vector< equation > > best_vectors = get_best_vectors( submatrix, 1 ); @@ -763,24 +931,25 @@ generated_path iid_node_dependence_props::generate_probabilities() return return_empty_path(); } - nodes_to_counts node_counts = compute_node_counts( new_subset_counts.value(), computation_subset ); - generated_path path = generate_path_from_node_counts( node_counts ); + nodes_to_counts node_counts = + compute_node_counts( new_subset_counts.value(), computation_subset, loop_to_properties, subset_ids ); + generated_path path = generate_path_from_node_counts( node_counts, loop_to_properties ); - // std::cout << "Generated path: " << std::endl << path << std::endl; + if ( iid_dependencies::verbose ) { + std::cout << "Generated path: " << std::endl << path << std::endl; + } return return_path( path ); } + // ------------------------------------------------------------------------------------------------ -void iid_node_dependence_props::process_node( branching_node* end_node ) +void fuzzing::iid_node_dependence_props::process_path_effective( + branching_node* end_node, + const std::unordered_map< node_id_with_direction, int >& directions_in_path ) { - loop_head_to_bodies_t loop_heads_to_bodies; - loop_endings loop_heads_ending = get_loop_heads_ending( end_node, loop_heads_to_bodies ); - - matrix.process_node( end_node ); - - compute_dependencies_by_loops( loop_heads_to_bodies, loop_heads_ending ); - compute_dependencies_by_loading( end_node, loop_heads_to_bodies, loop_heads_ending ); + // TMPROF_BLOCK(); + matrix.process_node_effective( end_node, continuously_gathering_data, directions_in_path ); } // ------------------------------------------------------------------------------------------------ @@ -859,114 +1028,17 @@ bool iid_node_dependence_props::is_equal_branching_predicate() const return matrix.get_branching_predicate() == BRANCHING_PREDICATE::BP_EQUAL; } -// ------------------------------------------------------------------------------------------------ -void fuzzing::iid_node_dependence_props::combine_props( const iid_node_dependence_props& other ) -{ - std::vector< loop_properties > loops_to_add; - - for ( const auto& other_loop : other.loop_to_properties.loops ) { - for ( const auto& loop : loop_to_properties.loops ) { - std::set< location_id::id_type > other_ids = other_loop.get_loop_head_ids(); - std::set< location_id::id_type > ids = loop.get_loop_head_ids(); - - std::set< location_id::id_type > intersection; - - std::set_intersection( other_ids.begin(), - other_ids.end(), - ids.begin(), - ids.end(), - std::inserter( intersection, intersection.begin() ) ); - - if ( !intersection.empty() ) { - loops_to_add.push_back( other_loop ); - } - } - } - - for ( const auto& loop : loops_to_add ) { - loop_to_properties.loops.push_back( loop ); - } - - loop_to_properties.merge_properties(); -} // ------------------------------------------------------------------------------------------------ -void iid_node_dependence_props::print_dependencies() const +void iid_node_dependence_props::print_stats( bool only_state ) const { - bool print_dependencies_by_loops = true; - bool print_dependencies_by_loading = true; - - if ( !print_dependencies_by_loops && !print_dependencies_by_loading ) { - return; - } - - std::cout << "# Dependencies:" << std::endl; - if ( print_dependencies_by_loops ) { - std::cout << "## Dependencies by loops:" << std::endl; - for ( const auto& loop : loop_to_properties.loops ) { - if ( loop.is_loading_loop ) { - continue; - } - - std::cout << "Loop heads:" << std::endl; - for ( const auto& [ head, head_props ] : loop.heads ) { - std::cout << "- " << head << " (" << head_props.count << ")" << std::endl; - } - - std::cout << "Loop bodies:" << std::endl; - for ( const auto& body : loop.bodies ) { - std::cout << "- " << body << std::endl; - } - - if ( loop.is_loading_loop ) { - std::cout << "Loading loop" << std::endl; - std::cout << "Average bits per loop: " << loop.loaded_bits_per_loop.mean << std::endl; - for ( const auto& [ body, body_props ] : loop.bits_read_by_node ) { - std::cout << "- " << body << ", Bits: " << body_props.average_bits_read.mean - << ", offset: " << body_props.minimal_bit_offset << std::endl; - } - } - } - } - - if ( print_dependencies_by_loading ) { - std::cout << "## Dependencies by loading:" << std::endl; - for ( const auto& loop : loop_to_properties.loops ) { - if ( !loop.is_loading_loop ) { - continue; - } - - std::cout << "Loop heads:" << std::endl; - for ( const auto& [ head, head_props ] : loop.heads ) { - std::cout << "- " << head << " (" << head_props.count << ")" << std::endl; - } - - std::cout << "Loop bodies:" << std::endl; - for ( const auto& body : loop.bodies ) { - std::cout << "- " << body << std::endl; - } - - std::cout << "Dependent nodes:" << std::endl; - for ( const auto& [ body, body_props ] : loop.bits_read_by_node ) { - std::cout << "- " << body << ", Bits: " << body_props.average_bits_read.mean - << ", offset: " << body_props.minimal_bit_offset << std::endl; - } - - std::cout << "Loaded bits per loop: " << loop.loaded_bits_per_loop.mean << std::endl; - } - } -} - -// ------------------------------------------------------------------------------------------------ -void iid_node_dependence_props::print_stats( bool only_state ) const -{ - switch ( stats.state ) { - case generation_state::STATE_NOT_COVERED: - std::cout << "Status: STATE_NOT_COVERED" << std::endl; - if ( !only_state ) { - std::cout << "Failed generations/Total generations: " << stats.failed_generations << "/" - << stats.generation_starts << std::endl; - std::cout << "Failed generations in row: " << stats.failed_generations_in_row << std::endl; + switch ( stats.state ) { + case generation_state::STATE_NOT_COVERED: + std::cout << "Status: STATE_NOT_COVERED" << std::endl; + if ( !only_state ) { + std::cout << "Failed generations/Total generations: " << stats.failed_generations << "/" + << stats.generation_starts << std::endl; + std::cout << "Failed generations in row: " << stats.failed_generations_in_row << std::endl; } break; case generation_state::STATE_GENERATION_MORE: @@ -997,6 +1069,8 @@ void iid_node_dependence_props::print_stats( bool only_state ) const void fuzzing::iid_node_dependence_props::generate_vectors_if_not_enough_data( std::vector< equation >& best_vectors, equation_matrix& submatrix ) { + // TMPROF_BLOCK(); + best_vectors = std::vector< equation >(); int desired_direction = submatrix.get_desired_vector_direction(); std::map< equation, int > vectors = submatrix.compute_vectors_with_hits(); @@ -1032,6 +1106,8 @@ void fuzzing::iid_node_dependence_props::generate_vectors_if_not_enough_data( st std::optional< std::vector< equation > > fuzzing::iid_node_dependence_props::get_best_vectors( equation_matrix& submatrix, int number_of_vectors ) { + // TMPROF_BLOCK(); + std::map< equation, int > vectors = submatrix.compute_vectors_with_hits(); if ( vectors.empty() ) { return std::nullopt; @@ -1093,12 +1169,16 @@ generated_path iid_node_dependence_props::return_path( const generated_path& pat } // ------------------------------------------------------------------------------------------------ -void iid_node_dependence_props::compute_node_counts_for_nested_loops( nodes_to_counts& path_counts, - std::map< location_id::id_type, int >& child_loop_counts, - location_id::id_type loop_head_id, - int minimum_count, - bool use_random ) +void iid_node_dependence_props::compute_node_counts_for_nested_loops( + nodes_to_counts& path_counts, + std::map< location_id::id_type, int >& child_loop_counts, + location_id::id_type loop_head_id, + int minimum_count, + const loop_dependencies& loop_to_properties, + bool use_random ) { + // TMPROF_BLOCK(); + INVARIANT( !child_loop_counts.empty() ); int max_child_count = @@ -1131,7 +1211,7 @@ void iid_node_dependence_props::compute_node_counts_for_nested_loops( nodes_to_c *possible_counts.rbegin(); for ( auto& [ node_id, count ] : child_loop_counts ) { - loop_properties& props = loop_to_properties.get_props_by_loop_head_id( node_id ); + const loop_properties& props = loop_to_properties.get_props_by_loop_head_id( node_id ); for ( auto& [ head, _ ] : props.heads ) { auto& [ left_count, right_count ] = path_counts[ head.node_id ]; @@ -1145,7 +1225,7 @@ void iid_node_dependence_props::compute_node_counts_for_nested_loops( nodes_to_c } - loop_properties& props = loop_to_properties.get_props_by_loop_head_id( loop_head_id ); + const loop_properties& props = loop_to_properties.get_props_by_loop_head_id( loop_head_id ); for ( auto& [ head, _ ] : props.heads ) { if ( head.branching_direction ) { path_counts[ head.node_id ] = { highest_count, 1 }; @@ -1156,13 +1236,16 @@ void iid_node_dependence_props::compute_node_counts_for_nested_loops( nodes_to_c } // ------------------------------------------------------------------------------------------------ -int fuzzing::iid_node_dependence_props::compute_loading_loop_interation( nodes_to_counts& path_counts, - location_id::id_type id, - const std::set< location_id::id_type >& loop_heads, - const loop_properties& props, - float loaded_bits_per_loop ) +int fuzzing::iid_node_dependence_props::compute_loading_loop_interation( + nodes_to_counts& path_counts, + location_id::id_type id, + const std::set< location_id::id_type >& loop_heads, + const loop_properties& props, + const loop_dependencies& loop_to_properties ) { - float loaded_per_loop = props.loaded_bits_per_loop.mean - loaded_bits_per_loop; + // TMPROF_BLOCK(); + + float loaded_per_loop = props.loaded_bits_per_loop.mean; if ( loaded_per_loop <= 0 ) { loaded_per_loop = 8; } @@ -1191,8 +1274,11 @@ int fuzzing::iid_node_dependence_props::compute_loading_loop_interation( nodes_t // ------------------------------------------------------------------------------------------------ void iid_node_dependence_props::compute_node_counts_for_loading_loops( nodes_to_counts& path_counts, const equation& path, - const std::set< location_id::id_type >& loop_heads ) + const std::set< location_id::id_type >& loop_heads, + const loop_dependencies& loop_to_properties ) { + // TMPROF_BLOCK(); + for ( const auto& loop_props : std::ranges::views::reverse( loop_to_properties.loops ) ) { if ( !loop_props.is_loading_loop ) { continue; @@ -1201,27 +1287,13 @@ void iid_node_dependence_props::compute_node_counts_for_loading_loops( nodes_to_ int loop_count = 1; std::map< location_id::id_type, int > child_loop_counts; - float loaded_bits_per_loop = 0.0f; - - for ( const auto& body : loop_props.bodies ) { - if ( !loop_heads.contains( body.node_id ) ) { - continue; - } - - - const loop_properties& body_props = loop_to_properties.get_props_by_loop_head_id( body.node_id ); - if ( body_props.is_loading_loop ) { - // loaded_bits_per_loop += body_props.loaded_bits_per_loop.mean; - } - } - for ( const auto& body : loop_props.nodes_dependent_by_loading ) { if ( !path_counts.contains( body.node_id ) ) { continue; } int minimal_count = compute_loading_loop_interation( - path_counts, body.node_id, loop_heads, loop_props, loaded_bits_per_loop ); + path_counts, body.node_id, loop_heads, loop_props, loop_to_properties ); loop_count = std::max( loop_count, minimal_count ); } @@ -1231,7 +1303,7 @@ void iid_node_dependence_props::compute_node_counts_for_loading_loops( nodes_to_ if ( inner_loop_props.is_loading_loop ) { loop_count = 1; } - } + } } for ( const auto& body : loop_props.bodies ) { @@ -1260,6 +1332,7 @@ void iid_node_dependence_props::compute_node_counts_for_loading_loops( nodes_to_ child_loop_counts, ( *loop_props.chosen_loop_head ).node_id, loop_count, + loop_to_properties, iid_dependencies::random_nested_loop_counts ); } } @@ -1268,10 +1341,14 @@ void iid_node_dependence_props::compute_node_counts_for_loading_loops( nodes_to_ // ------------------------------------------------------------------------------------------------ void iid_node_dependence_props::compute_node_counts_for_loops( nodes_to_counts& path_counts, const equation& path, - const std::set< location_id::id_type >& loop_heads ) + const std::set< location_id::id_type >& loop_heads, + const loop_dependencies& loop_to_properties, + const std::unordered_set< location_id::id_type >& subset_ids ) { + // TMPROF_BLOCK(); + for ( const auto& props : std::ranges::views::reverse( loop_to_properties.loops ) ) { - if ( props.bodies.empty() || props.is_loading_loop ) { + if ( props.bodies.empty() || props.is_loading_loop || !subset_ids.contains( props.get_smallest_loop_head_id() ) ) { continue; } @@ -1296,25 +1373,25 @@ void iid_node_dependence_props::compute_node_counts_for_loops( nodes_to_counts& child_loop_counts, ( *props.chosen_loop_head ).node_id, non_loop_child_max_count, + loop_to_properties, iid_dependencies::random_nested_loop_counts ); } } // ------------------------------------------------------------------------------------------------ nodes_to_counts iid_node_dependence_props::compute_node_counts( const equation& path, - std::set< node_id_with_direction > const& computation_subset ) + std::set< node_id_with_direction > const& computation_subset, + const loop_dependencies& loop_to_properties, + const std::unordered_set< location_id::id_type >& subset_ids ) { + // TMPROF_BLOCK(); + nodes_to_counts path_counts; std::vector< node_id_with_direction > leafs = std::vector< node_id_with_direction >( computation_subset.begin(), computation_subset.end() ); INVARIANT( leafs.size() == path.values.size() ); - std::set< location_id::id_type > loop_heads = get_loop_heads( false ); - - for ( auto& loop : loop_to_properties.loops ) { - loop.set_chosen_loop_head(); - } for ( int i = 0; i < leafs.size(); ++i ) { auto& [ left_count, right_count ] = path_counts[ leafs[ i ].node_id ]; @@ -1326,7 +1403,7 @@ nodes_to_counts iid_node_dependence_props::compute_node_counts( const equation& } for ( const auto& loop_props : std::ranges::views::reverse( loop_to_properties.loops ) ) { - if ( loop_props.bodies.empty() ) { + if ( loop_props.bodies.empty() || !subset_ids.contains( loop_props.get_smallest_loop_head_id() ) ) { continue; } @@ -1347,9 +1424,10 @@ nodes_to_counts iid_node_dependence_props::compute_node_counts( const equation& } } - compute_node_counts_for_loops( path_counts, path, loop_heads ); - loop_heads = get_loop_heads( true ); - compute_node_counts_for_loading_loops( path_counts, path, loop_heads ); + std::set< location_id::id_type > loop_heads = loop_to_properties.get_loop_heads( false ); + compute_node_counts_for_loops( path_counts, path, loop_heads, loop_to_properties, subset_ids ); + loop_heads = loop_to_properties.get_loop_heads( true ); + compute_node_counts_for_loading_loops( path_counts, path, loop_heads, loop_to_properties ); return path_counts; } @@ -1361,6 +1439,8 @@ std::vector< equation > iid_node_dependence_props::compute_best_vectors( const s int desired_direction, float biggest_branching_value ) { + // TMPROF_BLOCK(); + if ( vectors_with_hits.empty() ) { throw std::invalid_argument( "Input map is empty." ); } @@ -1474,81 +1554,6 @@ std::vector< equation > iid_node_dependence_props::get_random_vector( const std: return selected_equations; } -// ------------------------------------------------------------------------------------------------ -std::set< node_id_with_direction > iid_node_dependence_props::get_node_subsets_for_computation() -{ - std::set< node_id_with_direction > computation_subset; - auto loop_heads = get_loop_heads( false ); - - for ( const auto& loop : loop_to_properties.loops ) { - if ( loop.bodies.empty() || loop.is_loading_loop ) { - continue; - } - - for ( const auto& body : loop.bodies ) { - location_id::id_type body_id = body.node_id; - if ( !loop_heads.contains( body_id ) ) { - computation_subset.insert( body ); - } - } - - for ( const auto& [ head, _ ] : loop.heads ) { - computation_subset.insert( { head.node_id, !head.branching_direction } ); - } - } - - return computation_subset; -} - -// ------------------------------------------------------------------------------------------------ -std::map< location_id::id_type, bool > -iid_node_dependence_props::get_loop_heads_ending( branching_node* end_node, - loop_head_to_bodies_t& loop_heads_to_bodies ) -{ - std::vector< fuzzer::loop_boundary_props > loops; - fuzzer::detect_loops_along_path_to_node( end_node, loop_heads_to_bodies, &loops ); - - std::map< location_id::id_type, bool > loop_heads_ending; - - auto is_outside_loop = [ & ]( branching_node* successor, - location_id loop_head_id, - const std::unordered_set< location_id >& loop_bodies ) { - if ( successor == nullptr ) { - return false; - } - - if ( successor->get_location_id() == loop_head_id ) { - return false; - } - - for ( const auto& bodies : loop_bodies ) { - if ( bodies.id == successor->get_location_id().id ) { - return false; - } - } - - return true; - }; - - for ( const auto& loop : loops ) { - location_id loop_head_id = loop.exit->get_location_id(); - - const auto& loop_bodies = loop_heads_to_bodies.at( loop_head_id ); - - branching_node* loop_end_node = loop.exit; - branching_node* left_successor = loop_end_node->successor( false ).pointer; - branching_node* right_successor = loop_end_node->successor( true ).pointer; - - if ( is_outside_loop( left_successor, loop_head_id, loop_bodies ) ) { - loop_heads_ending[ loop_end_node->get_location_id().id ] = false; - } else if ( is_outside_loop( right_successor, loop_head_id, loop_bodies ) ) { - loop_heads_ending[ loop_end_node->get_location_id().id ] = true; - } - } - - return loop_heads_ending; -} - // ------------------------------------------------------------------------------------------------ void fuzzing::iid_node_dependence_props::compute_loading_loops( branching_node* end_node, const loop_head_to_bodies_t& loop_heads_to_bodies, @@ -1606,217 +1611,68 @@ void fuzzing::iid_node_dependence_props::compute_loading_loops( branching_node* } } + // ------------------------------------------------------------------------------------------------ -void iid_node_dependence_props::compute_dependencies_by_loading( branching_node* end_node, - const loop_head_to_bodies_t& loop_heads_to_bodies, - const loop_endings& loop_heads_ending ) +generated_path iid_node_dependence_props::generate_path_from_node_counts( const nodes_to_counts& path_counts, + const loop_dependencies& loop_to_properties ) { - loop_head_to_loaded_bits_counter loading_loops; - compute_loading_loops( end_node, loop_heads_to_bodies, loading_loops, loop_heads_ending ); - - branching_node* node = end_node; - - struct loading_body_props_tmp { - natural_32_bit min = std::numeric_limits< natural_32_bit >::max(); - natural_32_bit max = std::numeric_limits< natural_32_bit >::min(); - std::vector< natural_32_bit > sensitive_stdin_bit_counts; - }; - - std::map< location_id::id_type, std::map< location_id::id_type, loading_body_props_tmp > > loop_to_props; - - while ( node != nullptr ) { - location_id::id_type node_id = node->get_location_id().id; - - for ( const auto& [ loop_head, props ] : loading_loops ) { - if ( !loop_heads_ending.contains( loop_head ) ) { - continue; - } - - auto min = props.min; - auto max = props.max; - - auto it = std::find_if( node->sensitive_stdin_bits.begin(), - node->sensitive_stdin_bits.end(), - [ & ]( natural_32_bit bit_index ) { - return bit_index >= min && bit_index < max; - } ); - if ( it == node->sensitive_stdin_bits.end() ) - continue; + // TMPROF_BLOCK(); - loading_body_props_tmp& loop_props = loop_to_props[ loop_head ][ node_id ]; + std::map< location_id::id_type, node_props_in_path > path; + for ( const auto& [ id, counts ] : path_counts ) { + if ( counts.left_count == 0 && counts.right_count == 0 ) { + continue; + } - auto min_it = std::min_element( node->sensitive_stdin_bits.begin(), node->sensitive_stdin_bits.end() ); - if ( min_it != node->sensitive_stdin_bits.end() ) { - loop_props.min = std::min( loop_props.min, *min_it ); - } + bool loop_head_end_direction = false; + bool is_loop_head = false; - auto max_it = std::max_element( node->sensitive_stdin_bits.begin(), node->sensitive_stdin_bits.end() ); - if ( max_it != node->sensitive_stdin_bits.end() ) { - loop_props.max = std::max( loop_props.max, *max_it ); + for ( const auto& loop : loop_to_properties.loops ) { + for ( const auto& [ head, _ ] : loop.heads ) { + if ( head.node_id == id ) { + is_loop_head = true; + loop_head_end_direction = head.branching_direction; + } } - - loop_props.sensitive_stdin_bit_counts.push_back( node->sensitive_stdin_bits.size() ); } - node = node->predecessor; + node_props_in_path props = { counts, is_loop_head, loop_head_end_direction }; + path.emplace( id, props ); } - for ( const auto& [ loop_head_id, body ] : loop_to_props ) { - auto loading_props = loading_loops.at( loop_head_id ); - - natural_32_bit loaded_bits = loading_props.max - loading_props.min; - - if ( loading_props.loop_count == 0 || loaded_bits == 0 ) { - continue; - } - - double per_loop = double( loaded_bits ) / double( loading_props.loop_count ); - - loop_properties& dependencies = loop_to_properties.get_props_by_loop_head_id( loop_head_id ); - dependencies.is_loading_loop = true; - dependencies.loaded_bits_per_loop.add( per_loop ); - - bool loop_head_end_direction = loop_heads_ending.at( loop_head_id ); - - for ( const auto& [ body_id, props ] : body ) { - auto& body_props = dependencies.bits_read_by_node[ body_id ]; - natural_32_bit minimal_offset = props.min - loading_props.min; - INVARIANT( minimal_offset >= 0 ); - body_props.minimal_bit_offset = std::min( body_props.minimal_bit_offset, minimal_offset ); - - for ( const auto& count : props.sensitive_stdin_bit_counts ) { - body_props.average_bits_read.add( count ); - } - - for ( bool direction : { true, false } ) { - node_id_with_direction node_id_direction = { body_id, direction }; + return generated_path( path ); +} - if ( matrix.contains( node_id_direction ) ) { - dependencies.nodes_dependent_by_loading.insert( node_id_direction ); - } +// iid_dependencies +// ------------------------------------------------------------------------------------------------ +void iid_dependencies::update_ignored_nodes( sensitivity_analysis& sensitivity ) +{ + for ( branching_node* node : sensitivity.get_changed_nodes() ) { + if ( node->is_did_branching() ) { + location_id::id_type location_id = node->get_location_id().id; + if ( ignored_node_ids.insert( location_id ).second ) { + node_id_to_equation_map.erase( location_id ); } } } } - // ------------------------------------------------------------------------------------------------ -void iid_node_dependence_props::compute_dependencies_by_loops( const loop_head_to_bodies_t& loop_heads_to_bodies, - const loop_endings& loop_heads_ending ) +void iid_dependencies::process_node_dependence_from_full_path( branching_node* end_node ) { - for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { - location_id::id_type loop_head_id = loop_head.id; - if ( !loop_heads_ending.contains( loop_head_id ) ) { - continue; - } - - bool loop_head_end_direction = loop_heads_ending.at( loop_head_id ); - node_id_with_direction loop_head_direction = { loop_head_id, loop_head_end_direction }; + TMPROF_BLOCK(); - std::set< location_id::id_type > all_ids; - all_ids.insert( loop_head_id ); - for ( const auto& loop_body_id : loop_bodies ) { - all_ids.insert( loop_body_id.id ); - } + loop_head_to_bodies_t loop_heads_to_bodies; + loop_endings loop_heads_ending = get_loop_heads_ending( end_node, loop_heads_to_bodies ); - loop_properties& props = loop_to_properties.get_props( all_ids, loop_head_id ); + compute_dependencies_by_loops( loop_heads_to_bodies, loop_heads_ending ); + compute_dependencies_by_loading( end_node, loop_heads_to_bodies, loop_heads_ending ); - props.heads[ loop_head_direction ].count++; - - for ( const auto& body : loop_bodies ) { - for ( bool direction : { true, false } ) { - node_id_with_direction node_id_direction = { body.id, direction }; - - for ( const auto& [ head, _ ] : props.heads ) { - if ( head.node_id == body.id ) { - continue; - } - } - - if ( matrix.contains( node_id_direction ) ) { - props.bodies.insert( node_id_direction ); - } - } - } - } - - loop_to_properties.merge_properties(); - - std::sort( loop_to_properties.loops.begin(), loop_to_properties.loops.end(), []( const auto& a, const auto& b ) { - return a.get_smallest_loop_head_id() < b.get_smallest_loop_head_id(); - } ); -} - -// ------------------------------------------------------------------------------------------------ -generated_path iid_node_dependence_props::generate_path_from_node_counts( const nodes_to_counts& path_counts ) -{ - std::map< location_id::id_type, node_props_in_path > path; - for ( const auto& [ id, counts ] : path_counts ) { - if ( counts.left_count == 0 && counts.right_count == 0 ) { - continue; - } - - bool loop_head_end_direction = false; - bool is_loop_head = false; - - for ( const auto& loop : loop_to_properties.loops ) { - for ( const auto& [ head, _ ] : loop.heads ) { - if ( head.node_id == id ) { - is_loop_head = true; - loop_head_end_direction = head.branching_direction; - } - } - } - - node_props_in_path props = { counts, is_loop_head, loop_head_end_direction }; - path.emplace( id, props ); - } - - return generated_path( path ); -} - -// ------------------------------------------------------------------------------------------------ -std::set< location_id::id_type > iid_node_dependence_props::get_loop_heads( bool include_loading_loops ) -{ - std::set< location_id::id_type > loop_heads; - for ( const auto& props : loop_to_properties.loops ) { - if ( props.is_loading_loop && !include_loading_loops ) { - continue; - } - - for ( const auto& [ head, _ ] : props.heads ) { - loop_heads.insert( head.node_id ); - } - } - - return loop_heads; -} - -// iid_dependencies -// ------------------------------------------------------------------------------------------------ -void iid_dependencies::update_ignored_nodes( sensitivity_analysis& sensitivity ) -{ - for ( branching_node* node : sensitivity.get_changed_nodes() ) { - if ( node->is_did_branching() ) { - location_id::id_type location_id = node->get_location_id().id; - if ( ignored_node_ids.insert( location_id ).second ) { - node_id_to_equation_map.erase( location_id ); - } - } + for ( auto& loop : loop_to_properties.loops ) { + loop.set_chosen_loop_head(); } -} -// ------------------------------------------------------------------------------------------------ -void iid_dependencies::process_node_dependence( branching_node* node ) -{ - TMPROF_BLOCK(); - - if ( ignored_node_ids.contains( node->get_location_id().id ) ) - return; - - iid_node_dependence_props& props = node_id_to_equation_map[ node->get_location_id().id ]; - props.process_node( node ); - processed_nodes++; + compute_paths( end_node ); } // ------------------------------------------------------------------------------------------------ @@ -1828,11 +1684,6 @@ void iid_dependencies::remove_node_dependence( location_id::id_type id ) stats.state = generation_state::STATE_COVERED; ignored_node_ids.insert( id ); - if ( it != node_id_to_equation_map.end() ) { - auto next_it = std::next( it ); - next_it->second.combine_props( it->second ); - } - if ( iid_dependencies::generate_more_data_after_coverage && !it->second.is_equal_branching_predicate() ) { stats.state = generation_state::STATE_GENERATION_MORE; int max_generation_after_covered = std::max( iid_dependencies::minimal_max_generation_after_covered, @@ -1851,7 +1702,8 @@ void fuzzing::iid_dependencies::remove_all_covered( const std::unordered_set< lo } for ( auto it = node_id_to_equation_map.begin(); it != node_id_to_equation_map.end(); ) { - if ( covered_ids.contains( it->first ) && it->second.get_generations_stats().state == generation_state::STATE_NOT_COVERED ) { + if ( covered_ids.contains( it->first ) && + it->second.get_generations_stats().state == generation_state::STATE_NOT_COVERED ) { ignored_node_ids.insert( it->first ); it = node_id_to_equation_map.erase( it ); } else { @@ -1913,6 +1765,22 @@ std::optional< location_id::id_type > iid_dependencies::get_next_iid_node() return best_id; } +// ------------------------------------------------------------------------------------------------ +generated_path fuzzing::iid_dependencies::generate_probabilities() +{ + std::optional< location_id::id_type > id = get_next_iid_node(); + if ( !id.has_value() ) { + return {}; + } + + if ( verbose ) { + std::cout << "Generating probabilities for node: " << *id << std::endl; + } + + iid_node_dependence_props& props = get_props( id.value() ); + generated_path path = props.generate_probabilities( loop_to_properties ); + return path; +} // ------------------------------------------------------------------------------------------------ iid_vector_analysis_statistics fuzzing::iid_dependencies::get_stats() const @@ -1922,7 +1790,7 @@ iid_vector_analysis_statistics fuzzing::iid_dependencies::get_stats() const iid_vector_analysis_statistics_per_node node_stats; for ( const auto& [ id, props ] : node_id_to_equation_map ) { node_stats.generation_stats = props.get_generations_stats(); - node_stats.loop_to_properties = props.get_dependencies_by_loops(); + node_stats.loop_to_properties = {}; stats.iid_nodes_stats[ id ] = node_stats; } @@ -1934,10 +1802,303 @@ iid_vector_analysis_statistics fuzzing::iid_dependencies::get_stats() const return stats; } +// ------------------------------------------------------------------------------------------------ +loop_endings fuzzing::iid_dependencies::get_loop_heads_ending( branching_node* end_node, + loop_head_to_bodies_t& loop_heads_to_bodies ) +{ + TMPROF_BLOCK(); + + std::vector< fuzzer::loop_boundary_props > loops; + fuzzer::detect_loops_along_path_to_node( end_node, loop_heads_to_bodies, &loops ); + + loop_endings loop_heads_ending; + + auto is_outside_loop = [ & ]( branching_node* successor, + location_id loop_head_id, + const std::unordered_set< location_id >& loop_bodies ) { + if ( successor == nullptr ) { + return false; + } + + if ( successor->get_location_id() == loop_head_id ) { + return false; + } + + return !loop_bodies.contains( successor->get_location_id().id ); + }; + + for ( const auto& loop : loops ) { + location_id loop_head_id = loop.exit->get_location_id(); + + const auto& loop_bodies = loop_heads_to_bodies.at( loop_head_id ); + + branching_node* left_successor = loop.exit->successor( false ).pointer; + branching_node* right_successor = loop.exit->successor( true ).pointer; + + if ( is_outside_loop( left_successor, loop_head_id, loop_bodies ) ) { + loop_heads_ending[ loop.exit->get_location_id().id ] = false; + } else if ( is_outside_loop( right_successor, loop_head_id, loop_bodies ) ) { + loop_heads_ending[ loop.exit->get_location_id().id ] = true; + } + } + + return loop_heads_ending; +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::iid_dependencies::compute_dependencies_by_loading( branching_node* end_node, + const loop_head_to_bodies_t& loop_heads_to_bodies, + const loop_endings& loop_heads_ending ) +{ + TMPROF_BLOCK(); + + loop_head_to_loaded_bits_counter loading_loops; + compute_loading_loops( end_node, loop_heads_to_bodies, loading_loops, loop_heads_ending ); + + branching_node* node = end_node; + + struct loading_body_props_tmp { + natural_32_bit min = std::numeric_limits< natural_32_bit >::max(); + natural_32_bit max = std::numeric_limits< natural_32_bit >::min(); + std::vector< natural_32_bit > sensitive_stdin_bit_counts; + }; + + std::map< location_id::id_type, std::map< location_id::id_type, loading_body_props_tmp > > loop_to_props; + + while ( node != nullptr ) { + location_id::id_type node_id = node->get_location_id().id; + + for ( const auto& [ loop_head, props ] : loading_loops ) { + if ( !loop_heads_ending.contains( loop_head ) ) { + continue; + } + + auto min = props.min; + auto max = props.max; + + auto it = std::find_if( node->sensitive_stdin_bits.begin(), + node->sensitive_stdin_bits.end(), + [ & ]( natural_32_bit bit_index ) { + return bit_index >= min && bit_index < max; + } ); + if ( it == node->sensitive_stdin_bits.end() ) + continue; + + loading_body_props_tmp& loop_props = loop_to_props[ loop_head ][ node_id ]; + + auto min_it = std::min_element( node->sensitive_stdin_bits.begin(), node->sensitive_stdin_bits.end() ); + if ( min_it != node->sensitive_stdin_bits.end() ) { + loop_props.min = std::min( loop_props.min, *min_it ); + } + + auto max_it = std::max_element( node->sensitive_stdin_bits.begin(), node->sensitive_stdin_bits.end() ); + if ( max_it != node->sensitive_stdin_bits.end() ) { + loop_props.max = std::max( loop_props.max, *max_it ); + } + + loop_props.sensitive_stdin_bit_counts.push_back( node->sensitive_stdin_bits.size() ); + } + + node = node->predecessor; + } + + for ( const auto& [ loop_head_id, body ] : loop_to_props ) { + auto loading_props = loading_loops.at( loop_head_id ); + + natural_32_bit loaded_bits = loading_props.max - loading_props.min; + + if ( loading_props.loop_count == 0 || loaded_bits == 0 ) { + continue; + } + + double per_loop = double( loaded_bits ) / double( loading_props.loop_count ); + + loop_properties& dependencies = loop_to_properties.get_props_by_loop_head_id( loop_head_id ); + dependencies.is_loading_loop = true; + dependencies.loaded_bits_per_loop.add( per_loop ); + + bool loop_head_end_direction = loop_heads_ending.at( loop_head_id ); + + for ( const auto& [ body_id, props ] : body ) { + auto& body_props = dependencies.bits_read_by_node[ body_id ]; + natural_32_bit minimal_offset = props.min - loading_props.min; + INVARIANT( minimal_offset >= 0 ); + body_props.minimal_bit_offset = std::min( body_props.minimal_bit_offset, minimal_offset ); + + for ( const auto& count : props.sensitive_stdin_bit_counts ) { + body_props.average_bits_read.add( count ); + } + + for ( bool direction : { true, false } ) { + node_id_with_direction node_id_direction = { body_id, direction }; + dependencies.nodes_dependent_by_loading.insert( node_id_direction ); + } + } + } +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::iid_dependencies::compute_dependencies_by_loops( const loop_head_to_bodies_t& loop_heads_to_bodies, + const loop_endings& loop_heads_ending ) +{ + TMPROF_BLOCK(); + + bool changed = false; + + for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { + location_id::id_type loop_head_id = loop_head.id; + + auto loop_heads_ending_it = loop_heads_ending.find( loop_head_id ); + if ( loop_heads_ending_it == loop_heads_ending.end() ) { + continue; + } + bool loop_head_end_direction = loop_heads_ending_it->second; + + node_id_with_direction loop_head_direction = { loop_head_id, loop_head_end_direction }; + + std::unordered_set< location_id::id_type > all_ids; + + { + + all_ids.reserve( loop_bodies.size() + 1 ); + all_ids.insert( loop_head_id ); + + std::vector< location_id::id_type > temp_ids; + temp_ids.reserve( loop_bodies.size() ); + std::transform( loop_bodies.begin(), + loop_bodies.end(), + std::back_inserter( temp_ids ), + []( const auto& loop_body ) { return loop_body.id; } ); + + all_ids.insert( temp_ids.begin(), temp_ids.end() ); + } + + loop_properties& props = loop_to_properties.get_props( all_ids, loop_head_id ); + + auto [ props_heads_it, inserted ] = props.heads.insert( { loop_head_direction, {} } ); + props_heads_it->second.count++; + changed |= inserted; + + const std::unordered_set< location_id::id_type >& all_props_ids = props.get_all_ids(); + + for ( const auto& body : loop_bodies ) { + if ( all_props_ids.contains( body.id ) ) { + continue; + } + + for ( bool direction : { true, false } ) { + // TMPROF_BLOCK(); + auto [ it, inserted ] = props.bodies.emplace( body.id, direction ); + changed |= inserted; + } + } + } + + if ( changed ) { + loop_to_properties.merge_properties(); + + std::sort( loop_to_properties.loops.begin(), + loop_to_properties.loops.end(), + []( const auto& a, const auto& b ) { + return a.get_smallest_loop_head_id() < b.get_smallest_loop_head_id(); + } ); + } +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::iid_dependencies::compute_loading_loops( branching_node* end_node, + const loop_head_to_bodies_t& loop_heads_to_bodies, + loop_head_to_loaded_bits_counter& loading_loops, + const loop_endings& loop_heads_ending ) +{ + for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { + loading_loops[ loop_head.id ] = { std::numeric_limits< natural_32_bit >::max(), + std::numeric_limits< natural_32_bit >::min() }; + } + + branching_node* node = end_node; + branching_node* prev_node = nullptr; + + while ( node != nullptr ) { + for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { + if ( loop_head.id == node->get_location_id().id ) { + natural_32_bit bits_count = node->get_num_stdin_bits(); + + auto& props = loading_loops[ loop_head.id ]; + props.min = std::min( props.min, bits_count ); + props.max = std::max( props.max, bits_count ); + props.loop_count++; + } + } + + // Remove one loop count for branching that ends the loop + prev_node = node->predecessor; + if ( prev_node != nullptr ) { + bool node_direction = prev_node->successor( true ).pointer == node; + + for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { + auto it = loop_heads_ending.find( loop_head.id ); + if ( it == loop_heads_ending.end() ) { + continue; + } + + if ( loop_head.id == prev_node->get_location_id().id && it->second == node_direction ) { + auto& props = loading_loops[ loop_head.id ]; + props.loop_count--; + } + } + } + + node = prev_node; + } + + // Remove all loops that did not load any data inside + for ( auto it = loading_loops.begin(); it != loading_loops.end(); ) { + if ( it->second.min == it->second.max ) { + it = loading_loops.erase( it ); + } else { + ++it; + } + } +} + +// ------------------------------------------------------------------------------------------------ +void fuzzing::iid_dependencies::compute_paths( branching_node* end_node ) +{ + TMPROF_BLOCK(); + + INVARIANT( end_node != nullptr ); + + branching_node* current_node = end_node; + while ( current_node->predecessor != nullptr ) { + current_node = current_node->predecessor; + } + + std::vector< node_id_with_direction > full_path = get_path( end_node ); + std::unordered_map< node_id_with_direction, int > directions_in_path; + + for ( auto it = full_path.rbegin(); it != full_path.rend(); ++it ) { + const auto& path_node = *it; + location_id::id_type node_id = path_node.node_id; + + directions_in_path[ path_node ]++; + current_node = current_node->successor( path_node.branching_direction ).pointer; + + if ( ignored_node_ids.contains( current_node->get_location_id().id ) ) { + continue; + } + + iid_node_dependence_props& props = node_id_to_equation_map[ current_node->get_location_id().id ]; + props.process_path_effective( current_node, directions_in_path ); + } +} + // non member functions // ------------------------------------------------------------------------------------------------ std::vector< node_id_with_direction > get_path( branching_node* node ) { + // TMPROF_BLOCK(); + std::vector< node_id_with_direction > result; branching_node* current = node; @@ -1954,6 +2115,29 @@ std::vector< node_id_with_direction > get_path( branching_node* node ) return result; } +// ------------------------------------------------------------------------------------------------ +std::unordered_map< node_id_with_direction, int > get_directions_in_path( branching_node* node ) +{ + // TMPROF_BLOCK(); + + std::unordered_map< node_id_with_direction, int > result; + + branching_node* current = node; + while ( current != nullptr ) { + branching_node* predecessor = current->predecessor; + if ( predecessor != nullptr ) { + node_id_with_direction nav = { predecessor->get_location_id().id, + predecessor->successor_direction( current ) }; + result[ nav ]++; + } + + current = predecessor; + } + + return result; +} + +// ------------------------------------------------------------------------------------------------ bool should_generate_more_data( const generation_state& state ) { return state == generation_state::STATE_GENERATION_MORE || From 65abb9b5e592d2898808d3ba3b40370a123bdf1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Thu, 6 Mar 2025 09:34:08 +0100 Subject: [PATCH 119/144] feat: make computation of loading loops more efficient --- .../include/fuzzing/iid_vector_analysis.hpp | 11 +- src/fuzzing/src/iid_vector_analysis.cpp | 205 +++++++----------- 2 files changed, 83 insertions(+), 133 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index 7dff9b31..4d5f7a9a 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -119,12 +119,13 @@ struct loop_dependencies { struct iid_vector_analysis_statistics_per_node { iid_node_generations_stats generation_stats; - loop_dependencies loop_to_properties; + std::vector< location_id::id_type > node_ids; }; struct iid_vector_analysis_statistics { std::map< location_id::id_type, iid_vector_analysis_statistics_per_node > iid_nodes_stats; std::vector< location_id::id_type > ignored_node_ids; + loop_dependencies loop_to_properties; int processed_nodes; }; @@ -248,12 +249,10 @@ using nodes_to_counts = std::map< location_id::id_type, node_counts >; struct equation_matrix { equation_matrix get_submatrix( std::set< node_id_with_direction > const& subset, bool unique ) const; - void process_node( branching_node* end_node, const std::vector< node_id_with_direction >& path ); void process_node_effective( branching_node* end_node, bool compute_matrix, const std::unordered_map< node_id_with_direction, int >& directions_in_path ); void start_compute_matrix(); - void add_equation( branching_node* end_node, const std::vector< node_id_with_direction >& path ); bool contains( node_id_with_direction const& node ) const; std::pair< std::size_t, std::size_t > get_dimensions() const; std::map< equation, int > compute_vectors_with_hits(); @@ -281,15 +280,11 @@ struct equation_matrix { struct iid_node_dependence_props { generated_path generate_probabilities( const loop_dependencies& loop_to_properties ); - void process_node( branching_node* end_node ); - void process_path( branching_node* end_node, - const std::set< location_id::id_type >& seen_ids, - const std::vector< node_id_with_direction >& path ); void process_path_effective( branching_node* end_node, const std::unordered_map< node_id_with_direction, int >& directions_in_path ); iid_node_generations_stats& get_generations_stats() { return stats; } + const equation_matrix& get_matrix() const { return matrix; } const iid_node_generations_stats& get_generations_stats() const { return stats; } - // const loop_dependencies& get_dependencies_by_loops() const { return loop_to_properties; } bool should_generate() const; bool too_much_failed_in_row( int max_failed_generations_in_row ) const; diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 8c87a05e..05e0ee4c 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -237,7 +237,10 @@ int equation::get_one_way_branching_count() const int equation::get_biggest_value() const { return *std::max_element( values.begin(), values.end() ); } // ------------------------------------------------------------------------------------------------ -const std::unordered_set< location_id::id_type >& fuzzing::equation_matrix::get_node_ids() const { return node_ids; } +const std::unordered_set< location_id::id_type >& fuzzing::equation_matrix::get_node_ids() const +{ + return node_ids; +} // ------------------------------------------------------------------------------------------------ bool equation::is_any_negative() const @@ -905,7 +908,6 @@ generated_path iid_node_dependence_props::generate_probabilities( const loop_dep stats.generation_starts++; equation_matrix submatrix = matrix.get_submatrix( computation_subset, true ); - { // print_stats( true ); // loop_to_properties.print_dependencies(); @@ -1348,7 +1350,8 @@ void iid_node_dependence_props::compute_node_counts_for_loops( nodes_to_counts& // TMPROF_BLOCK(); for ( const auto& props : std::ranges::views::reverse( loop_to_properties.loops ) ) { - if ( props.bodies.empty() || props.is_loading_loop || !subset_ids.contains( props.get_smallest_loop_head_id() ) ) { + if ( props.bodies.empty() || props.is_loading_loop || + !subset_ids.contains( props.get_smallest_loop_head_id() ) ) { continue; } @@ -1379,10 +1382,11 @@ void iid_node_dependence_props::compute_node_counts_for_loops( nodes_to_counts& } // ------------------------------------------------------------------------------------------------ -nodes_to_counts iid_node_dependence_props::compute_node_counts( const equation& path, - std::set< node_id_with_direction > const& computation_subset, - const loop_dependencies& loop_to_properties, - const std::unordered_set< location_id::id_type >& subset_ids ) +nodes_to_counts +iid_node_dependence_props::compute_node_counts( const equation& path, + std::set< node_id_with_direction > const& computation_subset, + const loop_dependencies& loop_to_properties, + const std::unordered_set< location_id::id_type >& subset_ids ) { // TMPROF_BLOCK(); @@ -1554,63 +1558,6 @@ std::vector< equation > iid_node_dependence_props::get_random_vector( const std: return selected_equations; } -// ------------------------------------------------------------------------------------------------ -void fuzzing::iid_node_dependence_props::compute_loading_loops( branching_node* end_node, - const loop_head_to_bodies_t& loop_heads_to_bodies, - loop_head_to_loaded_bits_counter& loading_loops, - const loop_endings& loop_heads_ending ) -{ - for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { - loading_loops[ loop_head.id ] = { std::numeric_limits< natural_32_bit >::max(), - std::numeric_limits< natural_32_bit >::min() }; - } - - branching_node* node = end_node; - branching_node* prev_node = nullptr; - - while ( node != nullptr ) { - for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { - if ( loop_head.id == node->get_location_id().id ) { - natural_32_bit bits_count = node->get_num_stdin_bits(); - - auto& props = loading_loops[ loop_head.id ]; - props.min = std::min( props.min, bits_count ); - props.max = std::max( props.max, bits_count ); - props.loop_count++; - } - } - - // Remove one loop count for branching that ends the loop - prev_node = node->predecessor; - if ( prev_node != nullptr ) { - bool node_direction = prev_node->successor( true ).pointer == node; - - for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { - auto it = loop_heads_ending.find( loop_head.id ); - if ( it == loop_heads_ending.end() ) { - continue; - } - - if ( loop_head.id == prev_node->get_location_id().id && it->second == node_direction ) { - auto& props = loading_loops[ loop_head.id ]; - props.loop_count--; - } - } - } - - node = prev_node; - } - - // Remove all loops that did not load any data inside - for ( auto it = loading_loops.begin(); it != loading_loops.end(); ) { - if ( it->second.min == it->second.max ) { - it = loading_loops.erase( it ); - } else { - ++it; - } - } -} - // ------------------------------------------------------------------------------------------------ generated_path iid_node_dependence_props::generate_path_from_node_counts( const nodes_to_counts& path_counts, @@ -1787,17 +1734,20 @@ iid_vector_analysis_statistics fuzzing::iid_dependencies::get_stats() const { iid_vector_analysis_statistics stats; - iid_vector_analysis_statistics_per_node node_stats; for ( const auto& [ id, props ] : node_id_to_equation_map ) { + iid_vector_analysis_statistics_per_node node_stats; node_stats.generation_stats = props.get_generations_stats(); - node_stats.loop_to_properties = {}; + node_stats.node_ids = std::vector< location_id::id_type >( props.get_matrix().get_node_ids().begin(), + props.get_matrix().get_node_ids().end() ); + std::sort( node_stats.node_ids.begin(), node_stats.node_ids.end() ); stats.iid_nodes_stats[ id ] = node_stats; } stats.ignored_node_ids = std::vector< location_id::id_type >( ignored_node_ids.begin(), ignored_node_ids.end() ); - + std::sort( stats.ignored_node_ids.begin(), stats.ignored_node_ids.end() ); + stats.loop_to_properties = loop_to_properties; stats.processed_nodes = processed_nodes; return stats; } @@ -1809,6 +1759,7 @@ loop_endings fuzzing::iid_dependencies::get_loop_heads_ending( branching_node* e TMPROF_BLOCK(); std::vector< fuzzer::loop_boundary_props > loops; + // This is slow fuzzer::detect_loops_along_path_to_node( end_node, loop_heads_to_bodies, &loops ); loop_endings loop_heads_ending; @@ -1855,56 +1806,63 @@ void fuzzing::iid_dependencies::compute_dependencies_by_loading( branching_node* loop_head_to_loaded_bits_counter loading_loops; compute_loading_loops( end_node, loop_heads_to_bodies, loading_loops, loop_heads_ending ); - branching_node* node = end_node; - - struct loading_body_props_tmp { + struct dependent_body_props { natural_32_bit min = std::numeric_limits< natural_32_bit >::max(); natural_32_bit max = std::numeric_limits< natural_32_bit >::min(); std::vector< natural_32_bit > sensitive_stdin_bit_counts; }; - std::map< location_id::id_type, std::map< location_id::id_type, loading_body_props_tmp > > loop_to_props; + std::unordered_map< location_id::id_type, std::unordered_map< location_id::id_type, dependent_body_props > > loop_head_to_props; + std::unordered_map< location_id::id_type, dependent_body_props > node_id_to_props; + + branching_node* node = end_node; while ( node != nullptr ) { - location_id::id_type node_id = node->get_location_id().id; + const auto node_id = node->get_location_id().id; + const auto& sensitive_bits = node->sensitive_stdin_bits; + if ( sensitive_bits.empty() ) { + node = node->predecessor; + continue; + } - for ( const auto& [ loop_head, props ] : loading_loops ) { - if ( !loop_heads_ending.contains( loop_head ) ) { - continue; - } + auto& props = node_id_to_props[ node_id ]; + auto min_it = std::min_element( sensitive_bits.begin(), sensitive_bits.end() ); + auto max_it = std::max_element( sensitive_bits.begin(), sensitive_bits.end() ); + natural_32_bit node_min = ( min_it != sensitive_bits.end() ) ? + *min_it : + std::numeric_limits< natural_32_bit >::max(); + natural_32_bit node_max = ( max_it != sensitive_bits.end() ) ? + *max_it : + std::numeric_limits< natural_32_bit >::min(); - auto min = props.min; - auto max = props.max; + props.min = std::min( props.min, node_min ); + props.max = std::max( props.max, node_max ); + props.sensitive_stdin_bit_counts.push_back( sensitive_bits.size() ); - auto it = std::find_if( node->sensitive_stdin_bits.begin(), - node->sensitive_stdin_bits.end(), - [ & ]( natural_32_bit bit_index ) { - return bit_index >= min && bit_index < max; - } ); - if ( it == node->sensitive_stdin_bits.end() ) - continue; + node = node->predecessor; + } - loading_body_props_tmp& loop_props = loop_to_props[ loop_head ][ node_id ]; + for ( const auto& [ node_id, props ] : node_id_to_props ) { + const auto& [ node_min, node_max, sensitive_counts ] = props; - auto min_it = std::min_element( node->sensitive_stdin_bits.begin(), node->sensitive_stdin_bits.end() ); - if ( min_it != node->sensitive_stdin_bits.end() ) { - loop_props.min = std::min( loop_props.min, *min_it ); + for ( const auto& [ loop_head, loop_props ] : loading_loops ) { + if ( !loop_heads_ending.contains( loop_head ) ) { + continue; } - auto max_it = std::max_element( node->sensitive_stdin_bits.begin(), node->sensitive_stdin_bits.end() ); - if ( max_it != node->sensitive_stdin_bits.end() ) { - loop_props.max = std::max( loop_props.max, *max_it ); + if ( node_max >= loop_props.min && node_min < loop_props.max ) { + auto& loop_body_props = loop_head_to_props[ loop_head ][ node_id ]; + loop_body_props.min = std::min( loop_body_props.min, node_min ); + loop_body_props.max = std::max( loop_body_props.max, node_max ); + loop_body_props.sensitive_stdin_bit_counts.insert( loop_body_props.sensitive_stdin_bit_counts.end(), + sensitive_counts.begin(), + sensitive_counts.end() ); } - - loop_props.sensitive_stdin_bit_counts.push_back( node->sensitive_stdin_bits.size() ); } - - node = node->predecessor; } - for ( const auto& [ loop_head_id, body ] : loop_to_props ) { - auto loading_props = loading_loops.at( loop_head_id ); - + for ( const auto& [ loop_head_id, body ] : loop_head_to_props ) { + const auto& loading_props = loading_loops.at( loop_head_id ); natural_32_bit loaded_bits = loading_props.max - loading_props.min; if ( loading_props.loop_count == 0 || loaded_bits == 0 ) { @@ -2011,45 +1969,41 @@ void fuzzing::iid_dependencies::compute_loading_loops( branching_node* end_node, loop_head_to_loaded_bits_counter& loading_loops, const loop_endings& loop_heads_ending ) { + // TMPROF_BLOCK(); + for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { loading_loops[ loop_head.id ] = { std::numeric_limits< natural_32_bit >::max(), - std::numeric_limits< natural_32_bit >::min() }; + std::numeric_limits< natural_32_bit >::min(), + 0 }; } branching_node* node = end_node; - branching_node* prev_node = nullptr; - while ( node != nullptr ) { - for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { - if ( loop_head.id == node->get_location_id().id ) { - natural_32_bit bits_count = node->get_num_stdin_bits(); - - auto& props = loading_loops[ loop_head.id ]; - props.min = std::min( props.min, bits_count ); - props.max = std::max( props.max, bits_count ); - props.loop_count++; - } - } + const auto node_id = node->get_location_id().id; - // Remove one loop count for branching that ends the loop - prev_node = node->predecessor; - if ( prev_node != nullptr ) { - bool node_direction = prev_node->successor( true ).pointer == node; + if ( auto it = loop_heads_to_bodies.find( node->get_location_id() ); it != loop_heads_to_bodies.end() ) { + const auto& loop_head = it->first; + auto& props = loading_loops[ loop_head.id ]; - for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { - auto it = loop_heads_ending.find( loop_head.id ); - if ( it == loop_heads_ending.end() ) { - continue; - } + natural_32_bit bits_count = node->get_num_stdin_bits(); + props.min = std::min( props.min, bits_count ); + props.max = std::max( props.max, bits_count ); + props.loop_count++; + } + + if ( node->predecessor != nullptr ) { + branching_node* predecessor = node->predecessor; + const auto predecessor_id = predecessor->get_location_id().id; - if ( loop_head.id == prev_node->get_location_id().id && it->second == node_direction ) { - auto& props = loading_loops[ loop_head.id ]; - props.loop_count--; + if ( loop_heads_ending.contains( predecessor_id ) ) { + bool node_direction = predecessor->successor( true ).pointer == node; + if ( loop_heads_ending.at( predecessor_id ) == node_direction ) { + loading_loops[ predecessor_id ].loop_count--; } } } - node = prev_node; + node = node->predecessor; } // Remove all loops that did not load any data inside @@ -2078,6 +2032,7 @@ void fuzzing::iid_dependencies::compute_paths( branching_node* end_node ) std::unordered_map< node_id_with_direction, int > directions_in_path; for ( auto it = full_path.rbegin(); it != full_path.rend(); ++it ) { + processed_nodes++; const auto& path_node = *it; location_id::id_type node_id = path_node.node_id; From 2301ad2cb44c19f5a9cbce14d4f60a44cf6e8bc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Tue, 11 Mar 2025 17:53:14 +0100 Subject: [PATCH 120/144] feat: change computation to start only if iid is present --- .../include/fuzzing/iid_vector_analysis.hpp | 10 ++- src/fuzzing/src/fuzzer.cpp | 8 +- src/fuzzing/src/iid_vector_analysis.cpp | 74 ++++++++++--------- 3 files changed, 50 insertions(+), 42 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index 4d5f7a9a..24a1273b 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -249,7 +249,7 @@ using nodes_to_counts = std::map< location_id::id_type, node_counts >; struct equation_matrix { equation_matrix get_submatrix( std::set< node_id_with_direction > const& subset, bool unique ) const; - void process_node_effective( branching_node* end_node, + void process_node( branching_node* end_node, bool compute_matrix, const std::unordered_map< node_id_with_direction, int >& directions_in_path ); void start_compute_matrix(); @@ -258,7 +258,6 @@ struct equation_matrix { std::map< equation, int > compute_vectors_with_hits(); std::vector< equation >& get_matrix(); std::optional< equation > get_new_subset_counts_from_vectors( const std::vector< equation >& vector, - int generated_after_covered, const iid_node_generations_stats& state ); int get_desired_vector_direction() const; float get_biggest_branching_value() const; @@ -344,7 +343,7 @@ struct iid_node_dependence_props { equation_matrix matrix; iid_node_generations_stats stats; - bool continuously_gathering_data = false; + bool matrix_generated = false; }; struct iid_dependencies { @@ -379,6 +378,9 @@ struct iid_dependencies { loop_dependencies loop_to_properties; int processed_nodes = 0; + bool compute_data = false; + std::vector< branching_node* > end_nodes; + public: // Configurations inline static bool random_nested_loop_counts = false; @@ -391,7 +393,7 @@ struct iid_dependencies { inline static float percentage_to_add_to_path = 0.4; inline static bool create_artificial_data = true; - inline static bool verbose = false; + inline static bool verbose = true; }; diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index c1598134..6c02f3cc 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -11,7 +11,6 @@ #include constexpr bool use_vector_analysis = true; -constexpr bool use_only_probabilities = false; namespace fuzzing { @@ -745,7 +744,7 @@ branching_node* fuzzer::monte_carlo_search( branching_node* successor; while (true) { - if ( use_vector_analysis && !use_only_probabilities ) { + if ( use_vector_analysis ) { successor = monte_carlo_step_with_path( pivot, histogram, generators, location_miss_generator, path ); } else { successor = monte_carlo_step( pivot, histogram, generators, location_miss_generator ); @@ -1394,7 +1393,8 @@ void fuzzer::do_cleanup() update_close_flags_from(node); break; } - iid_dependences.update_ignored_nodes(sensitivity); + if ( use_vector_analysis ) + iid_dependences.update_ignored_nodes( sensitivity ); collect_iid_pivots_from_sensitivity_results(); break; case BITSHARE: @@ -1745,7 +1745,7 @@ branching_node* fuzzer::select_iid_coverage_target() entry_branching ); - if ( use_vector_analysis && !use_only_probabilities && !path.get_path().empty() ) { + if ( use_vector_analysis && !path.get_path().empty() ) { start_node = select_start_node_for_monte_carlo_search_with_vector( path, it_pivot->second.loop_boundaries, diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 05e0ee4c..43f50342 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -638,10 +638,9 @@ equation_matrix equation_matrix::get_submatrix( std::set< node_id_with_direction } // ------------------------------------------------------------------------------------------------ -void fuzzing::equation_matrix::process_node_effective( - branching_node* end_node, - bool compute_matrix, - const std::unordered_map< node_id_with_direction, int >& directions_in_path ) +void fuzzing::equation_matrix::process_node( branching_node* end_node, + bool compute_matrix, + const std::unordered_map< node_id_with_direction, int >& directions_in_path ) { // TMPROF_BLOCK(); @@ -749,7 +748,6 @@ float equation_matrix::get_biggest_branching_value() const // ------------------------------------------------------------------------------------------------ std::optional< equation > equation_matrix::get_new_subset_counts_from_vectors( const std::vector< equation >& vectors, - int generation_count_after_covered, const iid_node_generations_stats& stats ) { // TMPROF_BLOCK(); @@ -889,8 +887,8 @@ generated_path iid_node_dependence_props::generate_probabilities( const loop_dep stats.method_calls++; - if ( !continuously_gathering_data ) { - continuously_gathering_data = true; + if ( !matrix_generated ) { + matrix_generated = true; matrix.start_compute_matrix(); } @@ -926,15 +924,15 @@ generated_path iid_node_dependence_props::generate_probabilities( const loop_dep generate_vectors_if_not_enough_data( *best_vectors, submatrix ); } - std::optional< equation > new_subset_counts = - submatrix.get_new_subset_counts_from_vectors( *best_vectors, stats.generated_after_covered, stats ); + std::optional< equation > new_subset_counts = submatrix.get_new_subset_counts_from_vectors( *best_vectors, + stats ); if ( !new_subset_counts.has_value() ) { return return_empty_path(); } nodes_to_counts node_counts = - compute_node_counts( new_subset_counts.value(), computation_subset, loop_to_properties, subset_ids ); + compute_node_counts( *new_subset_counts, computation_subset, loop_to_properties, subset_ids ); generated_path path = generate_path_from_node_counts( node_counts, loop_to_properties ); if ( iid_dependencies::verbose ) { @@ -951,7 +949,7 @@ void fuzzing::iid_node_dependence_props::process_path_effective( const std::unordered_map< node_id_with_direction, int >& directions_in_path ) { // TMPROF_BLOCK(); - matrix.process_node_effective( end_node, continuously_gathering_data, directions_in_path ); + matrix.process_node( end_node, matrix_generated, directions_in_path ); } // ------------------------------------------------------------------------------------------------ @@ -1310,7 +1308,6 @@ void iid_node_dependence_props::compute_node_counts_for_loading_loops( nodes_to_ for ( const auto& body : loop_props.bodies ) { auto& [ left_count, right_count ] = path_counts[ body.node_id ]; - // loop_count = std::max( loop_count, left_count + right_count ); if ( loop_heads.contains( body.node_id ) ) { child_loop_counts[ body.node_id ] = std::max( left_count, right_count ); @@ -1392,11 +1389,9 @@ iid_node_dependence_props::compute_node_counts( const equation& path, nodes_to_counts path_counts; - std::vector< node_id_with_direction > leafs = - std::vector< node_id_with_direction >( computation_subset.begin(), computation_subset.end() ); + auto leafs = std::vector< node_id_with_direction >( computation_subset.begin(), computation_subset.end() ); INVARIANT( leafs.size() == path.values.size() ); - for ( int i = 0; i < leafs.size(); ++i ) { auto& [ left_count, right_count ] = path_counts[ leafs[ i ].node_id ]; if ( leafs[ i ].branching_direction ) { @@ -1609,6 +1604,11 @@ void iid_dependencies::process_node_dependence_from_full_path( branching_node* e { TMPROF_BLOCK(); + if ( !compute_data ) { + end_nodes.push_back( end_node ); + return; + } + loop_head_to_bodies_t loop_heads_to_bodies; loop_endings loop_heads_ending = get_loop_heads_ending( end_node, loop_heads_to_bodies ); @@ -1701,20 +1701,27 @@ std::optional< location_id::id_type > iid_dependencies::get_next_iid_node() } } - std::optional< location_id::id_type > best_id = std::nullopt; - for ( const auto& [ id, props ] : node_id_to_equation_map ) { if ( props.should_generate() ) { return id; } } - return best_id; + return std::nullopt; } // ------------------------------------------------------------------------------------------------ generated_path fuzzing::iid_dependencies::generate_probabilities() { + if ( !compute_data ) { + compute_data = true; + + for ( branching_node* end_node : end_nodes ) { + process_node_dependence_from_full_path( end_node ); + } + end_nodes.clear(); + } + std::optional< location_id::id_type > id = get_next_iid_node(); if ( !id.has_value() ) { return {}; @@ -1759,8 +1766,11 @@ loop_endings fuzzing::iid_dependencies::get_loop_heads_ending( branching_node* e TMPROF_BLOCK(); std::vector< fuzzer::loop_boundary_props > loops; - // This is slow - fuzzer::detect_loops_along_path_to_node( end_node, loop_heads_to_bodies, &loops ); + { + TMPROF_BLOCK(); + // This is slow + fuzzer::detect_loops_along_path_to_node( end_node, loop_heads_to_bodies, &loops ); + } loop_endings loop_heads_ending; @@ -1916,20 +1926,17 @@ void fuzzing::iid_dependencies::compute_dependencies_by_loops( const loop_head_t std::unordered_set< location_id::id_type > all_ids; - { + all_ids.reserve( loop_bodies.size() + 1 ); + all_ids.insert( loop_head_id ); - all_ids.reserve( loop_bodies.size() + 1 ); - all_ids.insert( loop_head_id ); + std::vector< location_id::id_type > temp_ids; + temp_ids.reserve( loop_bodies.size() ); + std::transform( loop_bodies.begin(), + loop_bodies.end(), + std::back_inserter( temp_ids ), + []( const auto& loop_body ) { return loop_body.id; } ); - std::vector< location_id::id_type > temp_ids; - temp_ids.reserve( loop_bodies.size() ); - std::transform( loop_bodies.begin(), - loop_bodies.end(), - std::back_inserter( temp_ids ), - []( const auto& loop_body ) { return loop_body.id; } ); - - all_ids.insert( temp_ids.begin(), temp_ids.end() ); - } + all_ids.insert( temp_ids.begin(), temp_ids.end() ); loop_properties& props = loop_to_properties.get_props( all_ids, loop_head_id ); @@ -1945,7 +1952,6 @@ void fuzzing::iid_dependencies::compute_dependencies_by_loops( const loop_head_t } for ( bool direction : { true, false } ) { - // TMPROF_BLOCK(); auto [ it, inserted ] = props.bodies.emplace( body.id, direction ); changed |= inserted; } @@ -1969,7 +1975,7 @@ void fuzzing::iid_dependencies::compute_loading_loops( branching_node* end_node, loop_head_to_loaded_bits_counter& loading_loops, const loop_endings& loop_heads_ending ) { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { loading_loops[ loop_head.id ] = { std::numeric_limits< natural_32_bit >::max(), From 713da71d1806875b2d12da78685f01f847287a3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Wed, 12 Mar 2025 13:55:53 +0100 Subject: [PATCH 121/144] feat: use context --- .../include/fuzzing/iid_vector_analysis.hpp | 21 +++++---- src/fuzzing/src/iid_vector_analysis.cpp | 44 +++++++++++-------- 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index 24a1273b..78664246 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -123,8 +123,9 @@ struct iid_vector_analysis_statistics_per_node { }; struct iid_vector_analysis_statistics { - std::map< location_id::id_type, iid_vector_analysis_statistics_per_node > iid_nodes_stats; + std::map< location_id, iid_vector_analysis_statistics_per_node > iid_nodes_stats; std::vector< location_id::id_type > ignored_node_ids; + std::vector< location_id > covered_node_ids; loop_dependencies loop_to_properties; int processed_nodes; }; @@ -333,10 +334,6 @@ struct iid_node_dependence_props { equation& best_vector ); std::vector< equation > get_random_vector( const std::map< equation, int >& vectors_with_hits, int number_of_vectors ); - void compute_loading_loops( branching_node* end_node, - const loop_head_to_bodies_t& loop_heads_to_bodies, - loop_head_to_loaded_bits_counter& loading_loops, - const loop_endings& loop_heads_ending ); generated_path generate_path_from_node_counts( const nodes_to_counts& path_counts, const loop_dependencies& loop_to_properties ); @@ -350,11 +347,12 @@ struct iid_dependencies { void update_ignored_nodes( sensitivity_analysis& sensitivity ); void process_node_dependence( branching_node* node ); void process_node_dependence_from_full_path( branching_node* end_node ); - void remove_node_dependence( location_id::id_type id ); + void remove_node_dependence( location_id id ); void remove_all_covered( const std::unordered_set< location_id >& covered_branchings ); - iid_node_dependence_props& get_props( location_id::id_type id ); - std::vector< location_id::id_type > get_iid_nodes(); - std::optional< location_id::id_type > get_next_iid_node(); + iid_node_dependence_props& get_props( location_id id ); + std::vector< location_id > get_iid_nodes(); + std::optional< location_id > get_next_iid_node(); + void start_gathering_data(); generated_path generate_probabilities(); @@ -373,8 +371,9 @@ struct iid_dependencies { const loop_endings& loop_heads_ending ); void compute_paths( branching_node* end_node ); - std::map< location_id::id_type, iid_node_dependence_props > node_id_to_equation_map; + std::map< location_id, iid_node_dependence_props > node_id_to_equation_map; std::unordered_set< location_id::id_type > ignored_node_ids; + std::unordered_set< location_id > covered_node_ids; loop_dependencies loop_to_properties; int processed_nodes = 0; @@ -393,7 +392,7 @@ struct iid_dependencies { inline static float percentage_to_add_to_path = 0.4; inline static bool create_artificial_data = true; - inline static bool verbose = true; + inline static bool verbose = false; }; diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 43f50342..8391003a 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -1623,13 +1623,13 @@ void iid_dependencies::process_node_dependence_from_full_path( branching_node* e } // ------------------------------------------------------------------------------------------------ -void iid_dependencies::remove_node_dependence( location_id::id_type id ) +void iid_dependencies::remove_node_dependence( location_id id ) { auto it = node_id_to_equation_map.find( id ); if ( it != node_id_to_equation_map.end() ) { iid_node_generations_stats& stats = it->second.get_generations_stats(); stats.state = generation_state::STATE_COVERED; - ignored_node_ids.insert( id ); + covered_node_ids.insert( id ); if ( iid_dependencies::generate_more_data_after_coverage && !it->second.is_equal_branching_predicate() ) { stats.state = generation_state::STATE_GENERATION_MORE; @@ -1643,7 +1643,7 @@ void iid_dependencies::remove_node_dependence( location_id::id_type id ) // ------------------------------------------------------------------------------------------------ void fuzzing::iid_dependencies::remove_all_covered( const std::unordered_set< location_id >& covered_branchings ) { - std::set< location_id::id_type > covered_ids; + std::set< location_id > covered_ids; for ( const auto& branching : covered_branchings ) { covered_ids.insert( branching.id ); } @@ -1651,7 +1651,7 @@ void fuzzing::iid_dependencies::remove_all_covered( const std::unordered_set< lo for ( auto it = node_id_to_equation_map.begin(); it != node_id_to_equation_map.end(); ) { if ( covered_ids.contains( it->first ) && it->second.get_generations_stats().state == generation_state::STATE_NOT_COVERED ) { - ignored_node_ids.insert( it->first ); + covered_node_ids.insert( it->first ); it = node_id_to_equation_map.erase( it ); } else { ++it; @@ -1660,15 +1660,15 @@ void fuzzing::iid_dependencies::remove_all_covered( const std::unordered_set< lo } // ------------------------------------------------------------------------------------------------ -iid_node_dependence_props& iid_dependencies::get_props( location_id::id_type id ) +iid_node_dependence_props& iid_dependencies::get_props( location_id id ) { return node_id_to_equation_map.at( id ); } // ------------------------------------------------------------------------------------------------ -std::vector< location_id::id_type > iid_dependencies::get_iid_nodes() +std::vector< location_id > iid_dependencies::get_iid_nodes() { - std::vector< location_id::id_type > result; + std::vector< location_id > result; for ( const auto& [ key, _ ] : node_id_to_equation_map ) { result.push_back( key ); } @@ -1678,7 +1678,7 @@ std::vector< location_id::id_type > iid_dependencies::get_iid_nodes() } // ------------------------------------------------------------------------------------------------ -std::optional< location_id::id_type > iid_dependencies::get_next_iid_node() +std::optional< location_id > iid_dependencies::get_next_iid_node() { bool previous_needs_more_data = false; for ( auto it = node_id_to_equation_map.rbegin(); it != node_id_to_equation_map.rend(); ++it ) { @@ -1711,18 +1711,19 @@ std::optional< location_id::id_type > iid_dependencies::get_next_iid_node() } // ------------------------------------------------------------------------------------------------ -generated_path fuzzing::iid_dependencies::generate_probabilities() +void fuzzing::iid_dependencies::start_gathering_data() { - if ( !compute_data ) { - compute_data = true; + compute_data = true; - for ( branching_node* end_node : end_nodes ) { - process_node_dependence_from_full_path( end_node ); - } - end_nodes.clear(); + for ( branching_node* end_node : end_nodes ) { + process_node_dependence_from_full_path( end_node ); } +} - std::optional< location_id::id_type > id = get_next_iid_node(); +// ------------------------------------------------------------------------------------------------ +generated_path fuzzing::iid_dependencies::generate_probabilities() +{ + std::optional< location_id > id = get_next_iid_node(); if ( !id.has_value() ) { return {}; } @@ -1754,6 +1755,11 @@ iid_vector_analysis_statistics fuzzing::iid_dependencies::get_stats() const stats.ignored_node_ids = std::vector< location_id::id_type >( ignored_node_ids.begin(), ignored_node_ids.end() ); std::sort( stats.ignored_node_ids.begin(), stats.ignored_node_ids.end() ); + + stats.covered_node_ids = std::vector< location_id >( covered_node_ids.begin(), + covered_node_ids.end() ); + std::sort( stats.covered_node_ids.begin(), stats.covered_node_ids.end() ); + stats.loop_to_properties = loop_to_properties; stats.processed_nodes = processed_nodes; return stats; @@ -2040,16 +2046,16 @@ void fuzzing::iid_dependencies::compute_paths( branching_node* end_node ) for ( auto it = full_path.rbegin(); it != full_path.rend(); ++it ) { processed_nodes++; const auto& path_node = *it; - location_id::id_type node_id = path_node.node_id; directions_in_path[ path_node ]++; current_node = current_node->successor( path_node.branching_direction ).pointer; + location_id current_node_id = current_node->get_location_id(); - if ( ignored_node_ids.contains( current_node->get_location_id().id ) ) { + if ( ignored_node_ids.contains( current_node_id.id ) || covered_node_ids.contains( current_node_id ) ) { continue; } - iid_node_dependence_props& props = node_id_to_equation_map[ current_node->get_location_id().id ]; + iid_node_dependence_props& props = node_id_to_equation_map[ current_node_id ]; props.process_path_effective( current_node, directions_in_path ); } } From 35276710f8e95531923f72a12b59a473ef8cfadd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Wed, 12 Mar 2025 14:42:13 +0100 Subject: [PATCH 122/144] feat: computation of path little bit more efficient --- .../include/fuzzing/iid_vector_analysis.hpp | 3 ++- src/fuzzing/src/iid_vector_analysis.cpp | 24 +++++++++++++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index 78664246..ac19122d 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -370,6 +370,8 @@ struct iid_dependencies { loop_head_to_loaded_bits_counter& loading_loops, const loop_endings& loop_heads_ending ); void compute_paths( branching_node* end_node ); + std::vector< node_id_with_direction > get_path( branching_node* node ); + bool is_tracked( location_id id ) const; std::map< location_id, iid_node_dependence_props > node_id_to_equation_map; std::unordered_set< location_id::id_type > ignored_node_ids; @@ -396,7 +398,6 @@ struct iid_dependencies { }; -std::vector< node_id_with_direction > get_path( branching_node* node ); std::unordered_map< node_id_with_direction, int > get_directions_in_path( branching_node* node ); bool should_generate_more_data( const generation_state& state ); } // namespace fuzzing diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 8391003a..e37b37fa 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -1711,7 +1711,7 @@ std::optional< location_id > iid_dependencies::get_next_iid_node() } // ------------------------------------------------------------------------------------------------ -void fuzzing::iid_dependencies::start_gathering_data() +void fuzzing::iid_dependencies::start_gathering_data() { compute_data = true; @@ -1756,8 +1756,7 @@ iid_vector_analysis_statistics fuzzing::iid_dependencies::get_stats() const ignored_node_ids.end() ); std::sort( stats.ignored_node_ids.begin(), stats.ignored_node_ids.end() ); - stats.covered_node_ids = std::vector< location_id >( covered_node_ids.begin(), - covered_node_ids.end() ); + stats.covered_node_ids = std::vector< location_id >( covered_node_ids.begin(), covered_node_ids.end() ); std::sort( stats.covered_node_ids.begin(), stats.covered_node_ids.end() ); stats.loop_to_properties = loop_to_properties; @@ -2051,7 +2050,7 @@ void fuzzing::iid_dependencies::compute_paths( branching_node* end_node ) current_node = current_node->successor( path_node.branching_direction ).pointer; location_id current_node_id = current_node->get_location_id(); - if ( ignored_node_ids.contains( current_node_id.id ) || covered_node_ids.contains( current_node_id ) ) { + if ( !is_tracked( current_node_id ) ) { continue; } @@ -2060,28 +2059,39 @@ void fuzzing::iid_dependencies::compute_paths( branching_node* end_node ) } } -// non member functions // ------------------------------------------------------------------------------------------------ -std::vector< node_id_with_direction > get_path( branching_node* node ) +std::vector< node_id_with_direction > fuzzing::iid_dependencies::get_path( branching_node* node ) { // TMPROF_BLOCK(); std::vector< node_id_with_direction > result; + bool iid_seen = false; branching_node* current = node; + while ( current != nullptr ) { branching_node* predecessor = current->predecessor; - if ( predecessor != nullptr ) { + iid_seen = iid_seen || is_tracked( current->get_location_id() ); + + if ( iid_seen && predecessor != nullptr ) { node_id_with_direction nav = { predecessor->get_location_id().id, predecessor->successor_direction( current ) }; result.push_back( nav ); } + current = predecessor; } return result; } +// ------------------------------------------------------------------------------------------------ +bool fuzzing::iid_dependencies::is_tracked( location_id id ) const +{ + return !ignored_node_ids.contains( id.id ) && !covered_node_ids.contains( id ); +} + +// non member functions // ------------------------------------------------------------------------------------------------ std::unordered_map< node_id_with_direction, int > get_directions_in_path( branching_node* node ) { From 575f7b91611ef3c8761f4c42d294bb138b5131a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Wed, 12 Mar 2025 17:06:40 +0100 Subject: [PATCH 123/144] feat: better integration into `select_iid_coverage_target` --- .../include/fuzzing/iid_vector_analysis.hpp | 4 +++ src/fuzzing/src/fuzzer.cpp | 36 ++++++++++++------- src/fuzzing/src/iid_vector_analysis.cpp | 1 + 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index ac19122d..41645df8 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -178,6 +178,7 @@ struct node_props_in_path { struct generated_path { generated_path( std::map< location_id::id_type, node_props_in_path > path ) : path( std::move( path ) ) + , iid_node_id( std::nullopt ) {} generated_path() = default; @@ -185,6 +186,8 @@ struct generated_path { bool contains( location_id::id_type id ) const; std::map< location_id::id_type, node_props_in_path > get_path() const; node_props_in_path& get_props( location_id::id_type id ) { return path.at( id ); } + std::optional< location_id > get_iid_node_id() const { return iid_node_id; } + void set_iid_node_id( location_id iid_node_id ) { this->iid_node_id = iid_node_id; } friend std::ostream& operator<<( std::ostream& os, const generated_path& eq ) { @@ -197,6 +200,7 @@ struct generated_path { private: std::map< location_id::id_type, node_props_in_path > path; + std::optional< location_id > iid_node_id; }; struct equation { diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index 6c02f3cc..0226d077 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -1430,7 +1430,7 @@ void fuzzer::do_cleanup() if (covered_branchings.contains(it->first)) { if ( use_vector_analysis ) - iid_dependences.remove_node_dependence(it->first.id); + iid_dependences.remove_node_dependence( it->first ); it = iid_pivots.erase(it); } @@ -1670,10 +1670,27 @@ branching_node* fuzzer::select_iid_coverage_target() if (iid_pivots.empty() || entry_branching->is_closed()) return nullptr; - auto const it_loc = std::next( - iid_pivots.begin(), - get_random_natural_32_bit_in_range(0, (natural_32_bit)iid_pivots.size() - 1, generator_for_iid_location_selection) - ); + if ( use_vector_analysis ) { + iid_dependences.start_gathering_data(); + } + + generated_path path; + if ( use_vector_analysis ) { + path = iid_dependences.generate_probabilities(); + } + + auto it_loc = std::next( + iid_pivots.begin(), + get_random_natural_32_bit_in_range(0, (natural_32_bit)iid_pivots.size() - 1, generator_for_iid_location_selection) + ); + + if ( use_vector_analysis && path.get_iid_node_id().has_value() ) { + auto it_loc_new = iid_pivots.find( path.get_iid_node_id().value() ); + if ( it_loc_new != iid_pivots.end() ) { + it_loc = it_loc_new; + } + } + auto const it_pivot = select_best_iid_pivot( it_loc->second.pivots, max_input_width, @@ -1692,16 +1709,9 @@ branching_node* fuzzer::select_iid_coverage_target() histogram_of_hit_counts_per_direction::hit_counts_map hit_counts; it_pivot->second.histogram_ptr->merge(hit_counts); - generated_path path; - if ( use_vector_analysis ) { - path = iid_dependences.generate_probabilities(); - for ( const auto& path_props : path.get_path() ) { - auto it = histogram.find( path_props.first ); - if ( it != histogram.end() ) { - it->second = path_props.second.get_false_direction_probability(); - } + histogram[path_props.first] = path_props.second.get_false_direction_probability(); } } diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index e37b37fa..45d3ddeb 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -1734,6 +1734,7 @@ generated_path fuzzing::iid_dependencies::generate_probabilities() iid_node_dependence_props& props = get_props( id.value() ); generated_path path = props.generate_probabilities( loop_to_properties ); + path.set_iid_node_id( id.value() ); return path; } From fcbf8a1e52c3d7940d3799f6fac0db399bacadbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 15 Mar 2025 18:23:01 +0100 Subject: [PATCH 124/144] feat: compute only when needed --- .../include/fuzzing/iid_vector_analysis.hpp | 16 ++- src/fuzzing/src/fuzzer.cpp | 4 +- src/fuzzing/src/iid_vector_analysis.cpp | 123 ++++++++++++------ 3 files changed, 95 insertions(+), 48 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index 41645df8..297bd3e7 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -48,6 +48,7 @@ struct iid_node_generations_stats { int method_calls = 0; int generation_starts = 0; int successful_generations = 0; + int successful_generations_artificial_data = 0; int failed_generations = 0; int generated_for_other_node_count = 0; @@ -119,7 +120,9 @@ struct loop_dependencies { struct iid_vector_analysis_statistics_per_node { iid_node_generations_stats generation_stats; + std::pair< std::size_t, std::size_t > vector_dimensions; std::vector< location_id::id_type > node_ids; + bool matrix_generated; }; struct iid_vector_analysis_statistics { @@ -128,6 +131,7 @@ struct iid_vector_analysis_statistics { std::vector< location_id > covered_node_ids; loop_dependencies loop_to_properties; int processed_nodes; + int dependencies_computed; }; @@ -188,6 +192,7 @@ struct generated_path { node_props_in_path& get_props( location_id::id_type id ) { return path.at( id ); } std::optional< location_id > get_iid_node_id() const { return iid_node_id; } void set_iid_node_id( location_id iid_node_id ) { this->iid_node_id = iid_node_id; } + bool empty() const { return path.empty(); } friend std::ostream& operator<<( std::ostream& os, const generated_path& eq ) { @@ -267,6 +272,7 @@ struct equation_matrix { int get_desired_vector_direction() const; float get_biggest_branching_value() const; const std::unordered_set< location_id::id_type >& get_node_ids() const; + bool empty() const { return matrix.empty(); } void print_matrix(); @@ -290,12 +296,13 @@ struct iid_node_dependence_props { const equation_matrix& get_matrix() const { return matrix; } const iid_node_generations_stats& get_generations_stats() const { return stats; } - bool should_generate() const; + bool should_generate( const loop_dependencies& loop_to_properties ) const; bool too_much_failed_in_row( int max_failed_generations_in_row ) const; void set_as_generating_for_other_node( int minimal_max_generation_for_other_node ); void set_as_generating_artificial_data( int minimal_max_generation_artificial_data ); failed_generation_method get_method_for_failed_generation( bool is_first ); bool is_equal_branching_predicate() const; + bool is_matrix_generated() const { return matrix_generated; } void print_stats( bool only_state = false ) const; @@ -350,13 +357,13 @@ struct iid_node_dependence_props { struct iid_dependencies { void update_ignored_nodes( sensitivity_analysis& sensitivity ); void process_node_dependence( branching_node* node ); - void process_node_dependence_from_full_path( branching_node* end_node ); + void process_node( branching_node* end_node ); void remove_node_dependence( location_id id ); void remove_all_covered( const std::unordered_set< location_id >& covered_branchings ); iid_node_dependence_props& get_props( location_id id ); std::vector< location_id > get_iid_nodes(); std::optional< location_id > get_next_iid_node(); - void start_gathering_data(); + void compute_dependencies(); generated_path generate_probabilities(); @@ -382,8 +389,8 @@ struct iid_dependencies { std::unordered_set< location_id > covered_node_ids; loop_dependencies loop_to_properties; int processed_nodes = 0; + int dependencies_computed = 0; - bool compute_data = false; std::vector< branching_node* > end_nodes; public: @@ -397,6 +404,7 @@ struct iid_dependencies { inline static int minimal_max_generation_artificial_data = 5; inline static float percentage_to_add_to_path = 0.4; inline static bool create_artificial_data = true; + inline static bool regenerate_all_data = true; inline static bool verbose = false; }; diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index 0226d077..98f7e9a9 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -1245,7 +1245,7 @@ execution_record::execution_flags fuzzer::process_execution_results() }); if ( use_vector_analysis ) - iid_dependences.process_node_dependence_from_full_path( construction_props.leaf ); + iid_dependences.process_node( construction_props.leaf ); if (construction_props.diverging_node != nullptr) { @@ -1671,7 +1671,7 @@ branching_node* fuzzer::select_iid_coverage_target() return nullptr; if ( use_vector_analysis ) { - iid_dependences.start_gathering_data(); + iid_dependences.compute_dependencies(); } generated_path path; diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 45d3ddeb..34ee9373 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -509,6 +509,8 @@ std::set< location_id::id_type > fuzzing::loop_dependencies::get_loop_heads( boo std::set< node_id_with_direction > fuzzing::loop_dependencies::get_node_subsets_for_computation( const std::unordered_set< location_id::id_type >& matrix_ids ) const { + // TMPROF_BLOCK(); + std::set< node_id_with_direction > computation_subset; for ( const auto& loop : loops ) { @@ -887,14 +889,30 @@ generated_path iid_node_dependence_props::generate_probabilities( const loop_dep stats.method_calls++; + if ( iid_dependencies::verbose ) { + // print_stats( true ); + loop_to_properties.print_dependencies(); + matrix.print_matrix(); + } + if ( !matrix_generated ) { matrix_generated = true; matrix.start_compute_matrix(); } + if ( loop_to_properties.loops.empty() ) { + if ( iid_dependencies::verbose ) + std::cout << "No loops" << std::endl; + + return {}; + } + std::set< node_id_with_direction > computation_subset = loop_to_properties.get_node_subsets_for_computation( matrix.get_node_ids() ); - if ( computation_subset.empty() || loop_to_properties.loops.empty() ) { + if ( computation_subset.empty() ) { + if ( iid_dependencies::verbose ) + std::cout << "No nodes in subset" << std::endl; + return {}; } @@ -905,12 +923,15 @@ generated_path iid_node_dependence_props::generate_probabilities( const loop_dep stats.generation_starts++; equation_matrix submatrix = matrix.get_submatrix( computation_subset, true ); + if ( submatrix.empty() ) { + if ( iid_dependencies::verbose ) + std::cout << "Empty submatrix" << std::endl; - { - // print_stats( true ); - // loop_to_properties.print_dependencies(); - // matrix.print_matrix(); - // submatrix.print_matrix(); + return return_empty_path(); + } + + if ( iid_dependencies::verbose ) { + submatrix.print_matrix(); } std::optional< std::vector< equation > > best_vectors = get_best_vectors( submatrix, 1 ); @@ -953,8 +974,16 @@ void fuzzing::iid_node_dependence_props::process_path_effective( } // ------------------------------------------------------------------------------------------------ -bool iid_node_dependence_props::should_generate() const +bool iid_node_dependence_props::should_generate( const loop_dependencies& loop_to_properties ) const { + if ( !matrix_generated ) { + return true; + } + + // if ( !loop_to_properties.get_node_subsets_for_computation( get_matrix().get_node_ids() ).empty() ) { + // return true; + // } + return stats.state != generation_state::STATE_COVERED; } @@ -1137,7 +1166,11 @@ generated_path iid_node_dependence_props::return_empty_path() generated_path iid_node_dependence_props::return_path( const generated_path& path ) { stats.failed_generations_in_row = 0; - stats.successful_generations++; + if ( stats.state == generation_state::STATE_GENERATING_ARTIFICIAL_DATA ) { + stats.successful_generations_artificial_data++; + } else { + stats.successful_generations++; + } if ( stats.state == generation_state::STATE_GENERATION_MORE ) { stats.generated_after_covered++; @@ -1600,26 +1633,11 @@ void iid_dependencies::update_ignored_nodes( sensitivity_analysis& sensitivity ) } // ------------------------------------------------------------------------------------------------ -void iid_dependencies::process_node_dependence_from_full_path( branching_node* end_node ) +void iid_dependencies::process_node( branching_node* end_node ) { - TMPROF_BLOCK(); - - if ( !compute_data ) { - end_nodes.push_back( end_node ); - return; - } - - loop_head_to_bodies_t loop_heads_to_bodies; - loop_endings loop_heads_ending = get_loop_heads_ending( end_node, loop_heads_to_bodies ); - - compute_dependencies_by_loops( loop_heads_to_bodies, loop_heads_ending ); - compute_dependencies_by_loading( end_node, loop_heads_to_bodies, loop_heads_ending ); - - for ( auto& loop : loop_to_properties.loops ) { - loop.set_chosen_loop_head(); - } + // TMPROF_BLOCK(); - compute_paths( end_node ); + end_nodes.push_back( end_node ); } // ------------------------------------------------------------------------------------------------ @@ -1680,6 +1698,8 @@ std::vector< location_id > iid_dependencies::get_iid_nodes() // ------------------------------------------------------------------------------------------------ std::optional< location_id > iid_dependencies::get_next_iid_node() { + // TMPROF_BLOCK(); + bool previous_needs_more_data = false; for ( auto it = node_id_to_equation_map.rbegin(); it != node_id_to_equation_map.rend(); ++it ) { iid_node_dependence_props& props = it->second; @@ -1702,7 +1722,7 @@ std::optional< location_id > iid_dependencies::get_next_iid_node() } for ( const auto& [ id, props ] : node_id_to_equation_map ) { - if ( props.should_generate() ) { + if ( props.should_generate( loop_to_properties ) ) { return id; } } @@ -1711,18 +1731,35 @@ std::optional< location_id > iid_dependencies::get_next_iid_node() } // ------------------------------------------------------------------------------------------------ -void fuzzing::iid_dependencies::start_gathering_data() +void fuzzing::iid_dependencies::compute_dependencies() { - compute_data = true; + // TMPROF_BLOCK(); + + dependencies_computed++; for ( branching_node* end_node : end_nodes ) { - process_node_dependence_from_full_path( end_node ); + // TMPROF_BLOCK(); + + loop_head_to_bodies_t loop_heads_to_bodies; + loop_endings loop_heads_ending = get_loop_heads_ending( end_node, loop_heads_to_bodies ); + + compute_dependencies_by_loops( loop_heads_to_bodies, loop_heads_ending ); + compute_dependencies_by_loading( end_node, loop_heads_to_bodies, loop_heads_ending ); + + for ( auto& loop : loop_to_properties.loops ) { + loop.set_chosen_loop_head(); + } + + compute_paths( end_node ); } + + end_nodes.clear(); } // ------------------------------------------------------------------------------------------------ generated_path fuzzing::iid_dependencies::generate_probabilities() { + // TMPROF_BLOCK(); std::optional< location_id > id = get_next_iid_node(); if ( !id.has_value() ) { return {}; @@ -1734,7 +1771,11 @@ generated_path fuzzing::iid_dependencies::generate_probabilities() iid_node_dependence_props& props = get_props( id.value() ); generated_path path = props.generate_probabilities( loop_to_properties ); - path.set_iid_node_id( id.value() ); + + if ( !path.empty() ) { + path.set_iid_node_id( id.value() ); + } + return path; } @@ -1749,7 +1790,8 @@ iid_vector_analysis_statistics fuzzing::iid_dependencies::get_stats() const node_stats.node_ids = std::vector< location_id::id_type >( props.get_matrix().get_node_ids().begin(), props.get_matrix().get_node_ids().end() ); std::sort( node_stats.node_ids.begin(), node_stats.node_ids.end() ); - + node_stats.vector_dimensions = props.get_matrix().get_dimensions(); + node_stats.matrix_generated = props.is_matrix_generated(); stats.iid_nodes_stats[ id ] = node_stats; } @@ -1762,6 +1804,7 @@ iid_vector_analysis_statistics fuzzing::iid_dependencies::get_stats() const stats.loop_to_properties = loop_to_properties; stats.processed_nodes = processed_nodes; + stats.dependencies_computed = dependencies_computed; return stats; } @@ -1769,14 +1812,10 @@ iid_vector_analysis_statistics fuzzing::iid_dependencies::get_stats() const loop_endings fuzzing::iid_dependencies::get_loop_heads_ending( branching_node* end_node, loop_head_to_bodies_t& loop_heads_to_bodies ) { - TMPROF_BLOCK(); + // TMPROF_BLOCK(); std::vector< fuzzer::loop_boundary_props > loops; - { - TMPROF_BLOCK(); - // This is slow - fuzzer::detect_loops_along_path_to_node( end_node, loop_heads_to_bodies, &loops ); - } + fuzzer::detect_loops_along_path_to_node( end_node, loop_heads_to_bodies, &loops ); loop_endings loop_heads_ending; @@ -1817,7 +1856,7 @@ void fuzzing::iid_dependencies::compute_dependencies_by_loading( branching_node* const loop_head_to_bodies_t& loop_heads_to_bodies, const loop_endings& loop_heads_ending ) { - TMPROF_BLOCK(); + // TMPROF_BLOCK(); loop_head_to_loaded_bits_counter loading_loops; compute_loading_loops( end_node, loop_heads_to_bodies, loading_loops, loop_heads_ending ); @@ -1915,7 +1954,7 @@ void fuzzing::iid_dependencies::compute_dependencies_by_loading( branching_node* void fuzzing::iid_dependencies::compute_dependencies_by_loops( const loop_head_to_bodies_t& loop_heads_to_bodies, const loop_endings& loop_heads_ending ) { - TMPROF_BLOCK(); + // TMPROF_BLOCK(); bool changed = false; @@ -1981,7 +2020,7 @@ void fuzzing::iid_dependencies::compute_loading_loops( branching_node* end_node, loop_head_to_loaded_bits_counter& loading_loops, const loop_endings& loop_heads_ending ) { - TMPROF_BLOCK(); + // TMPROF_BLOCK(); for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { loading_loops[ loop_head.id ] = { std::numeric_limits< natural_32_bit >::max(), @@ -2031,7 +2070,7 @@ void fuzzing::iid_dependencies::compute_loading_loops( branching_node* end_node, // ------------------------------------------------------------------------------------------------ void fuzzing::iid_dependencies::compute_paths( branching_node* end_node ) { - TMPROF_BLOCK(); + // TMPROF_BLOCK(); INVARIANT( end_node != nullptr ); From 0f4779703bbecff0b75474b9cfa13835e47011f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 15 Mar 2025 18:23:20 +0100 Subject: [PATCH 125/144] feat: better dumping --- src/fuzzing/src/dump.cpp | 190 ++++++++++++++++++++++++--------------- src/tools/server/run.cpp | 2 + 2 files changed, 119 insertions(+), 73 deletions(-) diff --git a/src/fuzzing/src/dump.cpp b/src/fuzzing/src/dump.cpp index ff6d5720..4c47275d 100644 --- a/src/fuzzing/src/dump.cpp +++ b/src/fuzzing/src/dump.cpp @@ -235,15 +235,6 @@ void print_analysis_outcomes(std::ostream& ostr, analysis_outcomes const& res ostr << "}"; } - -void log_analysis_outcomes(analysis_outcomes const& results) -{ - std::stringstream sstr; - print_analysis_outcomes(sstr, results); - LOG(LSL_INFO, sstr.str()); -} - - void save_iid_vector_analysis( std::ostream& ostr, analysis_outcomes const& result ) { bool print_dependencies = true; @@ -257,9 +248,101 @@ void save_iid_vector_analysis( std::ostream& ostr, analysis_outcomes const& resu return result; }; + fuzzing::iid_node_generations_stats total_stats; + for ( const auto& [ _ ,node_stat ] : result.iid_vector_analysis_statistics.iid_nodes_stats ) { + total_stats.method_calls += node_stat.generation_stats.method_calls; + total_stats.generation_starts += node_stat.generation_stats.generation_starts; + total_stats.successful_generations += node_stat.generation_stats.successful_generations; + total_stats.successful_generations_artificial_data += node_stat.generation_stats.successful_generations_artificial_data; + total_stats.failed_generations += node_stat.generation_stats.failed_generations; + total_stats.generate_artificial_data_count += node_stat.generation_stats.generate_artificial_data_count; + total_stats.generated_for_other_node_count += node_stat.generation_stats.generated_for_other_node_count; + } ostr << "{\n"; - ostr << indent( 1 ) << "\"Analyzed node\": {\n"; + + ostr << indent( 1 ) << "\"Total statistics\": {\n"; + ostr << indent( 2 ) << "\"Processed nodes\": " << result.iid_vector_analysis_statistics.processed_nodes << ",\n"; + ostr << indent( 2 ) << "\"Dependencies computed\": " << result.iid_vector_analysis_statistics.dependencies_computed << ",\n"; + ostr << indent( 2 ) << "\"Method calls\": " << total_stats.method_calls << ",\n"; + ostr << indent( 2 ) << "\"Generation starts\": " << total_stats.generation_starts << ",\n"; + ostr << indent( 2 ) << "\"Successful generations\": " << total_stats.successful_generations << ",\n"; + ostr << indent( 2 ) << "\"Successful generations of artificial data\": " << total_stats.successful_generations_artificial_data << ",\n"; + ostr << indent( 2 ) << "\"Failed generations\": " << total_stats.failed_generations << ",\n"; + ostr << indent( 2 ) << "\"Artificial generations\": " << total_stats.generate_artificial_data_count << ",\n"; + ostr << indent( 2 ) << "\"For other node generations\": " << total_stats.generated_for_other_node_count << "\n"; + ostr << indent( 1 ) << "},\n"; + + ostr << indent( 1 ) << "\"Dependencies\": [\n"; + for ( auto loop_it = result.iid_vector_analysis_statistics.loop_to_properties.loops.begin(); + loop_it != result.iid_vector_analysis_statistics.loop_to_properties.loops.end(); + ++loop_it ) { + + ostr << indent( 2 ) << "{\n"; + ostr << indent( 3 ) << "\"Loop heads\": [\n"; + for ( auto head_it = loop_it->heads.begin(); head_it != loop_it->heads.end(); ++head_it ) { + ostr << indent( 4 ) << std::dec << "\"" << head_it->first << "\"" + << ( std::next( head_it ) != loop_it->heads.end() ? "," : "" ) << '\n'; + } + ostr << indent( 3 ) << "],\n"; + + ostr << indent( 3 ) << "\"Loop bodies\": [\n"; + for ( auto body_it = loop_it->bodies.begin(); body_it != loop_it->bodies.end(); ++body_it ) { + ostr << indent( 4 ) << std::dec << "\"" << *body_it << "\"" + << ( std::next( body_it ) != loop_it->bodies.end() ? "," : "" ) << '\n'; + } + ostr << indent( 3 ) << "],\n"; + + ostr << indent( 3 ) + << "\"Is loading loop\": " << ( loop_it->is_loading_loop ? "true," : "false" ) << "\n"; + + if ( !loop_it->is_loading_loop ) { + ostr << indent( 2 ) << "}" + << ( std::next( loop_it ) != result.iid_vector_analysis_statistics.loop_to_properties.loops.end() ? "," : "" ) + << '\n'; + continue; + } + + ostr << indent( 3 ) << "\"Loaded bits per loop\": " << loop_it->loaded_bits_per_loop.mean << ",\n"; + ostr << indent( 3 ) << "\"Dependent nodes\": [\n"; + for ( auto body_it = loop_it->bits_read_by_node.begin(); + body_it != loop_it->bits_read_by_node.end(); + ++body_it ) { + ostr << indent( 4 ) << "{\n"; + ostr << indent( 5 ) << "\"Node\": " << std::dec << body_it->first << ",\n"; + ostr << indent( 5 ) << "\"Bits read\": " << body_it->second.average_bits_read.mean << ",\n"; + ostr << indent( 5 ) << "\"Minimal bit offset\": " << body_it->second.minimal_bit_offset + << '\n'; + ostr << indent( 4 ) << "}" + << ( std::next( body_it ) != loop_it->bits_read_by_node.end() ? "," : "" ) << '\n'; + } + ostr << indent( 3 ) << "]\n"; + + ostr << indent( 2 ) << "}" + << ( std::next( loop_it ) != result.iid_vector_analysis_statistics.loop_to_properties.loops.end() ? "," : "" ) << '\n'; + } + + ostr << indent( 1 ) << "],\n"; + + ostr << indent( 1 ) << "\"Ignored nodes\": [\n"; + for ( std::size_t i = 0, n = result.iid_vector_analysis_statistics.ignored_node_ids.size(); i < n; ++i ) { + ostr << indent( 2 ) << std::dec << result.iid_vector_analysis_statistics.ignored_node_ids.at( i ); + if ( i + 1 < n ) + ostr << ','; + ostr << '\n'; + } + ostr << indent( 1 ) << "],\n"; + + ostr << indent( 1 ) << "\"Covered nodes\": [\n"; + for ( std::size_t i = 0, n = result.iid_vector_analysis_statistics.covered_node_ids.size(); i < n; ++i ) { + ostr << indent( 2 ) << std::dec << "\"" << result.iid_vector_analysis_statistics.covered_node_ids.at( i ) <<"\""; + if ( i + 1 < n ) + ostr << ','; + ostr << '\n'; + } + ostr << indent( 1 ) << "],\n"; + + ostr << indent( 1 ) << "\"Analyzed nodes\": {\n"; for ( auto it = result.iid_vector_analysis_statistics.iid_nodes_stats.begin(); it != result.iid_vector_analysis_statistics.iid_nodes_stats.end(); @@ -285,6 +368,8 @@ void save_iid_vector_analysis( std::ostream& ostr, analysis_outcomes const& resu ostr << indent( 3 ) << "\"Generation starts\": " << it->second.generation_stats.generation_starts << ",\n"; ostr << indent( 3 ) << "\"Successful generations\": " << it->second.generation_stats.successful_generations << ",\n"; + ostr << indent( 3 ) << "\"Successful generations artificial data\": " + << it->second.generation_stats.successful_generations_artificial_data << ",\n"; ostr << indent( 3 ) << "\"Failed generations\": " << it->second.generation_stats.failed_generations << ",\n"; ostr << indent( 3 ) @@ -292,80 +377,39 @@ void save_iid_vector_analysis( std::ostream& ostr, analysis_outcomes const& resu << ",\n"; ostr << indent( 3 ) << "\"For other node generations\": " << it->second.generation_stats.generated_for_other_node_count - << ( print_dependencies ? "," : "" ) << "\n"; - - if ( print_dependencies ) { - ostr << indent( 3 ) << "\"Dependencies\": [\n"; - for ( auto loop_it = it->second.loop_to_properties.loops.begin(); - loop_it != it->second.loop_to_properties.loops.end(); - ++loop_it ) { - - ostr << indent( 4 ) << "{\n"; - ostr << indent( 5 ) << "\"Loop heads\": [\n"; - for ( auto head_it = loop_it->heads.begin(); head_it != loop_it->heads.end(); ++head_it ) { - ostr << indent( 6 ) << std::dec << "\"" << head_it->first << "\"" - << ( std::next( head_it ) != loop_it->heads.end() ? "," : "" ) << '\n'; - } - ostr << indent( 5 ) << "],\n"; - - ostr << indent( 5 ) << "\"Loop bodies\": [\n"; - for ( auto body_it = loop_it->bodies.begin(); body_it != loop_it->bodies.end(); ++body_it ) { - ostr << indent( 6 ) << std::dec << "\"" << *body_it << "\"" - << ( std::next( body_it ) != loop_it->bodies.end() ? "," : "" ) << '\n'; - } - ostr << indent( 5 ) << "],\n"; - - ostr << indent( 5 ) - << "\"Is loading loop\": " << ( loop_it->is_loading_loop ? "true," : "false" ) << "\n"; - - if ( !loop_it->is_loading_loop ) { - ostr << indent( 4 ) << "}" - << ( std::next( loop_it ) != it->second.loop_to_properties.loops.end() ? "," : "" ) - << '\n'; - continue; - } - - ostr << indent( 5 ) << "\"Loaded bits per loop\": " << loop_it->loaded_bits_per_loop.mean << ",\n"; - ostr << indent( 5 ) << "\"Dependent nodes\": [\n"; - for ( auto body_it = loop_it->bits_read_by_node.begin(); - body_it != loop_it->bits_read_by_node.end(); - ++body_it ) { - ostr << indent( 6 ) << "{\n"; - ostr << indent( 7 ) << "\"Node\": " << std::dec << body_it->first << ",\n"; - ostr << indent( 7 ) << "\"Bits read\": " << body_it->second.average_bits_read.mean << ",\n"; - ostr << indent( 7 ) << "\"Minimal bit offset\": " << body_it->second.minimal_bit_offset - << '\n'; - ostr << indent( 6 ) << "}" - << ( std::next( body_it ) != loop_it->bits_read_by_node.end() ? "," : "" ) << '\n'; - } - ostr << indent( 5 ) << "]\n"; - - ostr << indent( 4 ) << "}" - << ( std::next( loop_it ) != it->second.loop_to_properties.loops.end() ? "," : "" ) << '\n'; - } - - ostr << indent( 3 ) << "]\n"; + << ",\n"; + ostr << indent( 3 ) << "\" Is matrix generated\": " << ( it->second.matrix_generated ? "true" : "false" ) + << ",\n"; + ostr << indent( 3 ) << "\"Matrix dimension\": {\n"; + ostr << indent( 4 ) << "\"Rows\": " << it->second.vector_dimensions.first << ",\n"; + ostr << indent( 4 ) << "\"Columns\": " << it->second.vector_dimensions.second << "\n"; + ostr << indent( 3 ) << "},\n"; + ostr << indent( 3 ) << "\"Possible nodes\": [\n"; + for ( std::size_t i = 0, n = it->second.node_ids.size(); i < n; ++i ) { + ostr << indent( 4 ) << std::dec << it->second.node_ids.at( i ); + if ( i + 1 < n ) + ostr << ','; + ostr << '\n'; } + ostr << indent( 3 ) << "]\n"; ostr << indent( 2 ) << "}" << ( std::next( it ) != result.iid_vector_analysis_statistics.iid_nodes_stats.end() ? "," : "" ) << '\n'; } - ostr << indent( 1 ) << "},\n"; - - ostr << indent( 1 ) << "\"Ignored nodes\": [\n"; - for ( std::size_t i = 0, n = result.iid_vector_analysis_statistics.ignored_node_ids.size(); i < n; ++i ) { - ostr << indent( 2 ) << std::dec << result.iid_vector_analysis_statistics.ignored_node_ids.at( i ); - if ( i + 1 < n ) - ostr << ','; - ostr << '\n'; - } - ostr << indent( 1 ) << "]\n"; + ostr << indent( 1 ) << "}\n"; ostr << "}"; } +void log_analysis_outcomes(analysis_outcomes const& results) +{ + std::stringstream sstr; + print_analysis_outcomes(sstr, results); + LOG(LSL_INFO, sstr.str()); +} + void save_analysis_outcomes( std::filesystem::path const& output_dir, std::string const& benchmark, diff --git a/src/tools/server/run.cpp b/src/tools/server/run.cpp index 6b66f596..7433ab61 100644 --- a/src/tools/server/run.cpp +++ b/src/tools/server/run.cpp @@ -244,6 +244,8 @@ void run(int argc, char* argv[]) { std::cout << "\"fuzzing_results\": "; fuzzing::print_analysis_outcomes(std::cout, results); + std::cout << ", \"IID_analysis_info\": "; + fuzzing::save_iid_vector_analysis(std::cout, results); } fuzzing::log_analysis_outcomes(results); fuzzing::save_analysis_outcomes(output_dir, target_name, results); From 0c1ab64ea4b81b59a57bcf3b9c1c17948ea908a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 15 Mar 2025 18:23:38 +0100 Subject: [PATCH 126/144] feat: better dumping --- src/fuzzing/include/fuzzing/dump.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/fuzzing/include/fuzzing/dump.hpp b/src/fuzzing/include/fuzzing/dump.hpp index 10ec4631..1ad46119 100644 --- a/src/fuzzing/include/fuzzing/dump.hpp +++ b/src/fuzzing/include/fuzzing/dump.hpp @@ -30,6 +30,7 @@ void save_fuzzing_configuration( termination_info const& terminator ); +void save_iid_vector_analysis(std::ostream& ostr, analysis_outcomes const& results); void print_analysis_outcomes(std::ostream& ostr, analysis_outcomes const& results); void log_analysis_outcomes(analysis_outcomes const& results); void save_analysis_outcomes( From 09a954a71de39f37760c8f67dbcb799962db007d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sun, 16 Mar 2025 21:34:25 +0100 Subject: [PATCH 127/144] feat: compute vector with hits and submatrix only once --- .../include/fuzzing/iid_vector_analysis.hpp | 86 +++-- src/fuzzing/src/iid_vector_analysis.cpp | 320 +++++++++--------- 2 files changed, 204 insertions(+), 202 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index 297bd3e7..e0d28ef8 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -113,8 +114,8 @@ struct loop_dependencies { const loop_properties& get_props_by_loop_head_id( location_id::id_type loop_head_id ) const; loop_properties& get_props_by_loop_head_id( location_id::id_type loop_head_id ); std::set< location_id::id_type > get_loop_heads( bool include_loading_loops ) const; - std::set< node_id_with_direction > - get_node_subsets_for_computation( const std::unordered_set< location_id::id_type >& matrix_ids ) const; + bool compute_node_subsets_for_computation( std::set< node_id_with_direction >& computation_subset, + const std::unordered_set< location_id::id_type >& matrix_ids ) const; void print_dependencies() const; }; @@ -244,28 +245,58 @@ struct equation { return os << " -> " << eq.best_value; } }; +} // namespace fuzzing + +namespace std +{ +template <> +struct hash< fuzzing::node_id_with_direction > { + std::size_t operator()( const fuzzing::node_id_with_direction& key ) const noexcept + { + std::size_t h1 = std::hash< location_id::id_type >{}( key.node_id ); + std::size_t h2 = std::hash< bool >{}( key.branching_direction ); + return h1 ^ ( h2 << 1 ); + } +}; + +template <> +struct hash< fuzzing::equation > { + std::size_t operator()( const fuzzing::equation& eq ) const noexcept + { + std::size_t h1 = std::hash< double >{}( eq.best_value ); + for ( int val : eq.values ) { + h1 ^= std::hash< int >{}( val ) + 0x9e3779b9 + ( h1 << 6 ) + ( h1 >> 2 ); + } + return h1; + } +}; +} // namespace std + +namespace fuzzing +{ struct loaded_bits_counter { natural_32_bit min; natural_32_bit max; int loop_count; }; - +using path_id_direction_count = std::array< location_id::id_type, 20000 >; using loop_head_to_loaded_bits_counter = std::unordered_map< location_id::id_type, loaded_bits_counter >; using loop_endings = std::unordered_map< location_id::id_type, bool >; using loop_head_to_bodies_t = std::unordered_map< location_id, std::unordered_set< location_id > >; using nodes_to_counts = std::map< location_id::id_type, node_counts >; struct equation_matrix { - equation_matrix get_submatrix( std::set< node_id_with_direction > const& subset, bool unique ) const; + equation_matrix get_submatrix( std::set< node_id_with_direction > const& subset ) const; void process_node( branching_node* end_node, - bool compute_matrix, - const std::unordered_map< node_id_with_direction, int >& directions_in_path ); + bool compute_matrix, + const path_id_direction_count& directions_in_path, + bool add_columns ); void start_compute_matrix(); bool contains( node_id_with_direction const& node ) const; std::pair< std::size_t, std::size_t > get_dimensions() const; - std::map< equation, int > compute_vectors_with_hits(); + const std::unordered_map< equation, int >& compute_vectors_with_hits(); std::vector< equation >& get_matrix(); std::optional< equation > get_new_subset_counts_from_vectors( const std::vector< equation >& vector, const iid_node_generations_stats& state ); @@ -279,19 +310,21 @@ struct equation_matrix { BRANCHING_PREDICATE get_branching_predicate() const; private: - void add_path( branching_node* end_node, - const std::unordered_map< node_id_with_direction, int >& directions_in_path ); + void add_path( branching_node* end_node, const path_id_direction_count& directions_in_path, bool add_columns ); std::vector< equation > matrix; + std::unordered_set< equation > unique_rows; std::vector< branching_node* > all_paths; std::vector< node_id_with_direction > nodes; std::unordered_set< location_id::id_type > node_ids; + + std::unordered_map< equation, int > vectors_with_hits; + int computed_vectors = 0; }; struct iid_node_dependence_props { generated_path generate_probabilities( const loop_dependencies& loop_to_properties ); - void process_path_effective( branching_node* end_node, - const std::unordered_map< node_id_with_direction, int >& directions_in_path ); + void process_path_effective( branching_node* end_node, const path_id_direction_count& directions_in_path ); iid_node_generations_stats& get_generations_stats() { return stats; } const equation_matrix& get_matrix() const { return matrix; } const iid_node_generations_stats& get_generations_stats() const { return stats; } @@ -333,24 +366,25 @@ struct iid_node_dependence_props { const loop_dependencies& loop_to_properties, const std::unordered_set< location_id::id_type >& subset_ids ); nodes_to_counts compute_node_counts( const equation& path, - std::set< node_id_with_direction > const& all_leafs, + const std::set< node_id_with_direction >& all_leafs, const loop_dependencies& loop_to_properties, const std::unordered_set< location_id::id_type >& subset_ids ); - std::vector< equation > compute_best_vectors( const std::map< equation, int >& vectors_with_hits, + std::vector< equation > compute_best_vectors( const std::unordered_map< equation, int >& vectors_with_hits, int number_of_vectors, - bool use_random, int desired_direction, float biggest_branching_value ); - std::map< equation, int > get_linear_dependent_vector( const std::map< equation, int >& vectors_with_hits, - equation& best_vector ); - std::vector< equation > get_random_vector( const std::map< equation, int >& vectors_with_hits, - int number_of_vectors ); + std::unordered_map< equation, int > + get_linear_dependent_vector( const std::unordered_map< equation, int >& vectors_with_hits, + equation& best_vector ); generated_path generate_path_from_node_counts( const nodes_to_counts& path_counts, const loop_dependencies& loop_to_properties ); equation_matrix matrix; iid_node_generations_stats stats; + std::set< node_id_with_direction > computation_subset; + equation_matrix computation_submatrix; + bool matrix_generated = false; }; @@ -404,25 +438,11 @@ struct iid_dependencies { inline static int minimal_max_generation_artificial_data = 5; inline static float percentage_to_add_to_path = 0.4; inline static bool create_artificial_data = true; - inline static bool regenerate_all_data = true; inline static bool verbose = false; }; -std::unordered_map< node_id_with_direction, int > get_directions_in_path( branching_node* node ); +path_id_direction_count get_directions_in_path( branching_node* node ); bool should_generate_more_data( const generation_state& state ); } // namespace fuzzing - -namespace std -{ -template <> -struct hash< fuzzing::node_id_with_direction > { - std::size_t operator()( const fuzzing::node_id_with_direction& key ) const noexcept - { - std::size_t h1 = std::hash< location_id::id_type >{}( key.node_id ); - std::size_t h2 = std::hash< bool >{}( key.branching_direction ); - return h1 ^ ( h2 << 1 ); - } -}; -} // namespace std \ No newline at end of file diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 34ee9373..7d877f0f 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -312,28 +312,28 @@ auto node_id_with_direction::operator<=>( node_id_with_direction const& other ) // ------------------------------------------------------------------------------------------------ bool fuzzing::loop_properties::is_same( const std::unordered_set< location_id::id_type >& other_ids ) const { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); return all_ids == other_ids; } // ------------------------------------------------------------------------------------------------ const std::unordered_set< location_id::id_type >& fuzzing::loop_properties::get_all_ids() const { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); return all_ids; } // ------------------------------------------------------------------------------------------------ const std::unordered_set< location_id::id_type >& fuzzing::loop_properties::get_loop_head_ids() const { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); return loop_head_ids; } // ------------------------------------------------------------------------------------------------ location_id::id_type fuzzing::loop_properties::get_smallest_loop_head_id() const { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); location_id::id_type smallest_id = *loop_head_ids.begin(); @@ -349,7 +349,7 @@ location_id::id_type fuzzing::loop_properties::get_smallest_loop_head_id() const // ------------------------------------------------------------------------------------------------ location_id::id_type fuzzing::loop_properties::get_smallest_id() const { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); location_id::id_type smallest_id = *all_ids.begin(); for ( const auto& id : all_ids ) { @@ -364,7 +364,7 @@ location_id::id_type fuzzing::loop_properties::get_smallest_id() const // ------------------------------------------------------------------------------------------------ void fuzzing::loop_properties::update_stored_ids() { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); for ( const auto& [ head, props ] : heads ) { all_ids.insert( head.node_id ); @@ -386,7 +386,7 @@ void fuzzing::loop_properties::update_stored_ids() // ------------------------------------------------------------------------------------------------ void fuzzing::loop_properties::set_chosen_loop_head() { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); for ( const auto& [ head, props ] : heads ) { if ( !chosen_loop_head.has_value() ) { @@ -404,7 +404,7 @@ void fuzzing::loop_properties::set_chosen_loop_head() loop_properties& fuzzing::loop_dependencies::get_props( const std::unordered_set< location_id::id_type >& ids, location_id::id_type loop_head_id ) { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); for ( loop_properties& loop : loops ) { for ( const auto& [ head, props ] : loop.heads ) { @@ -425,7 +425,7 @@ loop_properties& fuzzing::loop_dependencies::get_props( const std::unordered_set // ------------------------------------------------------------------------------------------------ void fuzzing::loop_dependencies::merge_properties() { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); for ( loop_properties& loop : loops ) { loop.update_stored_ids(); @@ -506,12 +506,13 @@ std::set< location_id::id_type > fuzzing::loop_dependencies::get_loop_heads( boo } // ------------------------------------------------------------------------------------------------ -std::set< node_id_with_direction > fuzzing::loop_dependencies::get_node_subsets_for_computation( +bool fuzzing::loop_dependencies::compute_node_subsets_for_computation( + std::set< node_id_with_direction >& computation_subset, const std::unordered_set< location_id::id_type >& matrix_ids ) const { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); - std::set< node_id_with_direction > computation_subset; + bool changed = false; for ( const auto& loop : loops ) { if ( loop.is_loading_loop || !matrix_ids.contains( loop.get_smallest_loop_head_id() ) ) { @@ -519,15 +520,17 @@ std::set< node_id_with_direction > fuzzing::loop_dependencies::get_node_subsets_ } for ( const auto& body : loop.bodies ) { - computation_subset.insert( body ); + auto [ it, inserted ] = computation_subset.insert( body ); + changed |= inserted; } for ( const auto& [ head, _ ] : loop.heads ) { - computation_subset.insert( { head.node_id, !head.branching_direction } ); + auto [ it, inserted ] = computation_subset.insert( { head.node_id, !head.branching_direction } ); + changed |= inserted; } } - return computation_subset; + return changed; } // ------------------------------------------------------------------------------------------------ @@ -599,9 +602,9 @@ void fuzzing::loop_dependencies::print_dependencies() const // equation_matrix // ------------------------------------------------------------------------------------------------ -equation_matrix equation_matrix::get_submatrix( std::set< node_id_with_direction > const& subset, bool unique ) const +equation_matrix equation_matrix::get_submatrix( std::set< node_id_with_direction > const& subset ) const { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); equation_matrix result; result.nodes = std::vector< node_id_with_direction >( subset.begin(), subset.end() ); @@ -625,12 +628,8 @@ equation_matrix equation_matrix::get_submatrix( std::set< node_id_with_direction equation new_row = { new_row_values, row.best_value }; - if ( unique ) { - if ( std::find( result.matrix.begin(), result.matrix.end(), new_row ) == result.matrix.end() ) { - result.matrix.push_back( new_row ); - result.all_paths.push_back( all_paths[ i ] ); - } - } else { + auto [ it, inserted ] = result.unique_rows.insert( new_row ); + if ( inserted ) { result.matrix.push_back( new_row ); result.all_paths.push_back( all_paths[ i ] ); } @@ -642,9 +641,10 @@ equation_matrix equation_matrix::get_submatrix( std::set< node_id_with_direction // ------------------------------------------------------------------------------------------------ void fuzzing::equation_matrix::process_node( branching_node* end_node, bool compute_matrix, - const std::unordered_map< node_id_with_direction, int >& directions_in_path ) + const path_id_direction_count& directions_in_path, + bool add_columns ) { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); all_paths.push_back( end_node ); @@ -652,24 +652,24 @@ void fuzzing::equation_matrix::process_node( branching_node* end_node, return; } - add_path( end_node, directions_in_path ); + add_path( end_node, directions_in_path, add_columns ); } // ------------------------------------------------------------------------------------------------ void fuzzing::equation_matrix::start_compute_matrix() { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); for ( branching_node* node : all_paths ) { - std::unordered_map< node_id_with_direction, int > directions_in_path = get_directions_in_path( node ); - add_path( node, directions_in_path ); + path_id_direction_count directions_in_path = get_directions_in_path( node ); + add_path( node, directions_in_path, true ); } } // ------------------------------------------------------------------------------------------------ bool equation_matrix::contains( node_id_with_direction const& node ) const { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); auto it = std::find( nodes.begin(), nodes.end(), node ); return it != nodes.end(); @@ -682,13 +682,16 @@ std::pair< std::size_t, std::size_t > equation_matrix::get_dimensions() const } // ------------------------------------------------------------------------------------------------ -std::map< equation, int > equation_matrix::compute_vectors_with_hits() +const std::unordered_map< equation, int >& equation_matrix::compute_vectors_with_hits() { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); - std::map< equation, int > vectors_with_hits; + if ( computed_vectors == matrix.size() ) { + return vectors_with_hits; + } - for ( int i = 0; i < matrix.size(); ++i ) { + + for ( int i = computed_vectors; i < matrix.size(); ++i ) { for ( int j = 0; j < matrix.size(); ++j ) { if ( i == j ) continue; @@ -698,15 +701,19 @@ std::map< equation, int > equation_matrix::compute_vectors_with_hits() if ( difference.is_any_negative() || difference.best_value == 0 ) continue; - vectors_with_hits[ difference ] = 0; + vectors_with_hits[ difference ]; } - } + computed_vectors++; + } + for ( auto& [ vector, hits ] : vectors_with_hits ) { + hits = 0; + } for ( const auto& [ vector, hits ] : vectors_with_hits ) { for ( const auto& row : matrix ) { equation new_possible_equation = row + vector; - if ( std::find( matrix.begin(), matrix.end(), new_possible_equation ) != matrix.end() ) { + if ( std::find( unique_rows.begin(), unique_rows.end(), new_possible_equation ) != unique_rows.end() ) { vectors_with_hits[ vector ]++; } } @@ -721,6 +728,8 @@ std::vector< equation >& equation_matrix::get_matrix() { return matrix; } // ------------------------------------------------------------------------------------------------ int equation_matrix::get_desired_vector_direction() const { + TMPROF_BLOCK(); + auto is_positive = []( const equation& eq ) { return eq.best_value > 0; }; auto is_negative = []( const equation& eq ) { return eq.best_value < 0; }; @@ -736,6 +745,8 @@ int equation_matrix::get_desired_vector_direction() const // ------------------------------------------------------------------------------------------------ float equation_matrix::get_biggest_branching_value() const { + TMPROF_BLOCK(); + float biggest_value = 0.0f; for ( const equation& row : matrix ) { @@ -752,7 +763,7 @@ std::optional< equation > equation_matrix::get_new_subset_counts_from_vectors( const std::vector< equation >& vectors, const iid_node_generations_stats& stats ) { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); INVARIANT( !vectors.empty() ); INVARIANT( vectors[ 0 ].values.size() == nodes.size() ); @@ -843,76 +854,75 @@ BRANCHING_PREDICATE equation_matrix::get_branching_predicate() const // ------------------------------------------------------------------------------------------------ void fuzzing::equation_matrix::add_path( branching_node* end_node, - const std::unordered_map< node_id_with_direction, int >& directions_in_path ) + const path_id_direction_count& directions_in_path, + bool add_columns ) { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); - int added_nodes = 0; + if ( add_columns ) { + int added_nodes = 0; - for ( const auto& [ nav, count ] : directions_in_path ) { - auto [ it, inserted ] = node_ids.insert( nav.node_id ); - if ( !inserted ) { - continue; - } + for ( size_t i = 0; i < directions_in_path.size(); ++i ) { + if ( directions_in_path[ i ] == 0 ) { + continue; + } + + int id = i / 2; + bool direction = i % 2; - for ( bool direction : { true, false } ) { - nodes.push_back( { nav.node_id, direction } ); - added_nodes++; + auto [ it, inserted ] = node_ids.insert( id ); + if ( !inserted ) { + continue; + } + + for ( bool direction : { true, false } ) { + nodes.push_back( { location_id::id_type( id ), direction } ); + added_nodes++; + } } - } - for ( int i = 0; i < added_nodes; ++i ) { - for ( int j = 0; j < matrix.size(); ++j ) { - matrix[ j ].values.push_back( 0 ); + for ( int i = 0; i < added_nodes; ++i ) { + for ( int j = 0; j < matrix.size(); ++j ) { + matrix[ j ].values.push_back( 0 ); + } } } std::vector< int > values_in_path; for ( const auto& node : nodes ) { - auto it = directions_in_path.find( node ); - if ( it == directions_in_path.end() ) { - values_in_path.push_back( 0 ); - } else { - values_in_path.push_back( it->second ); - } + int index = 2 * node.node_id + ( node.branching_direction ? 1 : 0 ); + values_in_path.push_back( directions_in_path[ index ] ); } equation row = { values_in_path, end_node->best_coverage_value }; - matrix.push_back( row ); + + auto [ it, inserted ] = unique_rows.insert( row ); + if ( inserted ) { + matrix.push_back( row ); + } } // iid_node_dependence_props // ------------------------------------------------------------------------------------------------ generated_path iid_node_dependence_props::generate_probabilities( const loop_dependencies& loop_to_properties ) { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); stats.method_calls++; - if ( iid_dependencies::verbose ) { - // print_stats( true ); - loop_to_properties.print_dependencies(); - matrix.print_matrix(); - } - if ( !matrix_generated ) { matrix_generated = true; matrix.start_compute_matrix(); } if ( loop_to_properties.loops.empty() ) { - if ( iid_dependencies::verbose ) - std::cout << "No loops" << std::endl; - return {}; } - std::set< node_id_with_direction > computation_subset = - loop_to_properties.get_node_subsets_for_computation( matrix.get_node_ids() ); - if ( computation_subset.empty() ) { - if ( iid_dependencies::verbose ) - std::cout << "No nodes in subset" << std::endl; + bool subset_changed = loop_to_properties.compute_node_subsets_for_computation( computation_subset, + matrix.get_node_ids() ); + if ( computation_subset.empty() ) { return {}; } @@ -922,8 +932,12 @@ generated_path iid_node_dependence_props::generate_probabilities( const loop_dep } stats.generation_starts++; - equation_matrix submatrix = matrix.get_submatrix( computation_subset, true ); - if ( submatrix.empty() ) { + + if ( subset_changed ) { + computation_submatrix = matrix.get_submatrix( computation_subset ); + } + + if ( computation_submatrix.empty() ) { if ( iid_dependencies::verbose ) std::cout << "Empty submatrix" << std::endl; @@ -931,10 +945,11 @@ generated_path iid_node_dependence_props::generate_probabilities( const loop_dep } if ( iid_dependencies::verbose ) { - submatrix.print_matrix(); + loop_to_properties.print_dependencies(); + computation_submatrix.print_matrix(); } - std::optional< std::vector< equation > > best_vectors = get_best_vectors( submatrix, 1 ); + std::optional< std::vector< equation > > best_vectors = get_best_vectors( computation_submatrix, 1 ); if ( !best_vectors.has_value() ) { if ( stats.state != generation_state::STATE_GENERATING_ARTIFICIAL_DATA ) { @@ -942,11 +957,11 @@ generated_path iid_node_dependence_props::generate_probabilities( const loop_dep } best_vectors = std::vector< equation >(); - generate_vectors_if_not_enough_data( *best_vectors, submatrix ); + generate_vectors_if_not_enough_data( *best_vectors, computation_submatrix ); } - std::optional< equation > new_subset_counts = submatrix.get_new_subset_counts_from_vectors( *best_vectors, - stats ); + std::optional< equation > new_subset_counts = + computation_submatrix.get_new_subset_counts_from_vectors( *best_vectors, stats ); if ( !new_subset_counts.has_value() ) { return return_empty_path(); @@ -965,12 +980,15 @@ generated_path iid_node_dependence_props::generate_probabilities( const loop_dep // ------------------------------------------------------------------------------------------------ -void fuzzing::iid_node_dependence_props::process_path_effective( - branching_node* end_node, - const std::unordered_map< node_id_with_direction, int >& directions_in_path ) +void fuzzing::iid_node_dependence_props::process_path_effective( branching_node* end_node, + const path_id_direction_count& directions_in_path ) { - // TMPROF_BLOCK(); - matrix.process_node( end_node, matrix_generated, directions_in_path ); + TMPROF_BLOCK(); + matrix.process_node( end_node, matrix_generated, directions_in_path, true ); + + if ( !computation_submatrix.empty() ) { + computation_submatrix.process_node( end_node, true, directions_in_path, false ); + } } // ------------------------------------------------------------------------------------------------ @@ -1098,11 +1116,11 @@ void iid_node_dependence_props::print_stats( bool only_state ) const void fuzzing::iid_node_dependence_props::generate_vectors_if_not_enough_data( std::vector< equation >& best_vectors, equation_matrix& submatrix ) { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); best_vectors = std::vector< equation >(); int desired_direction = submatrix.get_desired_vector_direction(); - std::map< equation, int > vectors = submatrix.compute_vectors_with_hits(); + const std::unordered_map< equation, int >& vectors = submatrix.compute_vectors_with_hits(); auto add_to_best_vectors = [ & ]( std::vector< int > values ) { if ( desired_direction == 0 ) { @@ -1135,9 +1153,9 @@ void fuzzing::iid_node_dependence_props::generate_vectors_if_not_enough_data( st std::optional< std::vector< equation > > fuzzing::iid_node_dependence_props::get_best_vectors( equation_matrix& submatrix, int number_of_vectors ) { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); - std::map< equation, int > vectors = submatrix.compute_vectors_with_hits(); + const std::unordered_map< equation, int >& vectors = submatrix.compute_vectors_with_hits(); if ( vectors.empty() ) { return std::nullopt; } @@ -1145,7 +1163,7 @@ fuzzing::iid_node_dependence_props::get_best_vectors( equation_matrix& submatrix int desired_vector_direction = submatrix.get_desired_vector_direction(); float biggest_branching_value = submatrix.get_biggest_branching_value(); std::vector< equation > best_vectors = - compute_best_vectors( vectors, number_of_vectors, false, desired_vector_direction, biggest_branching_value ); + compute_best_vectors( vectors, number_of_vectors, desired_vector_direction, biggest_branching_value ); if ( best_vectors.empty() ) { return std::nullopt; @@ -1210,7 +1228,7 @@ void iid_node_dependence_props::compute_node_counts_for_nested_loops( const loop_dependencies& loop_to_properties, bool use_random ) { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); INVARIANT( !child_loop_counts.empty() ); @@ -1276,7 +1294,7 @@ int fuzzing::iid_node_dependence_props::compute_loading_loop_interation( const loop_properties& props, const loop_dependencies& loop_to_properties ) { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); float loaded_per_loop = props.loaded_bits_per_loop.mean; if ( loaded_per_loop <= 0 ) { @@ -1310,7 +1328,7 @@ void iid_node_dependence_props::compute_node_counts_for_loading_loops( nodes_to_ const std::set< location_id::id_type >& loop_heads, const loop_dependencies& loop_to_properties ) { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); for ( const auto& loop_props : std::ranges::views::reverse( loop_to_properties.loops ) ) { if ( !loop_props.is_loading_loop ) { @@ -1377,7 +1395,7 @@ void iid_node_dependence_props::compute_node_counts_for_loops( nodes_to_counts& const loop_dependencies& loop_to_properties, const std::unordered_set< location_id::id_type >& subset_ids ) { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); for ( const auto& props : std::ranges::views::reverse( loop_to_properties.loops ) ) { if ( props.bodies.empty() || props.is_loading_loop || @@ -1418,7 +1436,7 @@ iid_node_dependence_props::compute_node_counts( const equation& path, const loop_dependencies& loop_to_properties, const std::unordered_set< location_id::id_type >& subset_ids ) { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); nodes_to_counts path_counts; @@ -1465,19 +1483,19 @@ iid_node_dependence_props::compute_node_counts( const equation& path, } // ------------------------------------------------------------------------------------------------ -std::vector< equation > iid_node_dependence_props::compute_best_vectors( const std::map< equation, int >& vectors_with_hits, - int number_of_vectors, - bool use_random, - int desired_direction, - float biggest_branching_value ) +std::vector< equation > +iid_node_dependence_props::compute_best_vectors( const std::unordered_map< equation, int >& vectors_with_hits, + int number_of_vectors, + int desired_direction, + float biggest_branching_value ) { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); if ( vectors_with_hits.empty() ) { throw std::invalid_argument( "Input map is empty." ); } - std::map< equation, int > filtered_vectors_with_hits; + std::unordered_map< equation, int > filtered_vectors_with_hits; if ( desired_direction < 0 ) { std::copy_if( vectors_with_hits.begin(), vectors_with_hits.end(), @@ -1501,10 +1519,6 @@ std::vector< equation > iid_node_dependence_props::compute_best_vectors( const s return {}; } - if ( use_random ) { - return get_random_vector( filtered_vectors_with_hits, number_of_vectors ); - } - std::vector< std::pair< equation, int > > sorted_vectors( filtered_vectors_with_hits.begin(), filtered_vectors_with_hits.end() ); @@ -1519,7 +1533,7 @@ std::vector< equation > iid_node_dependence_props::compute_best_vectors( const s std::vector< equation > best_vectors; for ( int i = 0; i < number_of_vectors && i < sorted_vectors.size(); ++i ) { if ( use_linear_dependency ) { - std::map< equation, int > dependent_vectors_with_hits = + std::unordered_map< equation, int > dependent_vectors_with_hits = get_linear_dependent_vector( filtered_vectors_with_hits, sorted_vectors[ i ].first ); auto it = std::min_element( dependent_vectors_with_hits.begin(), @@ -1537,11 +1551,11 @@ std::vector< equation > iid_node_dependence_props::compute_best_vectors( const s } // ------------------------------------------------------------------------------------------------ -std::map< equation, int > -iid_node_dependence_props::get_linear_dependent_vector( const std::map< equation, int >& vectors_with_hits, - equation& best_vector ) +std::unordered_map< equation, int > iid_node_dependence_props::get_linear_dependent_vector( + const std::unordered_map< equation, int >& vectors_with_hits, + equation& best_vector ) { - std::map< equation, int > dependent_vectors_with_hits; + std::unordered_map< equation, int > dependent_vectors_with_hits; for ( const auto& [ vector, hits ] : vectors_with_hits ) { if ( best_vector.is_linear_dependent( vector ) ) { @@ -1553,45 +1567,11 @@ iid_node_dependence_props::get_linear_dependent_vector( const std::map< equation return dependent_vectors_with_hits; } -// ------------------------------------------------------------------------------------------------ -std::vector< equation > iid_node_dependence_props::get_random_vector( const std::map< equation, int >& vectors_with_hits, - int number_of_vectors ) -{ - std::vector< equation > equations; - std::vector< double > probabilities; - - int total_hits = std::accumulate( vectors_with_hits.begin(), - vectors_with_hits.end(), - 0, - []( int sum, const auto& pair ) { return sum + pair.second; } ); - - if ( total_hits == 0 ) { - throw std::invalid_argument( "Total hits is zero." ); - } - - for ( const auto& [ eq, hits ] : vectors_with_hits ) { - equations.push_back( eq ); - probabilities.push_back( static_cast< double >( hits ) / total_hits ); - } - - std::random_device rd; - std::mt19937 gen( rd() ); - std::discrete_distribution<> dist( probabilities.begin(), probabilities.end() ); - - std::vector< equation > selected_equations; - for ( int i = 0; i < number_of_vectors; ++i ) { - selected_equations.push_back( equations[ dist( gen ) ] ); - } - - return selected_equations; -} - - // ------------------------------------------------------------------------------------------------ generated_path iid_node_dependence_props::generate_path_from_node_counts( const nodes_to_counts& path_counts, const loop_dependencies& loop_to_properties ) { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); std::map< location_id::id_type, node_props_in_path > path; for ( const auto& [ id, counts ] : path_counts ) { @@ -1635,7 +1615,7 @@ void iid_dependencies::update_ignored_nodes( sensitivity_analysis& sensitivity ) // ------------------------------------------------------------------------------------------------ void iid_dependencies::process_node( branching_node* end_node ) { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); end_nodes.push_back( end_node ); } @@ -1698,7 +1678,7 @@ std::vector< location_id > iid_dependencies::get_iid_nodes() // ------------------------------------------------------------------------------------------------ std::optional< location_id > iid_dependencies::get_next_iid_node() { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); bool previous_needs_more_data = false; for ( auto it = node_id_to_equation_map.rbegin(); it != node_id_to_equation_map.rend(); ++it ) { @@ -1733,13 +1713,11 @@ std::optional< location_id > iid_dependencies::get_next_iid_node() // ------------------------------------------------------------------------------------------------ void fuzzing::iid_dependencies::compute_dependencies() { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); dependencies_computed++; for ( branching_node* end_node : end_nodes ) { - // TMPROF_BLOCK(); - loop_head_to_bodies_t loop_heads_to_bodies; loop_endings loop_heads_ending = get_loop_heads_ending( end_node, loop_heads_to_bodies ); @@ -1759,7 +1737,7 @@ void fuzzing::iid_dependencies::compute_dependencies() // ------------------------------------------------------------------------------------------------ generated_path fuzzing::iid_dependencies::generate_probabilities() { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); std::optional< location_id > id = get_next_iid_node(); if ( !id.has_value() ) { return {}; @@ -1771,7 +1749,7 @@ generated_path fuzzing::iid_dependencies::generate_probabilities() iid_node_dependence_props& props = get_props( id.value() ); generated_path path = props.generate_probabilities( loop_to_properties ); - + if ( !path.empty() ) { path.set_iid_node_id( id.value() ); } @@ -1812,7 +1790,7 @@ iid_vector_analysis_statistics fuzzing::iid_dependencies::get_stats() const loop_endings fuzzing::iid_dependencies::get_loop_heads_ending( branching_node* end_node, loop_head_to_bodies_t& loop_heads_to_bodies ) { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); std::vector< fuzzer::loop_boundary_props > loops; fuzzer::detect_loops_along_path_to_node( end_node, loop_heads_to_bodies, &loops ); @@ -1856,7 +1834,7 @@ void fuzzing::iid_dependencies::compute_dependencies_by_loading( branching_node* const loop_head_to_bodies_t& loop_heads_to_bodies, const loop_endings& loop_heads_ending ) { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); loop_head_to_loaded_bits_counter loading_loops; compute_loading_loops( end_node, loop_heads_to_bodies, loading_loops, loop_heads_ending ); @@ -1954,7 +1932,7 @@ void fuzzing::iid_dependencies::compute_dependencies_by_loading( branching_node* void fuzzing::iid_dependencies::compute_dependencies_by_loops( const loop_head_to_bodies_t& loop_heads_to_bodies, const loop_endings& loop_heads_ending ) { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); bool changed = false; @@ -2020,7 +1998,7 @@ void fuzzing::iid_dependencies::compute_loading_loops( branching_node* end_node, loop_head_to_loaded_bits_counter& loading_loops, const loop_endings& loop_heads_ending ) { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { loading_loops[ loop_head.id ] = { std::numeric_limits< natural_32_bit >::max(), @@ -2070,7 +2048,7 @@ void fuzzing::iid_dependencies::compute_loading_loops( branching_node* end_node, // ------------------------------------------------------------------------------------------------ void fuzzing::iid_dependencies::compute_paths( branching_node* end_node ) { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); INVARIANT( end_node != nullptr ); @@ -2080,13 +2058,15 @@ void fuzzing::iid_dependencies::compute_paths( branching_node* end_node ) } std::vector< node_id_with_direction > full_path = get_path( end_node ); - std::unordered_map< node_id_with_direction, int > directions_in_path; + path_id_direction_count directions_in_path = {}; for ( auto it = full_path.rbegin(); it != full_path.rend(); ++it ) { processed_nodes++; const auto& path_node = *it; - directions_in_path[ path_node ]++; + int index = 2 * path_node.node_id + ( path_node.branching_direction ? 1 : 0 ); + directions_in_path[ index ]++; + current_node = current_node->successor( path_node.branching_direction ).pointer; location_id current_node_id = current_node->get_location_id(); @@ -2102,7 +2082,7 @@ void fuzzing::iid_dependencies::compute_paths( branching_node* end_node ) // ------------------------------------------------------------------------------------------------ std::vector< node_id_with_direction > fuzzing::iid_dependencies::get_path( branching_node* node ) { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); std::vector< node_id_with_direction > result; @@ -2133,19 +2113,21 @@ bool fuzzing::iid_dependencies::is_tracked( location_id id ) const // non member functions // ------------------------------------------------------------------------------------------------ -std::unordered_map< node_id_with_direction, int > get_directions_in_path( branching_node* node ) +path_id_direction_count get_directions_in_path( branching_node* node ) { - // TMPROF_BLOCK(); + TMPROF_BLOCK(); - std::unordered_map< node_id_with_direction, int > result; + path_id_direction_count result = {}; branching_node* current = node; while ( current != nullptr ) { branching_node* predecessor = current->predecessor; if ( predecessor != nullptr ) { - node_id_with_direction nav = { predecessor->get_location_id().id, - predecessor->successor_direction( current ) }; - result[ nav ]++; + int id = predecessor->get_location_id().id; + bool direction = predecessor->successor_direction( current ); + int index = 2 * id + ( direction ? 1 : 0 ); + + result[ index ]++; } current = predecessor; From 9a2052d2a774228f859101baea53f0d1075fc299 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Tue, 18 Mar 2025 15:16:53 +0100 Subject: [PATCH 128/144] feat: improvement of path_id_direction_count --- .../include/fuzzing/iid_vector_analysis.hpp | 18 ++++-- src/fuzzing/src/fuzzer.cpp | 6 ++ src/fuzzing/src/iid_vector_analysis.cpp | 62 +++++++++++-------- 3 files changed, 54 insertions(+), 32 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index e0d28ef8..2c30f9b2 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -281,7 +281,7 @@ struct loaded_bits_counter { int loop_count; }; -using path_id_direction_count = std::array< location_id::id_type, 20000 >; +using path_id_direction_count = std::vector< location_id::id_type >; using loop_head_to_loaded_bits_counter = std::unordered_map< location_id::id_type, loaded_bits_counter >; using loop_endings = std::unordered_map< location_id::id_type, bool >; using loop_head_to_bodies_t = std::unordered_map< location_id, std::unordered_set< location_id > >; @@ -292,7 +292,8 @@ struct equation_matrix { void process_node( branching_node* end_node, bool compute_matrix, const path_id_direction_count& directions_in_path, - bool add_columns ); + bool add_columns, + std::size_t max_directions_in_path_index ); void start_compute_matrix(); bool contains( node_id_with_direction const& node ) const; std::pair< std::size_t, std::size_t > get_dimensions() const; @@ -310,7 +311,10 @@ struct equation_matrix { BRANCHING_PREDICATE get_branching_predicate() const; private: - void add_path( branching_node* end_node, const path_id_direction_count& directions_in_path, bool add_columns ); + void add_path( branching_node* end_node, + const path_id_direction_count& directions_in_path, + bool add_columns, + std::size_t max_directions_in_path_index ); std::vector< equation > matrix; std::unordered_set< equation > unique_rows; @@ -324,7 +328,9 @@ struct equation_matrix { struct iid_node_dependence_props { generated_path generate_probabilities( const loop_dependencies& loop_to_properties ); - void process_path_effective( branching_node* end_node, const path_id_direction_count& directions_in_path ); + void process_path_effective( branching_node* end_node, + const path_id_direction_count& directions_in_path, + std::size_t max_directions_in_path_index ); iid_node_generations_stats& get_generations_stats() { return stats; } const equation_matrix& get_matrix() const { return matrix; } const iid_node_generations_stats& get_generations_stats() const { return stats; } @@ -428,6 +434,8 @@ struct iid_dependencies { std::vector< branching_node* > end_nodes; public: + inline static std::size_t biggest_node_id = 0; + // Configurations inline static bool random_nested_loop_counts = false; inline static bool random_direction_in_path = true; @@ -443,6 +451,6 @@ struct iid_dependencies { }; -path_id_direction_count get_directions_in_path( branching_node* node ); +std::pair< path_id_direction_count, std::size_t > get_directions_in_path( branching_node* node ); bool should_generate_more_data( const generation_state& state ); } // namespace fuzzing diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index 98f7e9a9..3de8c23d 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -10,6 +10,7 @@ #include #include +// constexpr bool use_vector_analysis = false; constexpr bool use_vector_analysis = true; namespace fuzzing { @@ -1220,6 +1221,11 @@ execution_record::execution_flags fuzzer::process_execution_results() succ_info.predicate ); + if ( use_vector_analysis ) { + iid_dependencies::biggest_node_id = std::max( iid_dependencies::biggest_node_id, + std::size_t( new_node->id.id ) ); + } + construction_props.leaf->set_successor(info.direction, { branching_node::successor_pointer::VISITED, new_node diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 7d877f0f..a3fb943e 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -642,7 +642,8 @@ equation_matrix equation_matrix::get_submatrix( std::set< node_id_with_direction void fuzzing::equation_matrix::process_node( branching_node* end_node, bool compute_matrix, const path_id_direction_count& directions_in_path, - bool add_columns ) + bool add_columns, + std::size_t max_directions_in_path_index ) { TMPROF_BLOCK(); @@ -652,7 +653,7 @@ void fuzzing::equation_matrix::process_node( branching_node* end_node, return; } - add_path( end_node, directions_in_path, add_columns ); + add_path( end_node, directions_in_path, add_columns, max_directions_in_path_index ); } // ------------------------------------------------------------------------------------------------ @@ -661,8 +662,8 @@ void fuzzing::equation_matrix::start_compute_matrix() TMPROF_BLOCK(); for ( branching_node* node : all_paths ) { - path_id_direction_count directions_in_path = get_directions_in_path( node ); - add_path( node, directions_in_path, true ); + auto [ directions_in_path, max_directions_in_path_index ] = get_directions_in_path( node ); + add_path( node, directions_in_path, true, max_directions_in_path_index ); } } @@ -690,7 +691,6 @@ const std::unordered_map< equation, int >& equation_matrix::compute_vectors_with return vectors_with_hits; } - for ( int i = computed_vectors; i < matrix.size(); ++i ) { for ( int j = 0; j < matrix.size(); ++j ) { if ( i == j ) @@ -855,14 +855,16 @@ BRANCHING_PREDICATE equation_matrix::get_branching_predicate() const // ------------------------------------------------------------------------------------------------ void fuzzing::equation_matrix::add_path( branching_node* end_node, const path_id_direction_count& directions_in_path, - bool add_columns ) + bool add_columns, + std::size_t max_directions_in_path_index ) { TMPROF_BLOCK(); if ( add_columns ) { int added_nodes = 0; + INVARIANT( max_directions_in_path_index < directions_in_path.size() ); - for ( size_t i = 0; i < directions_in_path.size(); ++i ) { + for ( size_t i = 0; i < max_directions_in_path_index + 1; ++i ) { if ( directions_in_path[ i ] == 0 ) { continue; } @@ -888,10 +890,10 @@ void fuzzing::equation_matrix::add_path( branching_node* end_node, } } - std::vector< int > values_in_path; - for ( const auto& node : nodes ) { - int index = 2 * node.node_id + ( node.branching_direction ? 1 : 0 ); - values_in_path.push_back( directions_in_path[ index ] ); + std::vector< int > values_in_path( nodes.size() ); + for ( size_t i = 0; i < nodes.size(); ++i ) { + int index = 2 * nodes[ i ].node_id + ( nodes[ i ].branching_direction ? 1 : 0 ); + values_in_path[ i ] = directions_in_path[ index ]; } equation row = { values_in_path, end_node->best_coverage_value }; @@ -938,15 +940,13 @@ generated_path iid_node_dependence_props::generate_probabilities( const loop_dep } if ( computation_submatrix.empty() ) { - if ( iid_dependencies::verbose ) - std::cout << "Empty submatrix" << std::endl; - return return_empty_path(); } if ( iid_dependencies::verbose ) { + print_stats( true ); loop_to_properties.print_dependencies(); - computation_submatrix.print_matrix(); + // computation_submatrix.print_matrix(); } std::optional< std::vector< equation > > best_vectors = get_best_vectors( computation_submatrix, 1 ); @@ -981,13 +981,14 @@ generated_path iid_node_dependence_props::generate_probabilities( const loop_dep // ------------------------------------------------------------------------------------------------ void fuzzing::iid_node_dependence_props::process_path_effective( branching_node* end_node, - const path_id_direction_count& directions_in_path ) + const path_id_direction_count& directions_in_path, + std::size_t max_directions_in_path_index ) { TMPROF_BLOCK(); - matrix.process_node( end_node, matrix_generated, directions_in_path, true ); + matrix.process_node( end_node, matrix_generated, directions_in_path, true, max_directions_in_path_index ); if ( !computation_submatrix.empty() ) { - computation_submatrix.process_node( end_node, true, directions_in_path, false ); + computation_submatrix.process_node( end_node, true, directions_in_path, false, max_directions_in_path_index ); } } @@ -2058,15 +2059,18 @@ void fuzzing::iid_dependencies::compute_paths( branching_node* end_node ) } std::vector< node_id_with_direction > full_path = get_path( end_node ); - path_id_direction_count directions_in_path = {}; + path_id_direction_count directions_in_path( iid_dependencies::biggest_node_id * 2 + 2 ); + std::size_t max_directions_in_path_index = 0; for ( auto it = full_path.rbegin(); it != full_path.rend(); ++it ) { processed_nodes++; const auto& path_node = *it; - int index = 2 * path_node.node_id + ( path_node.branching_direction ? 1 : 0 ); + std::size_t index = 2 * path_node.node_id + ( path_node.branching_direction ? 1 : 0 ); directions_in_path[ index ]++; + max_directions_in_path_index = std::max( max_directions_in_path_index, index ); + current_node = current_node->successor( path_node.branching_direction ).pointer; location_id current_node_id = current_node->get_location_id(); @@ -2075,19 +2079,19 @@ void fuzzing::iid_dependencies::compute_paths( branching_node* end_node ) } iid_node_dependence_props& props = node_id_to_equation_map[ current_node_id ]; - props.process_path_effective( current_node, directions_in_path ); + props.process_path_effective( current_node, directions_in_path, max_directions_in_path_index ); } } // ------------------------------------------------------------------------------------------------ -std::vector< node_id_with_direction > fuzzing::iid_dependencies::get_path( branching_node* node ) +std::vector< node_id_with_direction > fuzzing::iid_dependencies::get_path( branching_node* end_node ) { TMPROF_BLOCK(); std::vector< node_id_with_direction > result; bool iid_seen = false; - branching_node* current = node; + branching_node* current = end_node; while ( current != nullptr ) { branching_node* predecessor = current->predecessor; @@ -2108,16 +2112,19 @@ std::vector< node_id_with_direction > fuzzing::iid_dependencies::get_path( branc // ------------------------------------------------------------------------------------------------ bool fuzzing::iid_dependencies::is_tracked( location_id id ) const { + TMPROF_BLOCK(); + return !ignored_node_ids.contains( id.id ) && !covered_node_ids.contains( id ); } // non member functions // ------------------------------------------------------------------------------------------------ -path_id_direction_count get_directions_in_path( branching_node* node ) +std::pair< path_id_direction_count, std::size_t > get_directions_in_path( branching_node* node ) { TMPROF_BLOCK(); - path_id_direction_count result = {}; + path_id_direction_count result( iid_dependencies::biggest_node_id * 2 + 2 ); + std::size_t max_directions_in_path_index = 0; branching_node* current = node; while ( current != nullptr ) { @@ -2125,15 +2132,16 @@ path_id_direction_count get_directions_in_path( branching_node* node ) if ( predecessor != nullptr ) { int id = predecessor->get_location_id().id; bool direction = predecessor->successor_direction( current ); - int index = 2 * id + ( direction ? 1 : 0 ); + std::size_t index = 2 * id + ( direction ? 1 : 0 ); result[ index ]++; + max_directions_in_path_index = std::max( max_directions_in_path_index, index ); } current = predecessor; } - return result; + return { result, max_directions_in_path_index }; } // ------------------------------------------------------------------------------------------------ From 8184671025b926c84ffa09caa17a7ff442e435dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Fri, 28 Mar 2025 11:58:09 +0100 Subject: [PATCH 129/144] feat: add FloatComparator and enhance generation state management --- .../include/fuzzing/iid_vector_analysis.hpp | 15 +++++- src/fuzzing/src/iid_vector_analysis.cpp | 54 ++++++++++++++++--- 2 files changed, 62 insertions(+), 7 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index 2c30f9b2..6193050a 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -40,7 +40,9 @@ enum generation_state { STATE_GENERATING_ARTIFICIAL_DATA, STATE_GENERATION_MORE, STATE_COVERED, - STATE_GENERATION_DATA_FOR_NEXT_NODE + STATE_GENERATION_DATA_FOR_NEXT_NODE, + + STATE_COVERED_BY_OTHER }; enum failed_generation_method { METHOD_GENERATE_FROM_OTHER_NODE, METHOD_GENERATE_ARTIFICIAL_DATA }; @@ -281,6 +283,15 @@ struct loaded_bits_counter { int loop_count; }; +struct FloatComparator { + bool operator()( const float& a, const float& b ) const + { + const float epsilon = 1e-6f; + return std::abs( a - b ) > epsilon && std::abs( a ) < std::abs( b ); + } +}; + + using path_id_direction_count = std::vector< location_id::id_type >; using loop_head_to_loaded_bits_counter = std::unordered_map< location_id::id_type, loaded_bits_counter >; using loop_endings = std::unordered_map< location_id::id_type, bool >; @@ -316,6 +327,7 @@ struct equation_matrix { bool add_columns, std::size_t max_directions_in_path_index ); + std::map< double, int, FloatComparator > branching_values; std::vector< equation > matrix; std::unordered_set< equation > unique_rows; std::vector< branching_node* > all_paths; @@ -444,6 +456,7 @@ struct iid_dependencies { inline static int minimal_max_generation_after_covered = 10; inline static int minimal_max_generation_for_other_node = 10; inline static int minimal_max_generation_artificial_data = 5; + inline static int maximal_number_of_equations_with_same_branching_value = 2000; inline static float percentage_to_add_to_path = 0.4; inline static bool create_artificial_data = true; diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index a3fb943e..1b3bdbe2 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -527,6 +527,12 @@ bool fuzzing::loop_dependencies::compute_node_subsets_for_computation( for ( const auto& [ head, _ ] : loop.heads ) { auto [ it, inserted ] = computation_subset.insert( { head.node_id, !head.branching_direction } ); changed |= inserted; + + auto it_head = computation_subset.find( head ); + if ( it_head != computation_subset.end() ) { + computation_subset.erase( it_head ); + changed = true; + } } } @@ -647,6 +653,12 @@ void fuzzing::equation_matrix::process_node( branching_node* end_node, { TMPROF_BLOCK(); + int& branching_value_count = branching_values[ end_node->best_coverage_value ]; + if ( branching_value_count >= iid_dependencies::maximal_number_of_equations_with_same_branching_value ) + return; + + branching_value_count++; + all_paths.push_back( end_node ); if ( !compute_matrix ) { @@ -706,9 +718,11 @@ const std::unordered_map< equation, int >& equation_matrix::compute_vectors_with computed_vectors++; } + for ( auto& [ vector, hits ] : vectors_with_hits ) { hits = 0; } + for ( const auto& [ vector, hits ] : vectors_with_hits ) { for ( const auto& row : matrix ) { equation new_possible_equation = row + vector; @@ -918,6 +932,9 @@ generated_path iid_node_dependence_props::generate_probabilities( const loop_dep } if ( loop_to_properties.loops.empty() ) { + if ( iid_dependencies::verbose ) + std::cout << "No loops" << std::endl; + return {}; } @@ -925,6 +942,9 @@ generated_path iid_node_dependence_props::generate_probabilities( const loop_dep matrix.get_node_ids() ); if ( computation_subset.empty() ) { + if ( iid_dependencies::verbose ) + std::cout << "No nodes to compute" << std::endl; + return {}; } @@ -940,11 +960,14 @@ generated_path iid_node_dependence_props::generate_probabilities( const loop_dep } if ( computation_submatrix.empty() ) { + if ( iid_dependencies::verbose ) + std::cout << "Empty submatrix" << std::endl; + return return_empty_path(); } if ( iid_dependencies::verbose ) { - print_stats( true ); + print_stats( false ); loop_to_properties.print_dependencies(); // computation_submatrix.print_matrix(); } @@ -953,6 +976,9 @@ generated_path iid_node_dependence_props::generate_probabilities( const loop_dep if ( !best_vectors.has_value() ) { if ( stats.state != generation_state::STATE_GENERATING_ARTIFICIAL_DATA ) { + if ( iid_dependencies::verbose ) + std::cout << "No vectors" << std::endl; + return return_empty_path(); } @@ -964,6 +990,9 @@ generated_path iid_node_dependence_props::generate_probabilities( const loop_dep computation_submatrix.get_new_subset_counts_from_vectors( *best_vectors, stats ); if ( !new_subset_counts.has_value() ) { + if ( iid_dependencies::verbose ) + std::cout << "No new subset counts" << std::endl; + return return_empty_path(); } @@ -1023,7 +1052,9 @@ bool iid_node_dependence_props::too_much_failed_in_row( int max_failed_generatio // ------------------------------------------------------------------------------------------------ void iid_node_dependence_props::set_as_generating_for_other_node( int minimal_max_generation_for_other_node ) { - INVARIANT( stats.state == generation_state::STATE_COVERED ); + if ( stats.state != generation_state::STATE_COVERED ) { + return; + } stats.state = generation_state::STATE_GENERATION_DATA_FOR_NEXT_NODE; stats.generated_for_other_node_max = minimal_max_generation_for_other_node; @@ -1110,6 +1141,9 @@ void iid_node_dependence_props::print_stats( bool only_state ) const << stats.generate_artificial_data_max << std::endl; } break; + case generation_state::STATE_COVERED_BY_OTHER: + std::cout << "Status: STATE_COVERED_BY_OTHER" << std::endl; + break; } } @@ -1651,10 +1685,11 @@ void fuzzing::iid_dependencies::remove_all_covered( const std::unordered_set< lo if ( covered_ids.contains( it->first ) && it->second.get_generations_stats().state == generation_state::STATE_NOT_COVERED ) { covered_node_ids.insert( it->first ); - it = node_id_to_equation_map.erase( it ); - } else { - ++it; + iid_node_generations_stats& stats = it->second.get_generations_stats(); + stats.state = generation_state::STATE_COVERED_BY_OTHER; } + + ++it; } } @@ -1741,6 +1776,9 @@ generated_path fuzzing::iid_dependencies::generate_probabilities() TMPROF_BLOCK(); std::optional< location_id > id = get_next_iid_node(); if ( !id.has_value() ) { + if ( verbose ) + std::cout << "No more nodes to generate probabilities for." << std::endl; + return {}; } @@ -1914,6 +1952,10 @@ void fuzzing::iid_dependencies::compute_dependencies_by_loading( branching_node* for ( const auto& [ body_id, props ] : body ) { auto& body_props = dependencies.bits_read_by_node[ body_id ]; natural_32_bit minimal_offset = props.min - loading_props.min; + if ( props.min < loading_props.min ) { + minimal_offset = 0; + } + INVARIANT( minimal_offset >= 0 ); body_props.minimal_bit_offset = std::min( body_props.minimal_bit_offset, minimal_offset ); @@ -1971,7 +2013,7 @@ void fuzzing::iid_dependencies::compute_dependencies_by_loops( const loop_head_t const std::unordered_set< location_id::id_type >& all_props_ids = props.get_all_ids(); for ( const auto& body : loop_bodies ) { - if ( all_props_ids.contains( body.id ) ) { + if ( all_props_ids.contains( body.id ) || loop_head.context_hash != body.context_hash ) { continue; } From 6a68200c25bb54b3b362f81d27f7b89946ef8ac0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Fri, 28 Mar 2025 12:03:49 +0100 Subject: [PATCH 130/144] temp changes --- benchmarks/benman.py | 2 +- .../include/fuzzing/iid_node_dependencies.hpp | 2 +- src/fuzzing/src/dump.cpp | 1 + src/fuzzing/src/fuzzer.cpp | 16 +++++++++++----- src/fuzzing/src/iid_node_dependencies.cpp | 2 +- src/tools/runner/sbt-fizzer.py | 2 +- 6 files changed, 16 insertions(+), 9 deletions(-) diff --git a/benchmarks/benman.py b/benchmarks/benman.py index d0532a50..43ba4d83 100755 --- a/benchmarks/benman.py +++ b/benchmarks/benman.py @@ -299,7 +299,7 @@ def search_for_benchmarks(folder : str) -> list: pass return benchmarks - kinds = ["fast", "iid_testing", "medium", "slow", "pending"] + kinds = ["fast", "iid_testing", "testcomp-selection-selection", "medium", "slow", "pending"] benchmarks = [] if name == "all": for kind in kinds: diff --git a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp index be6330be..15386943 100644 --- a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp +++ b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp @@ -220,7 +220,7 @@ struct iid_dependencies { std::unordered_map< location_id, iid_node_dependence_props > id_to_equation_map; std::set< location_id > non_iid_nodes; - void update_non_iid_nodes( sensitivity_analysis& sensitivity ); + void update_ignored_nodes( sensitivity_analysis& sensitivity ); void process_node_dependence( branching_node* node ); }; diff --git a/src/fuzzing/src/dump.cpp b/src/fuzzing/src/dump.cpp index 4c47275d..cb9dbdda 100644 --- a/src/fuzzing/src/dump.cpp +++ b/src/fuzzing/src/dump.cpp @@ -360,6 +360,7 @@ void save_iid_vector_analysis( std::ostream& ostr, analysis_outcomes const& resu case generation_state::STATE_GENERATION_DATA_FOR_NEXT_NODE: ostr << "STATE_GENERATION_DATA_FOR_NEXT_NODE"; break; + case generation_state::STATE_COVERED_BY_OTHER: ostr << "STATE_COVERED_BY_OTHER"; break; default: ostr << "UNKNOWN"; break; } ostr << "\",\n"; diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index 3de8c23d..b2b87cf3 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -1125,6 +1125,10 @@ execution_record::execution_flags fuzzer::process_execution_results() for (; true; ++trace_index) { branching_coverage_info const& info = trace->at(trace_index); + + if ( use_vector_analysis ) { + iid_dependencies::biggest_node_id = std::max( iid_dependencies::biggest_node_id, std::size_t( info.id.id ) ); + } INVARIANT(construction_props.leaf->id == info.id); @@ -1221,11 +1225,6 @@ execution_record::execution_flags fuzzer::process_execution_results() succ_info.predicate ); - if ( use_vector_analysis ) { - iid_dependencies::biggest_node_id = std::max( iid_dependencies::biggest_node_id, - std::size_t( new_node->id.id ) ); - } - construction_props.leaf->set_successor(info.direction, { branching_node::successor_pointer::VISITED, new_node @@ -1616,6 +1615,13 @@ void fuzzer::select_next_state() if (winner == nullptr && !entry_branching->is_closed()) winner = select_iid_coverage_target(); + + // else if (winner != nullptr && get_random_natural_32_bit_in_range(1, 10, generator_for_iid_approach_selection) == 10) + // { + // winner = select_iid_coverage_target(); + // } + + if (winner == nullptr) { state = FINISHED; diff --git a/src/fuzzing/src/iid_node_dependencies.cpp b/src/fuzzing/src/iid_node_dependencies.cpp index 68615fac..6e2450a8 100644 --- a/src/fuzzing/src/iid_node_dependencies.cpp +++ b/src/fuzzing/src/iid_node_dependencies.cpp @@ -920,7 +920,7 @@ int fuzzing::iid_node_dependence_props::get_possible_depth() const * @param sensitivity A reference to the sensitivity analysis object that * provides the changed nodes. */ -void fuzzing::iid_dependencies::update_non_iid_nodes( sensitivity_analysis& sensitivity ) +void fuzzing::iid_dependencies::update_ignored_nodes( sensitivity_analysis& sensitivity ) { for ( branching_node* node : sensitivity.get_changed_nodes() ) { if ( node->is_did_branching() ) { diff --git a/src/tools/runner/sbt-fizzer.py b/src/tools/runner/sbt-fizzer.py index 27a80b7e..d7c9436e 100755 --- a/src/tools/runner/sbt-fizzer.py +++ b/src/tools/runner/sbt-fizzer.py @@ -78,7 +78,7 @@ def build(self_dir, input_file, output_dir, options, use_m32, silent_mode): None).returncode: raise Exception("Linking has failed: " + input_file) t1 = time.time() - if silent_mode is False: print("%.2f," % (t1 - t0), flush=True) + if silent_mode is False: print("%.2f" % (t1 - t0), flush=True) if silent_mode is False: print("},", flush=True) From f2a2a7533333b0af992308d301dc24f9d1b62af3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 12 Apr 2025 21:31:13 +0200 Subject: [PATCH 131/144] feat: update executions counts --- benchmarks/iid_testing/combination_needed.json | 2 +- benchmarks/iid_testing/different_predicates_in_iid.json | 2 +- benchmarks/iid_testing/input_cycle_2_loops_combination.json | 4 ++-- benchmarks/iid_testing/input_cycle_dec_b.json | 2 +- benchmarks/iid_testing/more_iid_condition.json | 2 +- benchmarks/iid_testing/more_iid_condition_nested.json | 2 +- .../iid_testing/more_iid_condition_nested_big_values.json | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/benchmarks/iid_testing/combination_needed.json b/benchmarks/iid_testing/combination_needed.json index a984ac71..2564ccf3 100644 --- a/benchmarks/iid_testing/combination_needed.json +++ b/benchmarks/iid_testing/combination_needed.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 1929, + "num_executions": 1654, "num_covered_branchings": 8, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/different_predicates_in_iid.json b/benchmarks/iid_testing/different_predicates_in_iid.json index 43554337..ffe39afc 100644 --- a/benchmarks/iid_testing/different_predicates_in_iid.json +++ b/benchmarks/iid_testing/different_predicates_in_iid.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 15347, + "num_executions": 14204, "num_covered_branchings": 19, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/input_cycle_2_loops_combination.json b/benchmarks/iid_testing/input_cycle_2_loops_combination.json index faa90799..144ea353 100644 --- a/benchmarks/iid_testing/input_cycle_2_loops_combination.json +++ b/benchmarks/iid_testing/input_cycle_2_loops_combination.json @@ -48,12 +48,12 @@ ], "output_statistics": { "sensitivity_analysis": { - "num_generated_tests": 10, + "num_generated_tests": 9, "num_crashes": 0, "num_boundary_violations": 0 }, "typed_minimization_analysis": { - "num_generated_tests": 2, + "num_generated_tests": 3, "num_crashes": 0, "num_boundary_violations": 0 }, diff --git a/benchmarks/iid_testing/input_cycle_dec_b.json b/benchmarks/iid_testing/input_cycle_dec_b.json index da0a3aa3..f67e8fdc 100644 --- a/benchmarks/iid_testing/input_cycle_dec_b.json +++ b/benchmarks/iid_testing/input_cycle_dec_b.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 1388, + "num_executions": 1110, "num_covered_branchings": 10, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/more_iid_condition.json b/benchmarks/iid_testing/more_iid_condition.json index b19a5aea..e0eaf0e4 100644 --- a/benchmarks/iid_testing/more_iid_condition.json +++ b/benchmarks/iid_testing/more_iid_condition.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 6114, + "num_executions": 6913, "num_covered_branchings": 9, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/more_iid_condition_nested.json b/benchmarks/iid_testing/more_iid_condition_nested.json index ce2829ba..fcef796c 100644 --- a/benchmarks/iid_testing/more_iid_condition_nested.json +++ b/benchmarks/iid_testing/more_iid_condition_nested.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 7883, + "num_executions": 11207, "num_covered_branchings": 9, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/more_iid_condition_nested_big_values.json b/benchmarks/iid_testing/more_iid_condition_nested_big_values.json index 03c36964..d18aeb41 100644 --- a/benchmarks/iid_testing/more_iid_condition_nested_big_values.json +++ b/benchmarks/iid_testing/more_iid_condition_nested_big_values.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 47736, + "num_executions": 54514, "num_covered_branchings": 12, "covered_branchings": [ 1, From c899ae2a8afa93b68ee0eaa52b5480454c03c010 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 12 Apr 2025 21:33:23 +0200 Subject: [PATCH 132/144] feat: add ai generated benchmarks and benchmarks from testcomp --- benchmarks/iid_testing/ai_generated_01.c | 39 ++++++++++ benchmarks/iid_testing/ai_generated_01.json | 59 +++++++++++++++ benchmarks/iid_testing/ai_generated_02.c | 41 +++++++++++ benchmarks/iid_testing/ai_generated_02.json | 61 ++++++++++++++++ benchmarks/iid_testing/ai_generated_03.c | 44 ++++++++++++ benchmarks/iid_testing/ai_generated_03.json | 61 ++++++++++++++++ benchmarks/iid_testing/ai_generated_05.c | 41 +++++++++++ benchmarks/iid_testing/ai_generated_05.json | 60 ++++++++++++++++ benchmarks/iid_testing/ai_generated_07.c | 45 ++++++++++++ benchmarks/iid_testing/ai_generated_07.json | 71 +++++++++++++++++++ benchmarks/iid_testing/ai_generated_11.c | 43 +++++++++++ benchmarks/iid_testing/ai_generated_11.json | 56 +++++++++++++++ benchmarks/iid_testing/ai_generated_12.c | 47 ++++++++++++ benchmarks/iid_testing/ai_generated_12.json | 67 +++++++++++++++++ benchmarks/iid_testing/ai_generated_14.c | 48 +++++++++++++ benchmarks/iid_testing/ai_generated_14.json | 65 +++++++++++++++++ benchmarks/iid_testing/ai_generated_16.c | 42 +++++++++++ benchmarks/iid_testing/ai_generated_17.c | 44 ++++++++++++ benchmarks/iid_testing/ai_generated_17.json | 59 +++++++++++++++ benchmarks/iid_testing/ai_generated_18.c | 44 ++++++++++++ benchmarks/iid_testing/ai_generated_18.json | 61 ++++++++++++++++ benchmarks/iid_testing/ai_generated_19.c | 46 ++++++++++++ benchmarks/iid_testing/ai_generated_19.json | 67 +++++++++++++++++ benchmarks/iid_testing/ai_generated_20.c | 40 +++++++++++ benchmarks/iid_testing/ai_generated_20.json | 54 ++++++++++++++ benchmarks/iid_testing/ai_generated_21.c | 59 +++++++++++++++ benchmarks/iid_testing/ai_generated_21.json | 61 ++++++++++++++++ benchmarks/iid_testing/ai_generated_22.c | 59 +++++++++++++++ benchmarks/iid_testing/ai_generated_22.json | 65 +++++++++++++++++ benchmarks/iid_testing/ai_generated_23.c | 43 +++++++++++ benchmarks/iid_testing/ai_generated_23.json | 61 ++++++++++++++++ benchmarks/iid_testing/ai_generated_24.c | 53 ++++++++++++++ benchmarks/iid_testing/ai_generated_24.json | 53 ++++++++++++++ benchmarks/iid_testing/ai_generated_25.c | 43 +++++++++++ benchmarks/iid_testing/ai_generated_25.json | 47 ++++++++++++ benchmarks/iid_testing/ai_generated_26.c | 45 ++++++++++++ benchmarks/iid_testing/ai_generated_26.json | 59 +++++++++++++++ ...s}{verisec_sendmail_tTflag_arr_one_loop}.c | 47 ++++++++++++ ...verisec_sendmail_tTflag_arr_one_loop}.json | 55 ++++++++++++++ ...{ReachSafety-Loops}--{nla-digbench}{ps4}.c | 55 ++++++++++++++ ...achSafety-Loops}--{nla-digbench}{ps4}.json | 48 +++++++++++++ 41 files changed, 2158 insertions(+) create mode 100644 benchmarks/iid_testing/ai_generated_01.c create mode 100644 benchmarks/iid_testing/ai_generated_01.json create mode 100644 benchmarks/iid_testing/ai_generated_02.c create mode 100644 benchmarks/iid_testing/ai_generated_02.json create mode 100644 benchmarks/iid_testing/ai_generated_03.c create mode 100644 benchmarks/iid_testing/ai_generated_03.json create mode 100644 benchmarks/iid_testing/ai_generated_05.c create mode 100644 benchmarks/iid_testing/ai_generated_05.json create mode 100644 benchmarks/iid_testing/ai_generated_07.c create mode 100644 benchmarks/iid_testing/ai_generated_07.json create mode 100644 benchmarks/iid_testing/ai_generated_11.c create mode 100644 benchmarks/iid_testing/ai_generated_11.json create mode 100644 benchmarks/iid_testing/ai_generated_12.c create mode 100644 benchmarks/iid_testing/ai_generated_12.json create mode 100644 benchmarks/iid_testing/ai_generated_14.c create mode 100644 benchmarks/iid_testing/ai_generated_14.json create mode 100644 benchmarks/iid_testing/ai_generated_16.c create mode 100644 benchmarks/iid_testing/ai_generated_17.c create mode 100644 benchmarks/iid_testing/ai_generated_17.json create mode 100644 benchmarks/iid_testing/ai_generated_18.c create mode 100644 benchmarks/iid_testing/ai_generated_18.json create mode 100644 benchmarks/iid_testing/ai_generated_19.c create mode 100644 benchmarks/iid_testing/ai_generated_19.json create mode 100644 benchmarks/iid_testing/ai_generated_20.c create mode 100644 benchmarks/iid_testing/ai_generated_20.json create mode 100644 benchmarks/iid_testing/ai_generated_21.c create mode 100644 benchmarks/iid_testing/ai_generated_21.json create mode 100644 benchmarks/iid_testing/ai_generated_22.c create mode 100644 benchmarks/iid_testing/ai_generated_22.json create mode 100644 benchmarks/iid_testing/ai_generated_23.c create mode 100644 benchmarks/iid_testing/ai_generated_23.json create mode 100644 benchmarks/iid_testing/ai_generated_24.c create mode 100644 benchmarks/iid_testing/ai_generated_24.json create mode 100644 benchmarks/iid_testing/ai_generated_25.c create mode 100644 benchmarks/iid_testing/ai_generated_25.json create mode 100644 benchmarks/iid_testing/ai_generated_26.c create mode 100644 benchmarks/iid_testing/ai_generated_26.json create mode 100644 benchmarks/iid_testing/{ReachSafety-BitVectors}--{bitvector-loops}{verisec_sendmail_tTflag_arr_one_loop}.c create mode 100644 benchmarks/iid_testing/{ReachSafety-BitVectors}--{bitvector-loops}{verisec_sendmail_tTflag_arr_one_loop}.json create mode 100644 benchmarks/iid_testing/{ReachSafety-Loops}--{nla-digbench}{ps4}.c create mode 100644 benchmarks/iid_testing/{ReachSafety-Loops}--{nla-digbench}{ps4}.json diff --git a/benchmarks/iid_testing/ai_generated_01.c b/benchmarks/iid_testing/ai_generated_01.c new file mode 100644 index 00000000..de254423 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_01.c @@ -0,0 +1,39 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 100 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + + // 1. Fill data structure in a loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; // Invalid size + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop + int count = 0; + for ( short i = 0; i < size; ++i ) { + // 3. Modify internal state based on data/condition (linear update) + if ( data[ i ] == 'X' ) { + count++; // Linear change (+1) + } + } + + // 4. Final condition based on internal variable + // 5. Condition check is linear + if ( count > 15 ) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_01.json b/benchmarks/iid_testing/ai_generated_01.json new file mode 100644 index 00000000..b30659e8 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_01.json @@ -0,0 +1,59 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 5694, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_02.c b/benchmarks/iid_testing/ai_generated_02.c new file mode 100644 index 00000000..8707a6e7 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_02.c @@ -0,0 +1,41 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 50 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + + // 1. Fill data structure in a loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop + int balance = 0; + for ( short i = 0; i < size; ++i ) { + // 3. Modify internal state based on data/condition (linear update) + if ( data[ i ] == 'P' ) { + balance++; // Linear change (+1) + } else if ( data[ i ] == 'N' ) { + balance--; // Linear change (-1) + } + } + + // 4. Final condition based on internal variable + // 5. Condition check is linear + if ( balance == 15 ) { + return 1; // Balanced + } else { + return 0; // Not balanced + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_02.json b/benchmarks/iid_testing/ai_generated_02.json new file mode 100644 index 00000000..111ed923 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_02.json @@ -0,0 +1,61 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 4658, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_03.c b/benchmarks/iid_testing/ai_generated_03.c new file mode 100644 index 00000000..24c324ef --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_03.c @@ -0,0 +1,44 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 50 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + + // 1. Fill data structure in a loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop + int value = 0; + short idx = 0; + while ( idx < size ) { + // 3. Modify internal state based on data/condition + // 6. Change is static (+2 or +1) + if ( data[ idx ] == 'A' ) { + value += 2; // Static change + } else if ( data[ idx ] == 'B' ) { + value += 1; // Static change + } + idx++; + } + + // 4. Final condition based on internal variable + // 5. Condition check is linear + if ( value == 25 ) { + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_03.json b/benchmarks/iid_testing/ai_generated_03.json new file mode 100644 index 00000000..c960b594 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_03.json @@ -0,0 +1,61 @@ +{ + "args": { + "max_executions": 20000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 11717, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_05.c b/benchmarks/iid_testing/ai_generated_05.c new file mode 100644 index 00000000..13a00c97 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_05.c @@ -0,0 +1,41 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 80 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + + // 1. Fill data structure in a loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop + int special_count = 0; + for ( short i = 0; i < size; ++i ) { + // 3. Modify internal state based on data/condition + // Condition involves two checks on the same element + // 6. Change is static (+1) + if ( data[ i ] > '5' && data[ i ] < '9' ) { // Check if digit is 6, 7, or 8 + special_count++; // Static change + } + } + + // 4. Final condition based on internal variable + // 5. Condition check is linear + if ( special_count == 3 ) { + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_05.json b/benchmarks/iid_testing/ai_generated_05.json new file mode 100644 index 00000000..630af63e --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_05.json @@ -0,0 +1,60 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 2506, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_07.c b/benchmarks/iid_testing/ai_generated_07.c new file mode 100644 index 00000000..311203ca --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_07.c @@ -0,0 +1,45 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 50 +#define START_COUNT 100 // Start high and decrement + +int main() +{ + char data[ MAX_SIZE ]; + short size; + + // 1. Fill data structure in a loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop + int non_vowel_upper_count = START_COUNT; + for ( short i = 0; i < size; ++i ) { + // 3. Modify internal state based on data/condition (nested) + // 6. Change is static (-1) + if ( data[ i ] >= 'A' && data[ i ] <= 'Z' ) { // Is uppercase? + if ( data[ i ] != 'A' && data[ i ] != 'E' && data[ i ] != 'I' && data[ i ] != 'O' && + data[ i ] != 'U' ) { // Is not a vowel? + non_vowel_upper_count--; // Static change + } + } + } + + // 4. Final condition based on internal variable + // 5. Condition check is linear (< START_COUNT - 5) + // Checks if at least 6 non-vowel uppercase letters were found + if ( non_vowel_upper_count < START_COUNT - 5 ) { + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_07.json b/benchmarks/iid_testing/ai_generated_07.json new file mode 100644 index 00000000..9cd8bd82 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_07.json @@ -0,0 +1,71 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 2594, + "num_covered_branchings": 12, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0, + 11, + 0, + 12, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 6, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_11.c b/benchmarks/iid_testing/ai_generated_11.c new file mode 100644 index 00000000..03508ced --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_11.c @@ -0,0 +1,43 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 100 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + int countA = 0; // First state variable + int countB = 0; // Second state variable + + // 1. Fill data structure in a loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop + for ( short i = 0; i < size; ++i ) { + // 3. Modify internal state variables based on data/condition + // 6. Changes are static (+1) + if ( data[ i ] == 'A' ) { + countA++; // Static change + } else if ( data[ i ] == 'B' ) { + countB++; // Static change + } + } + + // 4. Final condition based on multiple internal variables + // 5. Condition check is linear (countA - countB > 5) + if ( countA > countB + 5 ) { // Check relation between the two counters + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_11.json b/benchmarks/iid_testing/ai_generated_11.json new file mode 100644 index 00000000..f72d9871 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_11.json @@ -0,0 +1,56 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 2350, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_12.c b/benchmarks/iid_testing/ai_generated_12.c new file mode 100644 index 00000000..68163642 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_12.c @@ -0,0 +1,47 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 150 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + int state = 0; // State variable (e.g., 0=Start, 1=SeenX, 2=SeenY) + int state2_entries = 0; // Counts entries into state 2 + + // 1. Fill data structure in a loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop + for ( short i = 0; i < size; ++i ) { + // 3. Modify internal state variables based on data/condition + // 6. State change is static (assignment), counter change is static (+1) + if ( state == 0 && data[ i ] == 'X' ) { + state = 1; // Static change (assignment) + } else if ( state == 1 && data[ i ] == 'Y' ) { + state = 2; // Static change (assignment) + state2_entries++; // Static change (+1) + } else if ( data[ i ] == 'R' ) { // Reset condition + state = 0; // Static change (assignment) + } + // Stay in current state otherwise, or remain in state 2 + } + + // 4. Final condition based on internal variable + // 5. Condition check is linear + if ( state2_entries > 3 ) { + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_12.json b/benchmarks/iid_testing/ai_generated_12.json new file mode 100644 index 00000000..09563861 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_12.json @@ -0,0 +1,67 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 6945, + "num_covered_branchings": 10, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_14.c b/benchmarks/iid_testing/ai_generated_14.c new file mode 100644 index 00000000..7b564b79 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_14.c @@ -0,0 +1,48 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 100 +#define UPPER_BOUND 10 +#define LOWER_BOUND -10 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + int bounded_counter = 0; // State variable with bounds + + // 1. Fill data structure in a loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop + for ( short i = 0; i < size; ++i ) { + // 3. Modify internal state based on data/condition + // 6. Changes are static (+1 or -1), but bounded + if ( data[ i ] == 'U' ) { + if ( bounded_counter < UPPER_BOUND ) { + bounded_counter++; // Static change + } + } else if ( data[ i ] == 'D' ) { + if ( bounded_counter > LOWER_BOUND ) { + bounded_counter--; // Static change + } + } + } + + // 4. Final condition based on internal variable + // 5. Condition check is linear + if ( bounded_counter == 5 ) { // Check final value against a specific target + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_14.json b/benchmarks/iid_testing/ai_generated_14.json new file mode 100644 index 00000000..d3a7a120 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_14.json @@ -0,0 +1,65 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 9340, + "num_covered_branchings": 9, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_16.c b/benchmarks/iid_testing/ai_generated_16.c new file mode 100644 index 00000000..4c3cb3f5 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_16.c @@ -0,0 +1,42 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 50 + +int main() +{ + char data1[ MAX_SIZE ]; + char data2[ MAX_SIZE ]; // Second array + short size; + int match_count = 0; // State variable + + // 1. Fill data structures in loops + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data1[ i ] = __VERIFIER_nondet_char(); + data2[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop iterating through data + for ( short i = 0; i < size; ++i ) { + // 3. Modify internal state based on data from both arrays + // 6. Change is static (+1) + if ( data1[ i ] == 'S' && data2[ i ] == 'T' ) { + match_count++; // Static change + } + } + + // 4. Final condition based on internal variable + // 5. Condition check is linear + if ( match_count > 5 ) { // Check if the S/T pair occurred more than once + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_17.c b/benchmarks/iid_testing/ai_generated_17.c new file mode 100644 index 00000000..e3dd7377 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_17.c @@ -0,0 +1,44 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 60 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + int p_count = 0; // State variable + + // 1. Fill data structure in a loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop using pointer arithmetic + char* ptr = data; + char* end_ptr = data + size; // Pointer to one past the last element + + while ( ptr < end_ptr ) { + // 3. Modify internal state based on data accessed via pointer + // 6. Change is static (+1) + if ( *ptr == 'P' ) { + p_count++; // Static change + } + ptr++; // Move pointer to the next element + } + + // 4. Final condition based on internal variable + // 5. Condition check is linear + if ( p_count == 5 ) { // Check if 'P' was never found + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_17.json b/benchmarks/iid_testing/ai_generated_17.json new file mode 100644 index 00000000..f0e39ed5 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_17.json @@ -0,0 +1,59 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 1426, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_18.c b/benchmarks/iid_testing/ai_generated_18.c new file mode 100644 index 00000000..2f2d2c3b --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_18.c @@ -0,0 +1,44 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 80 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + int counter = 0; // State variable + + // 1. Fill data structure in a loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop + for ( short i = 0; i < size; ++i ) { + // 3. Modify internal state based on data and current state + // 6. Change is static (+1 or +2) + if ( data[ i ] == 'X' ) { + if ( counter % 2 == 0 ) { // Check if counter is even + counter += 1; // Static change +1 + } else { // Counter is odd + counter += 2; // Static change +2 + } + } + } + + // 4. Final condition based on internal variable + // 5. Condition check is linear + if ( counter > 50 ) { + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_18.json b/benchmarks/iid_testing/ai_generated_18.json new file mode 100644 index 00000000..04b041fc --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_18.json @@ -0,0 +1,61 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 14605, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_19.c b/benchmarks/iid_testing/ai_generated_19.c new file mode 100644 index 00000000..f3e7aba6 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_19.c @@ -0,0 +1,46 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 100 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + short first_F_index = -1; // State variable 1 + short last_L_index = -1; // State variable 2 + + // 1. Fill data structure in a loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop + for ( short i = 0; i < size; ++i ) { + // 3. Modify internal state variables based on data/condition + // 6. Change is static (assignment of index, happens at most once for first_F) + if ( data[ i ] == 'F' ) { + if ( first_F_index == -1 ) { // Assign only if not already found + first_F_index = i; // Static change (assignment) + } + } else if ( data[ i ] == 'L' ) { + last_L_index = i; // Update last known L index (Static change - assignment) + } + } + + // 4. Final condition based on multiple internal variables + // 5. Condition check is linear (last_L_index - first_F_index > 10) + // Check if both were found and the distance is large enough + if ( first_F_index != -1 && last_L_index != -1 && last_L_index > first_F_index + 10 ) { + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_19.json b/benchmarks/iid_testing/ai_generated_19.json new file mode 100644 index 00000000..a4dcb4e7 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_19.json @@ -0,0 +1,67 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 5897, + "num_covered_branchings": 10, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_20.c b/benchmarks/iid_testing/ai_generated_20.c new file mode 100644 index 00000000..eb1f23d7 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_20.c @@ -0,0 +1,40 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 90 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + int odd_count = 0; // State variable + + // 1. Fill data structure in a loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop + for ( short i = 0; i < size; ++i ) { + // 3. Modify internal state based on bitwise check of data + // 6. Change is static (+1) + if ( ( data[ i ] & 1 ) != 0 ) { // Check if the LSB is 1 (ASCII value is odd) + odd_count++; // Static change + } + } + + // 4. Final condition based on internal variable + // 5. Condition check is linear + if ( odd_count * 2 > size ) { // Check if more than half the chars had odd ASCII values + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_20.json b/benchmarks/iid_testing/ai_generated_20.json new file mode 100644 index 00000000..11765dc2 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_20.json @@ -0,0 +1,54 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 536, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_21.c b/benchmarks/iid_testing/ai_generated_21.c new file mode 100644 index 00000000..b56346dd --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_21.c @@ -0,0 +1,59 @@ +#include +#include // For NULL potentially, though not strictly needed here + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern int __VERIFIER_nondet_int(); + +// Simple Node structure for linked list +typedef struct Node { + int value; + struct Node* next; +} Node; + +#define MAX_NODES 50 + +int main() +{ + Node nodes[ MAX_NODES ]; // Pre-allocate nodes for simplicity + Node* head = NULL; + Node* current = NULL; + short size; + int count_gt_10 = 0; // State variable + + // 1. Fill data structure (linked list) in an initial loop + size = __VERIFIER_nondet_short(); + if ( size < 0 || size > MAX_NODES ) { // Allow size 0 + return -1; + } + + for ( short i = 0; i < size; ++i ) { + nodes[ i ].value = __VERIFIER_nondet_int(); + nodes[ i ].next = NULL; + if ( head == NULL ) { + head = &nodes[ i ]; + current = head; + } else { + current->next = &nodes[ i ]; + current = current->next; + } + } + + // 2. Core logic in loop iterating through the data structure (linked list) + current = head; // Start iteration from head + while ( current != NULL ) { + // 3. Modify internal state based on data elements + // 5. Change is constant (+1) + if ( current->value == 10 ) { + count_gt_10++; // Constant change + } + current = current->next; // Move to next node + } // Loop completes when end of list is reached + + // 4. Final condition based on internal variable + if ( count_gt_10 >= 10 ) { // Non-trivial check + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_21.json b/benchmarks/iid_testing/ai_generated_21.json new file mode 100644 index 00000000..af0667ee --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_21.json @@ -0,0 +1,61 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 5088, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_22.c b/benchmarks/iid_testing/ai_generated_22.c new file mode 100644 index 00000000..468f7e7a --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_22.c @@ -0,0 +1,59 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 100 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + int countA = 0; // State variable for first loop + int countB = 0; // State variable for second loop + + // 1. Fill data structure in a loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in sequence of loops + // Loop 1 + for ( short i = 0; i < size; ++i ) { + // 3. Modify state variable (countA) + // 5. Change is constant (+1) + if ( data[ i ] == 'A' ) { + countA++; // Constant change + } + } + // Loop 2 + for ( short i = 0; i < size; ++i ) { + // 3. Modify state variable (countB) + // 5. Change is constant (+1) + if ( data[ i ] == 'B' ) { + countB++; // Constant change + } + } // Both loops always complete + + // 4. Final condition based on multiple internal variables + // if ( countA == countB && countA > 10 ) { // Check if counts are equal and non-zero + // return 1; + // } else { + // return 0; + // } + + if ( countA == 10 ) { // Check if counts are equal and non-zero + return 1; + } + + if ( countB == 10 ) { + return 1; + } + + return 0; +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_22.json b/benchmarks/iid_testing/ai_generated_22.json new file mode 100644 index 00000000..cb65b6e6 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_22.json @@ -0,0 +1,65 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 6061, + "num_covered_branchings": 9, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_23.c b/benchmarks/iid_testing/ai_generated_23.c new file mode 100644 index 00000000..b1fcbf5a --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_23.c @@ -0,0 +1,43 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 90 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + int f_count = 0; // State variable + + // 1. Fill data structure in a loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop using 'continue' + for ( short i = 0; i < size; ++i ) { + if ( data[ i ] == 'S' ) { // Skip character 'S' + continue; // Jumps to the next iteration + } + // 3. Modify internal state if not skipped + // 5. Change is constant (+1) + if ( data[ i ] == 'F' ) { + f_count++; // Constant change + } + // Potentially other processing for non-'S' characters here + } // Loop always completes (though iterations might be skipped) + + // 4. Final condition based on internal variable + if ( f_count >= 10 ) { // Check if 'F' was found enough times + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_23.json b/benchmarks/iid_testing/ai_generated_23.json new file mode 100644 index 00000000..a8ea38f4 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_23.json @@ -0,0 +1,61 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 4050, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_24.c b/benchmarks/iid_testing/ai_generated_24.c new file mode 100644 index 00000000..f49a5187 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_24.c @@ -0,0 +1,53 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern int __VERIFIER_nondet_int(); // Use int for values + +#define MAX_SIZE 60 + +int main() +{ + int data[ MAX_SIZE ]; + short size; + int pos_count = 0; // State variable 1 + int neg_count = 0; // State variable 2 + + // 1. Fill data structure in a loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_int(); // Fill with non-det integers + } + + // 2. Core logic in loop + for ( short i = 0; i < size; ++i ) { + // 3. Modify internal state variables based on data + // 5. Changes are constant (+1) + if ( data[ i ] > 0 ) { + pos_count++; // Constant change + } else if ( data[ i ] < 0 ) { + neg_count++; // Constant change + } + // Ignore zeros + } // Loop always completes + + // 4. Multiple final conditions based on internal variables + // if ( pos_count > neg_count + 2 ) { // Check relation 1 + // return 1; // More positives by a margin + // } else if ( neg_count > pos_count + 2 ) { // Check relation 2 + // return 2; // More negatives by a margin + // } else { + // return 0; // Neither condition met (counts are close or equal) + // } + + if ( pos_count > 20 ) { // Check relation 1 + return 1; // More positives by a margin + } else if ( neg_count > 20 ) { // Check relation 2 + return 2; // More negatives by a margin + } else { + return 0; // Neither condition met (counts are close or equal) + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_24.json b/benchmarks/iid_testing/ai_generated_24.json new file mode 100644 index 00000000..6fd503e2 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_24.json @@ -0,0 +1,53 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 600, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 48669, + "num_covered_branchings": 8, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 6, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_25.c b/benchmarks/iid_testing/ai_generated_25.c new file mode 100644 index 00000000..f4a7dff6 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_25.c @@ -0,0 +1,43 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 50 // Note: actual string length can be less + +int main() +{ + char data[ MAX_SIZE ]; + short max_len; + int len = 0; // State variable + + // 1. Fill data structure (making it a C string) + max_len = __VERIFIER_nondet_short(); + if ( max_len <= 0 || max_len >= MAX_SIZE ) { // Need space for null terminator + return -1; + } + // Fill up to max_len - 1 + for ( short i = 0; i < max_len; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + data[ max_len ] = '\0'; // Ensure null termination + + // 2. Core logic in loop to find length (simulates strlen) + // 6. Uses break based on C string convention + for ( int i = 0;; ++i ) { // Loop bound by buffer size + if ( data[ i ] == '\0' ) { + break; // Found null terminator + } + // 3. Modify internal state variable + // 5. Change is constant (+1) + len++; // Constant change + } // Loop breaks on null terminator + + // 4. Final condition based on internal variable (calculated length) + if ( len > 40 ) { + return 1; // Length is greater than 10 + } else { + return 0; // Length is 10 or less + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_25.json b/benchmarks/iid_testing/ai_generated_25.json new file mode 100644 index 00000000..ae81b51d --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_25.json @@ -0,0 +1,47 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 600, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 21189, + "num_covered_branchings": 5, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_26.c b/benchmarks/iid_testing/ai_generated_26.c new file mode 100644 index 00000000..33486456 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_26.c @@ -0,0 +1,45 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +// Simple struct definition +typedef struct { + char status; // e.g., 'A'ctive, 'I'nactive +} Item; + +#define MAX_ITEMS 40 + +int main() +{ + Item inventory[ MAX_ITEMS ]; // Array of structs + short num_items; + int active_count = 0; // State variable + + // 1. Fill data structure (array of structs) in an initial loop + num_items = __VERIFIER_nondet_short(); + if ( num_items <= 0 || num_items > MAX_ITEMS ) { + return -1; + } + for ( short i = 0; i < num_items; ++i ) { + inventory[ i ].status = __VERIFIER_nondet_char(); // Assign status + } + + // 2. Core logic in loop iterating through the array of structs + for ( short i = 0; i < num_items; ++i ) { + // 3. Modify internal state based on struct member + // 5. Change is constant (+1) + if ( inventory[ i ].status == 'A' ) { + active_count++; // Constant change + } + } // Loop always completes + + // 4. Final condition based on internal variable + if ( active_count >= 10 ) { // Check if at least half are active + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_26.json b/benchmarks/iid_testing/ai_generated_26.json new file mode 100644 index 00000000..c753c534 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_26.json @@ -0,0 +1,59 @@ +{ + "args": { + "max_executions": 100000, + "max_seconds": 600, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 2190, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/{ReachSafety-BitVectors}--{bitvector-loops}{verisec_sendmail_tTflag_arr_one_loop}.c b/benchmarks/iid_testing/{ReachSafety-BitVectors}--{bitvector-loops}{verisec_sendmail_tTflag_arr_one_loop}.c new file mode 100644 index 00000000..be8b8ed5 --- /dev/null +++ b/benchmarks/iid_testing/{ReachSafety-BitVectors}--{bitvector-loops}{verisec_sendmail_tTflag_arr_one_loop}.c @@ -0,0 +1,47 @@ +extern void abort( void ); +extern void __assert_fail( const char*, const char*, unsigned int, const char* ) + __attribute__( ( __nothrow__, __leaf__ ) ) __attribute__( ( __noreturn__ ) ); +void reach_error() { __assert_fail( "0", "verisec_sendmail_tTflag_arr_one_loop.c", 3, "reach_error" ); } + +extern char __VERIFIER_nondet_char(); + +// void __VERIFIER_assert( int cond ) +// { +// if ( !( cond ) ) { +// ERROR: { +// reach_error(); +// abort(); +// } +// } +// return; +// } + +int main( void ) +{ + char in[ 11 ]; // = "3277192070"; + char* s; + unsigned char c; + int i, j; + int idx_in; + for ( i = 0; i < 11; i++ ) + in[ i ] = __VERIFIER_nondet_char(); + in[ 10 ] = 0; + idx_in = 0; + s = in; + i = 0; + c = in[ idx_in ]; + while ( ( '0' <= c ) && ( c <= '9' ) ) { + j = c - '0'; + i = i * 10U + j; + idx_in++; + c = in[ idx_in ]; + } + /* BAD */ + // + // __VERIFIER_assert( i >= 0 ); + + if ( idx_in != 10 ) + return 1; + + return 0; +} diff --git a/benchmarks/iid_testing/{ReachSafety-BitVectors}--{bitvector-loops}{verisec_sendmail_tTflag_arr_one_loop}.json b/benchmarks/iid_testing/{ReachSafety-BitVectors}--{bitvector-loops}{verisec_sendmail_tTflag_arr_one_loop}.json new file mode 100644 index 00000000..99521c89 --- /dev/null +++ b/benchmarks/iid_testing/{ReachSafety-BitVectors}--{bitvector-loops}{verisec_sendmail_tTflag_arr_one_loop}.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 2296, + "num_covered_branchings": 4, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/{ReachSafety-Loops}--{nla-digbench}{ps4}.c b/benchmarks/iid_testing/{ReachSafety-Loops}--{nla-digbench}{ps4}.c new file mode 100644 index 00000000..9c60e2f3 --- /dev/null +++ b/benchmarks/iid_testing/{ReachSafety-Loops}--{nla-digbench}{ps4}.c @@ -0,0 +1,55 @@ +extern void abort( void ); +extern void __assert_fail( const char*, const char*, unsigned int, const char* ) + __attribute__( ( __nothrow__, __leaf__ ) ) __attribute__( ( __noreturn__ ) ); +void reach_error() { __assert_fail( "0", "ps4.c", 3, "reach_error" ); } +extern int __VERIFIER_nondet_int( void ); +extern void abort( void ); +// void assume_abort_if_not(int cond) +// { +// if (!cond) +// { +// abort(); +// } +// } +// void __VERIFIER_assert(int cond) +// { +// if (!(cond)) +// { +// ERROR: +// { +// reach_error(); +// } +// } +// return; +// } + +int main() +{ + int k, y, x, c; + k = __VERIFIER_nondet_int(); + + y = 0; + x = 0; + c = 0; + + while ( 1 ) { + // __VERIFIER_assert(4 * x - y * y * y * y - 2 * y * y * y - y * y == 0); + + if ( !( c < k ) ) + break; + + c = c + 1; + y = y + 1; + x = y * y * y + x; + } + // __VERIFIER_assert(k * y - (y * y) == 0); + // __VERIFIER_assert(4 * x - y * y * y * y - 2 * y * y * y - y * y == 0); + + if ( c == 14 ) + return 0; + + if ( y == 28 ) + return 0; + + return 0; +} diff --git a/benchmarks/iid_testing/{ReachSafety-Loops}--{nla-digbench}{ps4}.json b/benchmarks/iid_testing/{ReachSafety-Loops}--{nla-digbench}{ps4}.json new file mode 100644 index 00000000..c1eac294 --- /dev/null +++ b/benchmarks/iid_testing/{ReachSafety-Loops}--{nla-digbench}{ps4}.json @@ -0,0 +1,48 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 118, + "num_covered_branchings": 3, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 1 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file From fba44136fca898aa7f7bc5631b979101e0dcdc94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 12 Apr 2025 21:34:22 +0200 Subject: [PATCH 133/144] feat: better node choosing --- .../include/fuzzing/iid_vector_analysis.hpp | 36 +- src/fuzzing/src/iid_vector_analysis.cpp | 342 +++++++++++++----- 2 files changed, 274 insertions(+), 104 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index 6193050a..2689cd71 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -45,7 +45,12 @@ enum generation_state { STATE_COVERED_BY_OTHER }; -enum failed_generation_method { METHOD_GENERATE_FROM_OTHER_NODE, METHOD_GENERATE_ARTIFICIAL_DATA }; +enum failed_generation_method { + METHOD_NONE, + METHOD_GENERATE_FROM_OTHER_NODE, + METHOD_GENERATE_ARTIFICIAL_DATA, + METHOD_DO_NOT_GENERATE +}; struct iid_node_generations_stats { int method_calls = 0; @@ -69,10 +74,11 @@ struct iid_node_generations_stats { int generate_artificial_data = 0; int generate_artificial_data_max = 0; + int do_not_generate_counter = 0; generation_state state = generation_state::STATE_NOT_COVERED; - failed_generation_method last_failed_method = METHOD_GENERATE_ARTIFICIAL_DATA; + failed_generation_method last_failed_method = METHOD_NONE; }; struct loaded_bits_props { @@ -98,6 +104,7 @@ struct loop_properties { const std::unordered_set< location_id::id_type >& get_all_ids() const; const std::unordered_set< location_id::id_type >& get_loop_head_ids() const; location_id::id_type get_smallest_loop_head_id() const; + std::optional< location_id::id_type > get_smallest_body_id() const; location_id::id_type get_smallest_id() const; void set_chosen_loop_head(); void update_stored_ids(); @@ -231,6 +238,7 @@ struct equation { equation add_to_positive( int value ) const; equation add_to_values( const equation& other ) const; + int simplify_by_gcd(); int get_vector_size() const; int get_one_way_branching_count() const; int get_biggest_value() const; @@ -320,6 +328,11 @@ struct equation_matrix { void print_matrix(); BRANCHING_PREDICATE get_branching_predicate() const; + int get_number_of_different_branchings() const { return branching_values.size(); } + + std::map< double, int, FloatComparator > branching_values; + + std::vector< std::map< int, int > > vector_counts_histogram; private: void add_path( branching_node* end_node, @@ -327,13 +340,13 @@ struct equation_matrix { bool add_columns, std::size_t max_directions_in_path_index ); - std::map< double, int, FloatComparator > branching_values; std::vector< equation > matrix; std::unordered_set< equation > unique_rows; std::vector< branching_node* > all_paths; std::vector< node_id_with_direction > nodes; std::unordered_set< location_id::id_type > node_ids; + std::unordered_set< equation > vectors; std::unordered_map< equation, int > vectors_with_hits; int computed_vectors = 0; }; @@ -347,11 +360,12 @@ struct iid_node_dependence_props { const equation_matrix& get_matrix() const { return matrix; } const iid_node_generations_stats& get_generations_stats() const { return stats; } - bool should_generate( const loop_dependencies& loop_to_properties ) const; - bool too_much_failed_in_row( int max_failed_generations_in_row ) const; + bool is_covered() const; + bool should_generate_new( const loop_dependencies& loop_to_properties ) const; + failed_generation_method determine_recovery_strategy(); + bool too_much_failed_in_row(); void set_as_generating_for_other_node( int minimal_max_generation_for_other_node ); void set_as_generating_artificial_data( int minimal_max_generation_artificial_data ); - failed_generation_method get_method_for_failed_generation( bool is_first ); bool is_equal_branching_predicate() const; bool is_matrix_generated() const { return matrix_generated; } @@ -359,7 +373,7 @@ struct iid_node_dependence_props { private: - void generate_vectors_if_not_enough_data( std::vector< equation >& best_vectors, equation_matrix& submatrix ); + std::optional< std::vector< equation > > generate_vectors_if_not_enough_data( equation_matrix& submatrix ); std::optional< std::vector< equation > > get_best_vectors( equation_matrix& submatrix, int number_of_vectors ); generated_path return_empty_path(); generated_path return_path( const generated_path& path ); @@ -451,19 +465,25 @@ struct iid_dependencies { // Configurations inline static bool random_nested_loop_counts = false; inline static bool random_direction_in_path = true; + inline static bool random_node_selection = true; inline static bool generate_more_data_after_coverage = true; + inline static bool generate_for_bad_nodes = true; inline static int max_failed_generations_in_row = 2; inline static int minimal_max_generation_after_covered = 10; inline static int minimal_max_generation_for_other_node = 10; inline static int minimal_max_generation_artificial_data = 5; - inline static int maximal_number_of_equations_with_same_branching_value = 2000; + inline static int maximal_number_of_equations_with_same_branching_value = 250; + inline static int maximal_number_of_branching_values = 250; inline static float percentage_to_add_to_path = 0.4; inline static bool create_artificial_data = true; + inline static int biggest_value_in_difference_vector = 10; inline static bool verbose = false; + // inline static bool verbose = true; }; + std::pair< path_id_direction_count, std::size_t > get_directions_in_path( branching_node* node ); bool should_generate_more_data( const generation_state& state ); } // namespace fuzzing diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 1b3bdbe2..c4bea8ee 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -221,6 +221,32 @@ equation fuzzing::equation::add_to_values( const equation& other ) const return { new_values, best_value }; } +// ------------------------------------------------------------------------------------------------ +int fuzzing::equation::simplify_by_gcd() +{ + if ( values.empty() ) { + return 1; + } + + int gcd = std::abs( best_value ); + for ( int i = 0; i < values.size(); ++i ) { + gcd = std::gcd( gcd, values[ i ] ); + } + + if ( gcd == 0 || std::abs( gcd ) == 1 ) { + return 1; + } + + for ( int i = 0; i < values.size(); ++i ) { + values[ i ] /= gcd; + } + + best_value /= gcd; + + return gcd; +} + + // ------------------------------------------------------------------------------------------------ int equation::get_vector_size() const { @@ -346,6 +372,22 @@ location_id::id_type fuzzing::loop_properties::get_smallest_loop_head_id() const return smallest_id; } +// ------------------------------------------------------------------------------------------------ +std::optional< location_id::id_type > fuzzing::loop_properties::get_smallest_body_id() const +{ + TMPROF_BLOCK(); + + std::optional< location_id::id_type > smallest_id; + + for ( const auto& body : bodies ) { + if ( !smallest_id.has_value() || body.node_id < smallest_id.value() ) { + smallest_id = body.node_id; + } + } + + return smallest_id; +} + // ------------------------------------------------------------------------------------------------ location_id::id_type fuzzing::loop_properties::get_smallest_id() const { @@ -638,6 +680,7 @@ equation_matrix equation_matrix::get_submatrix( std::set< node_id_with_direction if ( inserted ) { result.matrix.push_back( new_row ); result.all_paths.push_back( all_paths[ i ] ); + result.branching_values[ row.best_value ]++; } } @@ -653,11 +696,17 @@ void fuzzing::equation_matrix::process_node( branching_node* end_node, { TMPROF_BLOCK(); - int& branching_value_count = branching_values[ end_node->best_coverage_value ]; - if ( branching_value_count >= iid_dependencies::maximal_number_of_equations_with_same_branching_value ) + auto it = branching_values.find( end_node->best_coverage_value ); + + if ( it != branching_values.end() && + branching_values.size() >= iid_dependencies::maximal_number_of_branching_values ) { return; + } - branching_value_count++; + if ( it != branching_values.end() && + it->second >= iid_dependencies::maximal_number_of_equations_with_same_branching_value ) { + return; + } all_paths.push_back( end_node ); @@ -691,7 +740,7 @@ bool equation_matrix::contains( node_id_with_direction const& node ) const // ------------------------------------------------------------------------------------------------ std::pair< std::size_t, std::size_t > equation_matrix::get_dimensions() const { - return { matrix.size(), nodes.size() }; + return { unique_rows.size(), nodes.size() }; } // ------------------------------------------------------------------------------------------------ @@ -710,25 +759,24 @@ const std::unordered_map< equation, int >& equation_matrix::compute_vectors_with equation difference = matrix[ i ] - matrix[ j ]; - if ( difference.is_any_negative() || difference.best_value == 0 ) + if ( difference.is_any_negative() || difference.best_value == 0 || difference.get_biggest_value() > iid_dependencies::biggest_value_in_difference_vector ) continue; - vectors_with_hits[ difference ]; + vectors.insert( difference ); } computed_vectors++; } - for ( auto& [ vector, hits ] : vectors_with_hits ) { - hits = 0; - } + vectors_with_hits.clear(); - for ( const auto& [ vector, hits ] : vectors_with_hits ) { + for ( const auto& vector : vectors ) { for ( const auto& row : matrix ) { equation new_possible_equation = row + vector; - if ( std::find( unique_rows.begin(), unique_rows.end(), new_possible_equation ) != unique_rows.end() ) { - vectors_with_hits[ vector ]++; + equation vector_copy = vector; + int gcd = vector_copy.simplify_by_gcd(); + vectors_with_hits[ vector_copy ] += gcd; } } } @@ -872,6 +920,18 @@ void fuzzing::equation_matrix::add_path( branching_node* end_node, bool add_columns, std::size_t max_directions_in_path_index ) { + auto branching_values_it = branching_values.find( end_node->best_coverage_value ); + + if ( branching_values.size() >= iid_dependencies::maximal_number_of_branching_values && + branching_values_it == branching_values.end() ) { + return; + } + + if ( branching_values_it != branching_values.end() && + branching_values_it->second >= iid_dependencies::maximal_number_of_equations_with_same_branching_value ) { + return; + } + TMPROF_BLOCK(); if ( add_columns ) { @@ -915,6 +975,7 @@ void fuzzing::equation_matrix::add_path( branching_node* end_node, auto [ it, inserted ] = unique_rows.insert( row ); if ( inserted ) { matrix.push_back( row ); + branching_values[ end_node->best_coverage_value ]++; } } @@ -969,21 +1030,22 @@ generated_path iid_node_dependence_props::generate_probabilities( const loop_dep if ( iid_dependencies::verbose ) { print_stats( false ); loop_to_properties.print_dependencies(); - // computation_submatrix.print_matrix(); + computation_submatrix.print_matrix(); } - std::optional< std::vector< equation > > best_vectors = get_best_vectors( computation_submatrix, 1 ); + std::optional< std::vector< equation > > best_vectors; - if ( !best_vectors.has_value() ) { - if ( stats.state != generation_state::STATE_GENERATING_ARTIFICIAL_DATA ) { - if ( iid_dependencies::verbose ) - std::cout << "No vectors" << std::endl; + if ( stats.state == generation_state::STATE_GENERATING_ARTIFICIAL_DATA ) { + best_vectors = generate_vectors_if_not_enough_data( computation_submatrix ); + } else { + best_vectors = get_best_vectors( computation_submatrix, 1 ); + } - return return_empty_path(); - } + if ( !best_vectors.has_value() ) { + if ( iid_dependencies::verbose ) + std::cout << "No vectors" << std::endl; - best_vectors = std::vector< equation >(); - generate_vectors_if_not_enough_data( *best_vectors, computation_submatrix ); + return return_empty_path(); } std::optional< equation > new_subset_counts = @@ -992,7 +1054,7 @@ generated_path iid_node_dependence_props::generate_probabilities( const loop_dep if ( !new_subset_counts.has_value() ) { if ( iid_dependencies::verbose ) std::cout << "No new subset counts" << std::endl; - + return return_empty_path(); } @@ -1022,27 +1084,71 @@ void fuzzing::iid_node_dependence_props::process_path_effective( branching_node* } // ------------------------------------------------------------------------------------------------ -bool iid_node_dependence_props::should_generate( const loop_dependencies& loop_to_properties ) const +bool iid_node_dependence_props::is_covered() const +{ + return stats.state == generation_state::STATE_COVERED || + stats.state == generation_state::STATE_COVERED_BY_OTHER; +} + +// ------------------------------------------------------------------------------------------------ +bool iid_node_dependence_props::should_generate_new( const loop_dependencies& loop_to_properties ) const { + if ( stats.state == generation_state::STATE_COVERED || + stats.state == generation_state::STATE_COVERED_BY_OTHER ) { + return false; + } + if ( !matrix_generated ) { return true; } - // if ( !loop_to_properties.get_node_subsets_for_computation( get_matrix().get_node_ids() ).empty() ) { - // return true; - // } + if ( stats.state == generation_state::STATE_GENERATING_ARTIFICIAL_DATA || + stats.state == generation_state::STATE_GENERATION_DATA_FOR_NEXT_NODE || + stats.state == generation_state::STATE_GENERATION_MORE ) { + return true; + } + + if ( matrix.get_number_of_different_branchings() <= 1 ) { + return false; + } + + std::set< fuzzing::node_id_with_direction > computation_subset; + loop_to_properties.compute_node_subsets_for_computation( computation_subset, matrix.get_node_ids() ); - return stats.state != generation_state::STATE_COVERED; + if ( !computation_subset.empty() ) { + return true; + } + + return false; } // ------------------------------------------------------------------------------------------------ -bool iid_node_dependence_props::too_much_failed_in_row( int max_failed_generations_in_row ) const +failed_generation_method fuzzing::iid_node_dependence_props::determine_recovery_strategy() +{ + switch ( stats.last_failed_method ) { + case failed_generation_method::METHOD_NONE: + return failed_generation_method::METHOD_GENERATE_ARTIFICIAL_DATA; + case failed_generation_method::METHOD_GENERATE_ARTIFICIAL_DATA: + return failed_generation_method::METHOD_GENERATE_FROM_OTHER_NODE; + case failed_generation_method::METHOD_GENERATE_FROM_OTHER_NODE: + return failed_generation_method::METHOD_DO_NOT_GENERATE; + case failed_generation_method::METHOD_DO_NOT_GENERATE: + return failed_generation_method::METHOD_GENERATE_ARTIFICIAL_DATA; + } +} + +// ------------------------------------------------------------------------------------------------ +bool iid_node_dependence_props::too_much_failed_in_row() { if ( stats.state != generation_state::STATE_NOT_COVERED ) { return false; } - if ( stats.failed_generations_in_row > max_failed_generations_in_row ) { + if ( stats.failed_generations_in_row > iid_dependencies::max_failed_generations_in_row ) { + return true; + } + + if ( stats.do_not_generate_counter > 3 ) { return true; } @@ -1065,49 +1171,18 @@ void iid_node_dependence_props::set_as_generating_for_other_node( int minimal_ma // ------------------------------------------------------------------------------------------------ void fuzzing::iid_node_dependence_props::set_as_generating_artificial_data( int minimal_max_generation_artificial_data ) { - INVARIANT( stats.state == generation_state::STATE_NOT_COVERED ); - stats.state = generation_state::STATE_GENERATING_ARTIFICIAL_DATA; stats.generate_artificial_data_max = minimal_max_generation_artificial_data; stats.generate_artificial_data = 0; stats.generate_artificial_data_count++; } -// ------------------------------------------------------------------------------------------------ -failed_generation_method fuzzing::iid_node_dependence_props::get_method_for_failed_generation( bool is_first ) -{ - failed_generation_method new_method; - - if ( !iid_dependencies::create_artificial_data ) { - return failed_generation_method::METHOD_GENERATE_FROM_OTHER_NODE; - } else if ( is_first ) { - new_method = failed_generation_method::METHOD_GENERATE_ARTIFICIAL_DATA; - } else { - switch ( stats.last_failed_method ) { - case failed_generation_method::METHOD_GENERATE_ARTIFICIAL_DATA: - new_method = failed_generation_method::METHOD_GENERATE_FROM_OTHER_NODE; - break; - case failed_generation_method::METHOD_GENERATE_FROM_OTHER_NODE: - new_method = failed_generation_method::METHOD_GENERATE_ARTIFICIAL_DATA; - break; - } - } - - if ( new_method == failed_generation_method::METHOD_GENERATE_ARTIFICIAL_DATA ) { - set_as_generating_artificial_data( iid_dependencies::minimal_max_generation_artificial_data ); - } - - stats.last_failed_method = new_method; - return new_method; -} - // ------------------------------------------------------------------------------------------------ bool iid_node_dependence_props::is_equal_branching_predicate() const { return matrix.get_branching_predicate() == BRANCHING_PREDICATE::BP_EQUAL; } - // ------------------------------------------------------------------------------------------------ void iid_node_dependence_props::print_stats( bool only_state ) const { @@ -1148,12 +1223,12 @@ void iid_node_dependence_props::print_stats( bool only_state ) const } // ------------------------------------------------------------------------------------------------ -void fuzzing::iid_node_dependence_props::generate_vectors_if_not_enough_data( std::vector< equation >& best_vectors, - equation_matrix& submatrix ) +std::optional< std::vector< equation > > +fuzzing::iid_node_dependence_props::generate_vectors_if_not_enough_data( equation_matrix& submatrix ) { TMPROF_BLOCK(); - best_vectors = std::vector< equation >(); + std::vector< equation > best_vectors; int desired_direction = submatrix.get_desired_vector_direction(); const std::unordered_map< equation, int >& vectors = submatrix.compute_vectors_with_hits(); @@ -1182,6 +1257,12 @@ void fuzzing::iid_node_dependence_props::generate_vectors_if_not_enough_data( st add_to_best_vectors( modified_vector.values ); } } + + if ( best_vectors.empty() ) { + return std::nullopt; + } + + return best_vectors; } // ------------------------------------------------------------------------------------------------ @@ -1195,6 +1276,17 @@ fuzzing::iid_node_dependence_props::get_best_vectors( equation_matrix& submatrix return std::nullopt; } + if ( iid_dependencies::verbose ) { + for ( const auto& [ vector, hits ] : vectors ) { + std::cout << "Vector: "; + for ( const auto& value : vector.values ) { + std::cout << value << " "; + } + std::cout << "-> Best Value: " << vector.best_value << " (" << hits << ")" << std::endl; + } + } + + int desired_vector_direction = submatrix.get_desired_vector_direction(); float biggest_branching_value = submatrix.get_biggest_branching_value(); std::vector< equation > best_vectors = @@ -1204,6 +1296,17 @@ fuzzing::iid_node_dependence_props::get_best_vectors( equation_matrix& submatrix return std::nullopt; } + + if ( iid_dependencies::verbose ) { + for ( const auto& vector : best_vectors ) { + std::cout << "Vector: "; + for ( const auto& value : vector.values ) { + std::cout << value << " "; + } + std::cout << "-> Best Value: " << vector.best_value << std::endl; + } + } + return best_vectors; } @@ -1488,7 +1591,7 @@ iid_node_dependence_props::compute_node_counts( const equation& path, } for ( const auto& loop_props : std::ranges::views::reverse( loop_to_properties.loops ) ) { - if ( loop_props.bodies.empty() || !subset_ids.contains( loop_props.get_smallest_loop_head_id() ) ) { + if ( !subset_ids.contains( loop_props.get_smallest_loop_head_id() ) ) { continue; } @@ -1498,6 +1601,14 @@ iid_node_dependence_props::compute_node_counts( const equation& path, loop_count = std::max( loop_count, path_counts[ body.node_id ].get_total_count() ); } + for ( const auto& head : loop_props.heads ) { + loop_count = std::max( loop_count, path_counts[ head.first.node_id ].get_total_count() ); + } + + if ( loop_count == 0 ) { + continue; + } + for ( const auto& [ head, _ ] : loop_props.heads ) { int end_count = head == ( *loop_props.chosen_loop_head ) ? 1 : 0; @@ -1564,24 +1675,20 @@ iid_node_dependence_props::compute_best_vectors( const std::unordered_map< equat return a.second > b.second; } ); - bool use_linear_dependency = true; - std::vector< equation > best_vectors; - for ( int i = 0; i < number_of_vectors && i < sorted_vectors.size(); ++i ) { - if ( use_linear_dependency ) { - std::unordered_map< equation, int > dependent_vectors_with_hits = - get_linear_dependent_vector( filtered_vectors_with_hits, sorted_vectors[ i ].first ); - - auto it = std::min_element( dependent_vectors_with_hits.begin(), - dependent_vectors_with_hits.end(), - []( const auto& a, const auto& b ) { - return a.first.get_vector_size() < b.first.get_vector_size(); - } ); - best_vectors.push_back( it->first ); - } else { - best_vectors.push_back( sorted_vectors[ i ].first ); - } + if ( sorted_vectors.size() > 5 ) { + sorted_vectors.erase( sorted_vectors.begin() + 5, sorted_vectors.end() ); } + std::sort( sorted_vectors.begin(), sorted_vectors.end(), []( const auto& a, const auto& b ) { + if ( a.first.get_vector_size() == b.first.get_vector_size() ) + return a.second > b.second; + + return a.first.get_vector_size() < b.first.get_vector_size(); + } ); + + std::vector< equation > best_vectors; + best_vectors.push_back( sorted_vectors[ 0 ].first ); + return best_vectors; } @@ -1716,34 +1823,70 @@ std::optional< location_id > iid_dependencies::get_next_iid_node() { TMPROF_BLOCK(); - bool previous_needs_more_data = false; - for ( auto it = node_id_to_equation_map.rbegin(); it != node_id_to_equation_map.rend(); ++it ) { + std::vector< location_id > all_non_covered; + std::vector< location_id > possible_nodes; + + for ( auto it = node_id_to_equation_map.begin(); it != node_id_to_equation_map.end(); ++it ) { iid_node_dependence_props& props = it->second; - if ( previous_needs_more_data ) { - props.set_as_generating_for_other_node( iid_dependencies::minimal_max_generation_for_other_node ); - previous_needs_more_data = false; + auto& stats = props.get_generations_stats(); + + if ( !props.is_covered() ) + all_non_covered.push_back( it->first ); + + if ( stats.last_failed_method == failed_generation_method::METHOD_DO_NOT_GENERATE ) { + stats.do_not_generate_counter++; } - if ( props.too_much_failed_in_row( iid_dependencies::max_failed_generations_in_row ) ) { - props.get_generations_stats().failed_generations_in_row = 0; - bool is_first = std::next( it ) == node_id_to_equation_map.rend(); + if ( props.too_much_failed_in_row() ) { + stats.failed_generations_in_row = 0; + stats.do_not_generate_counter = 0; - failed_generation_method method = props.get_method_for_failed_generation( is_first ); + failed_generation_method recovery_method = props.determine_recovery_strategy(); + auto prev_it = std::prev( it ); - if ( method == failed_generation_method::METHOD_GENERATE_FROM_OTHER_NODE ) { - previous_needs_more_data = true; + if ( recovery_method == failed_generation_method::METHOD_GENERATE_ARTIFICIAL_DATA ) { + props.set_as_generating_artificial_data( iid_dependencies::minimal_max_generation_artificial_data ); + stats.last_failed_method = failed_generation_method::METHOD_GENERATE_ARTIFICIAL_DATA; + } else if ( prev_it != node_id_to_equation_map.end() && + recovery_method == failed_generation_method::METHOD_GENERATE_FROM_OTHER_NODE ) { + iid_node_dependence_props& prev_props = prev_it->second; + prev_props.set_as_generating_for_other_node( + iid_dependencies::minimal_max_generation_for_other_node ); + possible_nodes.push_back( prev_it->first ); + stats.last_failed_method = failed_generation_method::METHOD_GENERATE_FROM_OTHER_NODE; + } else { + stats.last_failed_method = failed_generation_method::METHOD_DO_NOT_GENERATE; } } + + if ( props.should_generate_new( loop_to_properties ) ) { + possible_nodes.push_back( it->first ); + } } - for ( const auto& [ id, props ] : node_id_to_equation_map ) { - if ( props.should_generate( loop_to_properties ) ) { - return id; + if ( possible_nodes.empty() ) { + if ( verbose ) + std::cout << "No more nodes to generate." << std::endl; + + if ( generate_for_bad_nodes && !all_non_covered.empty() ) { + return random_node_selection ? all_non_covered[ rand() % all_non_covered.size() ] : + all_non_covered[ 0 ]; } + + return std::nullopt; } - return std::nullopt; + if ( verbose ) { + std::cout << "Possible nodes to generate: "; + for ( const auto& node : possible_nodes ) { + std::cout << node << " "; + } + std::cout << std::endl; + } + + return iid_dependencies::random_node_selection ? possible_nodes[ rand() % possible_nodes.size() ] : + possible_nodes[ 0 ]; } // ------------------------------------------------------------------------------------------------ @@ -1754,6 +1897,13 @@ void fuzzing::iid_dependencies::compute_dependencies() dependencies_computed++; for ( branching_node* end_node : end_nodes ) { + auto it = node_id_to_equation_map.find( end_node->get_location_id() ); + if ( it != node_id_to_equation_map.end() && + ( it->second.get_generations_stats().state == generation_state::STATE_COVERED || + it->second.get_generations_stats().state == generation_state::STATE_COVERED_BY_OTHER ) ) { + continue; + } + loop_head_to_bodies_t loop_heads_to_bodies; loop_endings loop_heads_ending = get_loop_heads_ending( end_node, loop_heads_to_bodies ); From 8fd147deb8d4eb1eb7cc2455aac0be407c08d146 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 12 Apr 2025 21:34:40 +0200 Subject: [PATCH 134/144] feat: formatter --- src/fuzzing/include/fuzzing/iid_vector_analysis.hpp | 1 - src/fuzzing/src/iid_vector_analysis.cpp | 7 ++++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index 2689cd71..d88ad399 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -483,7 +483,6 @@ struct iid_dependencies { }; - std::pair< path_id_direction_count, std::size_t > get_directions_in_path( branching_node* node ); bool should_generate_more_data( const generation_state& state ); } // namespace fuzzing diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index c4bea8ee..74375a5e 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -759,7 +759,8 @@ const std::unordered_map< equation, int >& equation_matrix::compute_vectors_with equation difference = matrix[ i ] - matrix[ j ]; - if ( difference.is_any_negative() || difference.best_value == 0 || difference.get_biggest_value() > iid_dependencies::biggest_value_in_difference_vector ) + if ( difference.is_any_negative() || difference.best_value == 0 || + difference.get_biggest_value() > iid_dependencies::biggest_value_in_difference_vector ) continue; vectors.insert( difference ); @@ -1868,10 +1869,10 @@ std::optional< location_id > iid_dependencies::get_next_iid_node() if ( possible_nodes.empty() ) { if ( verbose ) std::cout << "No more nodes to generate." << std::endl; - + if ( generate_for_bad_nodes && !all_non_covered.empty() ) { return random_node_selection ? all_non_covered[ rand() % all_non_covered.size() ] : - all_non_covered[ 0 ]; + all_non_covered[ 0 ]; } return std::nullopt; From 2485fb14ee616d93df9209df33267a4cfead8d01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sun, 13 Apr 2025 08:50:38 +0200 Subject: [PATCH 135/144] fix: json properties --- benchmarks/iid_testing/ai_generated_20.json | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/benchmarks/iid_testing/ai_generated_20.json b/benchmarks/iid_testing/ai_generated_20.json index 11765dc2..5f667ed5 100644 --- a/benchmarks/iid_testing/ai_generated_20.json +++ b/benchmarks/iid_testing/ai_generated_20.json @@ -35,12 +35,7 @@ ], "output_statistics": { "sensitivity_analysis": { - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 - }, - "typed_minimization_analysis": { - "num_generated_tests": 2, + "num_generated_tests": 4, "num_crashes": 0, "num_boundary_violations": 0 }, From 661e0ebf37f95009d30786bef40c0b068026df17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sun, 13 Apr 2025 23:09:55 +0200 Subject: [PATCH 136/144] feat: implement better handling of loading loops --- benchmarks/iid_testing/ai_generated_04.c | 47 +++++++++++++ benchmarks/iid_testing/ai_generated_04.json | 58 ++++++++++++++++ benchmarks/iid_testing/ai_generated_06.c | 42 ++++++++++++ benchmarks/iid_testing/ai_generated_06.json | 62 +++++++++++++++++ benchmarks/iid_testing/iid_with_dumy_load.c | 41 ++++++++++++ .../iid_testing/iid_with_dumy_load.json | 61 +++++++++++++++++ .../iid_testing/loading_loop_2_at_a_time.json | 2 +- .../include/fuzzing/iid_vector_analysis.hpp | 3 + src/fuzzing/src/iid_vector_analysis.cpp | 66 ++++++++++++++----- 9 files changed, 364 insertions(+), 18 deletions(-) create mode 100644 benchmarks/iid_testing/ai_generated_04.c create mode 100644 benchmarks/iid_testing/ai_generated_04.json create mode 100644 benchmarks/iid_testing/ai_generated_06.c create mode 100644 benchmarks/iid_testing/ai_generated_06.json create mode 100644 benchmarks/iid_testing/iid_with_dumy_load.c create mode 100644 benchmarks/iid_testing/iid_with_dumy_load.json diff --git a/benchmarks/iid_testing/ai_generated_04.c b/benchmarks/iid_testing/ai_generated_04.c new file mode 100644 index 00000000..52a36171 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_04.c @@ -0,0 +1,47 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +// Simple struct definition +typedef struct { + int id; + char status; // e.g., 'A'ctive, 'I'nactive +} Item; + +#define MAX_ITEMS 40 + +int main() +{ + Item inventory[ MAX_ITEMS ]; // Array of structs + short num_items; + int active_count = 0; // State variable + + // 1. Fill data structure (array of structs) in an initial loop + num_items = __VERIFIER_nondet_short(); + if ( num_items < 0 || num_items > MAX_ITEMS ) { + return -1; + } + for ( short i = 0; i < num_items; ++i ) { + inventory[ i ].id = __VERIFIER_nondet_int(); // Assign some ID + inventory[ i ].status = __VERIFIER_nondet_char(); // Assign status + } + + // 2. Core logic in loop iterating through the array of structs + for ( short i = 0; i < num_items; ++i ) { + // 3. Modify internal state based on struct member + // 5. Change is constant (+1) + if ( inventory[ i ].status == 'A' ) { + active_count++; // Constant change + } + } // Loop always completes + + // 4. Final condition based on internal variable + if ( active_count >= 10 ) { // Check if at least half are active + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_04.json b/benchmarks/iid_testing/ai_generated_04.json new file mode 100644 index 00000000..6da49d82 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_04.json @@ -0,0 +1,58 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 6508, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_06.c b/benchmarks/iid_testing/ai_generated_06.c new file mode 100644 index 00000000..aed7d04b --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_06.c @@ -0,0 +1,42 @@ +#include + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 50 + +int main() +{ + char data1[ MAX_SIZE ]; + char data2[ MAX_SIZE ]; // Second array + short size; + int match_count = 0; // State variable + + // 1. Fill data structures in loops + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data1[ i ] = __VERIFIER_nondet_char(); + data2[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop iterating through data + for ( short i = 0; i < size; ++i ) { + // 3. Modify internal state based on data from both arrays + // 6. Change is static (+1) + if ( data1[ i ] == 'S' && data2[ i ] == 'T' ) { + match_count++; // Static change + } + } + + // 4. Final condition based on internal variable + // 5. Condition check is linear + if ( match_count > 10 ) { // Check if the S/T pair occurred more than once + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_06.json b/benchmarks/iid_testing/ai_generated_06.json new file mode 100644 index 00000000..19a9682f --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_06.json @@ -0,0 +1,62 @@ +{ + "args": { + "max_executions": 20000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 10743, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "num_uncovered_branchings": 0, + "uncovered_branchings": [], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/iid_with_dumy_load.c b/benchmarks/iid_testing/iid_with_dumy_load.c new file mode 100644 index 00000000..09d0aa3d --- /dev/null +++ b/benchmarks/iid_testing/iid_with_dumy_load.c @@ -0,0 +1,41 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + char tmp; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) { // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + tmp = __VERIFIER_nondet_char(); + tmp = __VERIFIER_nondet_char(); + tmp = __VERIFIER_nondet_char(); + tmp = __VERIFIER_nondet_char(); + } + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + if ( s[ i ] == 'A' ) // ID: 6 + ++k; + ++i; + } + if ( k == 10 ) // ID: 7 + return 1; + return 0; + } +} diff --git a/benchmarks/iid_testing/iid_with_dumy_load.json b/benchmarks/iid_testing/iid_with_dumy_load.json new file mode 100644 index 00000000..cc7cbb2f --- /dev/null +++ b/benchmarks/iid_testing/iid_with_dumy_load.json @@ -0,0 +1,61 @@ +{ + "args": { + "max_executions": 20000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 13558, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/loading_loop_2_at_a_time.json b/benchmarks/iid_testing/loading_loop_2_at_a_time.json index 02cc3e73..7c9094b3 100644 --- a/benchmarks/iid_testing/loading_loop_2_at_a_time.json +++ b/benchmarks/iid_testing/loading_loop_2_at_a_time.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 3794, + "num_executions": 5316, "num_covered_branchings": 8, "covered_branchings": [ 1, diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index d88ad399..c5fc6a56 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -84,6 +84,7 @@ struct iid_node_generations_stats { struct loaded_bits_props { mean_counter< float > average_bits_read; natural_32_bit minimal_bit_offset = std::numeric_limits< natural_32_bit >::max(); + mean_counter< float > average_bits_used; }; struct loop_head_properties { @@ -289,6 +290,8 @@ struct loaded_bits_counter { natural_32_bit min; natural_32_bit max; int loop_count; + + std::set< std::pair< natural_32_bit, natural_32_bit > > loaded_intervals; }; struct FloatComparator { diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index 74375a5e..d6bae5db 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -640,7 +640,8 @@ void fuzzing::loop_dependencies::print_dependencies() const std::cout << "Dependent nodes:" << std::endl; for ( const auto& [ body, body_props ] : loop.bits_read_by_node ) { std::cout << "- " << body << ", Bits: " << body_props.average_bits_read.mean - << ", offset: " << body_props.minimal_bit_offset << std::endl; + << ", offset: " << body_props.minimal_bit_offset + << " bits used: " << body_props.average_bits_used.mean << std::endl; } std::cout << "Loaded bits per loop: " << loop.loaded_bits_per_loop.mean << std::endl; @@ -1059,6 +1060,14 @@ generated_path iid_node_dependence_props::generate_probabilities( const loop_dep return return_empty_path(); } + if ( iid_dependencies::verbose ) { + std::cout << "New subset counts: " << std::endl; + for (size_t i = 0; i < new_subset_counts->values.size(); ++i) { + std::cout << new_subset_counts->values[i] << " "; + } + std::cout << "-> Best Value: " << new_subset_counts->best_value << std::endl; + } + nodes_to_counts node_counts = compute_node_counts( *new_subset_counts, computation_subset, loop_to_properties, subset_ids ); generated_path path = generate_path_from_node_counts( node_counts, loop_to_properties ); @@ -1435,13 +1444,15 @@ int fuzzing::iid_node_dependence_props::compute_loading_loop_interation( { TMPROF_BLOCK(); - float loaded_per_loop = props.loaded_bits_per_loop.mean; - if ( loaded_per_loop <= 0 ) { - loaded_per_loop = 8; - } - float average_bits = props.bits_read_by_node.at( id ).average_bits_read.mean; natural_32_bit offset = props.bits_read_by_node.at( id ).minimal_bit_offset; + float average_used_bits = props.bits_read_by_node.at( id ).average_bits_used.mean; + + float loaded_per_loop = std::min( props.loaded_bits_per_loop.mean, average_used_bits ); + loaded_per_loop = std::max( loaded_per_loop, 8.0f ); + + if ( offset == natural_32_bit( props.loaded_bits_per_loop.mean - average_used_bits ) ) + offset = 0; bool is_loop_head = true; if ( !loop_heads.contains( id ) ) { @@ -2033,6 +2044,7 @@ void fuzzing::iid_dependencies::compute_dependencies_by_loading( branching_node* natural_32_bit min = std::numeric_limits< natural_32_bit >::max(); natural_32_bit max = std::numeric_limits< natural_32_bit >::min(); std::vector< natural_32_bit > sensitive_stdin_bit_counts; + std::unordered_set< stdin_bit_index > sensitivity_bits = {}; }; std::unordered_map< location_id::id_type, std::unordered_map< location_id::id_type, dependent_body_props > > loop_head_to_props; @@ -2061,12 +2073,13 @@ void fuzzing::iid_dependencies::compute_dependencies_by_loading( branching_node* props.min = std::min( props.min, node_min ); props.max = std::max( props.max, node_max ); props.sensitive_stdin_bit_counts.push_back( sensitive_bits.size() ); + props.sensitivity_bits.insert( sensitive_bits.begin(), sensitive_bits.end() ); node = node->predecessor; } for ( const auto& [ node_id, props ] : node_id_to_props ) { - const auto& [ node_min, node_max, sensitive_counts ] = props; + const auto& [ node_min, node_max, sensitive_counts, sensitive_bits ] = props; for ( const auto& [ loop_head, loop_props ] : loading_loops ) { if ( !loop_heads_ending.contains( loop_head ) ) { @@ -2080,6 +2093,7 @@ void fuzzing::iid_dependencies::compute_dependencies_by_loading( branching_node* loop_body_props.sensitive_stdin_bit_counts.insert( loop_body_props.sensitive_stdin_bit_counts.end(), sensitive_counts.begin(), sensitive_counts.end() ); + loop_body_props.sensitivity_bits.insert( sensitive_bits.begin(), sensitive_bits.end() ); } } } @@ -2102,6 +2116,22 @@ void fuzzing::iid_dependencies::compute_dependencies_by_loading( branching_node* for ( const auto& [ body_id, props ] : body ) { auto& body_props = dependencies.bits_read_by_node[ body_id ]; + + for ( const auto& [ start, end ] : loading_props.loaded_intervals ) { + int count = 0; + for ( int i = start; i < end; ++i ) { + if ( props.sensitivity_bits.contains( i ) ) { + count++; + } + } + + if ( count == 0 ) { + continue; + } + + body_props.average_bits_used.add( count ); + } + natural_32_bit minimal_offset = props.min - loading_props.min; if ( props.min < loading_props.min ) { minimal_offset = 0; @@ -2195,20 +2225,23 @@ void fuzzing::iid_dependencies::compute_loading_loops( branching_node* end_node, TMPROF_BLOCK(); for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { - loading_loops[ loop_head.id ] = { std::numeric_limits< natural_32_bit >::max(), - std::numeric_limits< natural_32_bit >::min(), - 0 }; + loading_loops[ loop_head.id ] = { + std::numeric_limits< natural_32_bit >::max(), std::numeric_limits< natural_32_bit >::min(), 0, {} + }; } branching_node* node = end_node; while ( node != nullptr ) { - const auto node_id = node->get_location_id().id; + location_id node_id = node->get_location_id(); - if ( auto it = loop_heads_to_bodies.find( node->get_location_id() ); it != loop_heads_to_bodies.end() ) { - const auto& loop_head = it->first; - auto& props = loading_loops[ loop_head.id ]; + if ( auto it = loop_heads_to_bodies.find( node_id ); it != loop_heads_to_bodies.end() ) { + auto& props = loading_loops[ node_id.id ]; natural_32_bit bits_count = node->get_num_stdin_bits(); + + if ( props.min != std::numeric_limits< natural_32_bit >::max() ) + props.loaded_intervals.insert( { bits_count, props.min } ); + props.min = std::min( props.min, bits_count ); props.max = std::max( props.max, bits_count ); props.loop_count++; @@ -2218,11 +2251,10 @@ void fuzzing::iid_dependencies::compute_loading_loops( branching_node* end_node, branching_node* predecessor = node->predecessor; const auto predecessor_id = predecessor->get_location_id().id; - if ( loop_heads_ending.contains( predecessor_id ) ) { + if ( auto it = loop_heads_ending.find( predecessor_id ); it != loop_heads_ending.end() ) { bool node_direction = predecessor->successor( true ).pointer == node; - if ( loop_heads_ending.at( predecessor_id ) == node_direction ) { + if ( it->second == node_direction ) loading_loops[ predecessor_id ].loop_count--; - } } } From 6c2b3daf1251e5dac0b5c17dec280743c199449a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Mon, 21 Apr 2025 11:29:39 +0200 Subject: [PATCH 137/144] feat: added more benchmarks --- .../gen_case_01_char_count_eq_20.c | 42 ++++++++++++ .../gen_case_01_char_count_eq_20.json | 53 +++++++++++++++ .../gen_case_02_char_count_gt_15.c | 41 ++++++++++++ .../gen_case_02_char_count_gt_15.json | 53 +++++++++++++++ .../gen_case_04_char_balance_AB_eq_13.c | 43 ++++++++++++ .../gen_case_04_char_balance_AB_eq_13.json | 55 ++++++++++++++++ .../gen_case_05_char_range_58_eq_3.c | 41 ++++++++++++ .../gen_case_05_char_range_58_eq_3.json | 55 ++++++++++++++++ ...en_case_07_char_continue_S_count_F_ge_10.c | 44 +++++++++++++ ...case_07_char_continue_S_count_F_ge_10.json | 55 ++++++++++++++++ .../gen_case_08_char_break_Q_count_P_eq_12.c | 44 +++++++++++++ ...en_case_08_char_break_Q_count_P_eq_12.json | 55 ++++++++++++++++ ...n_case_12_char_multi_state_A_gt_B_plus_5.c | 44 +++++++++++++ ...ase_12_char_multi_state_A_gt_B_plus_5.json | 55 ++++++++++++++++ ...case_13_char_multi_state_XYZ_combo_eq_10.c | 47 ++++++++++++++ ...e_13_char_multi_state_XYZ_combo_eq_10.json | 57 ++++++++++++++++ ...char_multi_state_firstF_lastL_dist_gt_10.c | 46 +++++++++++++ ...r_multi_state_firstF_lastL_dist_gt_10.json | 61 +++++++++++++++++ .../gen_case_15_char_two_loops_A_or_B_eq_10.c | 49 ++++++++++++++ ...n_case_15_char_two_loops_A_or_B_eq_10.json | 59 +++++++++++++++++ ...se_16_char_two_loops_dependent_ABCD_eq_7.c | 46 +++++++++++++ ...16_char_two_loops_dependent_ABCD_eq_7.json | 61 +++++++++++++++++ .../gen_case_18_int_count_pos_gt_20.c | 41 ++++++++++++ .../gen_case_18_int_count_pos_gt_20.json | 48 ++++++++++++++ .../gen_case_24_struct_item_status_A_ge_10.c | 49 ++++++++++++++ ...en_case_24_struct_item_status_A_ge_10.json | 53 +++++++++++++++ ..._case_26_struct_dataval_balance_PN_eq_20.c | 51 +++++++++++++++ ...se_26_struct_dataval_balance_PN_eq_20.json | 55 ++++++++++++++++ ...ase_28_char_count_X_nested_check_eq_8_11.c | 41 ++++++++++++ ..._28_char_count_X_nested_check_eq_8_11.json | 55 ++++++++++++++++ ...n_case_29_char_count_A_deep_nested_check.c | 42 ++++++++++++ ...ase_29_char_count_A_deep_nested_check.json | 57 ++++++++++++++++ ...en_case_30_char_count_A_mult_check_eq_30.c | 39 +++++++++++ ...case_30_char_count_A_mult_check_eq_30.json | 53 +++++++++++++++ ...n_case_31_char_state_dependent_inc_gt_50.c | 45 +++++++++++++ ...ase_31_char_state_dependent_inc_gt_50.json | 55 ++++++++++++++++ ...en_case_32_two_char_arrays_match_ST_gt_5.c | 41 ++++++++++++ ...case_32_two_char_arrays_match_ST_gt_5.json | 55 ++++++++++++++++ ...se_33_char_state_machine_XYZ_count2_gt_3.c | 47 ++++++++++++++ ...33_char_state_machine_XYZ_count2_gt_3.json | 61 +++++++++++++++++ ...n_case_35_char_pointer_loop_count_P_eq_5.c | 45 +++++++++++++ ...ase_35_char_pointer_loop_count_P_eq_5.json | 53 +++++++++++++++ ..._case_39_char_input_cycle_nested_A_eq_10.c | 54 +++++++++++++++ ...se_39_char_input_cycle_nested_A_eq_10.json | 65 +++++++++++++++++++ .../gen_case_40_char_count_K_lt_10.c | 41 ++++++++++++ .../gen_case_40_char_count_K_lt_10.json | 53 +++++++++++++++ .../gen_case_42_char_balance_U2_R1_gt_10.c | 43 ++++++++++++ .../gen_case_42_char_balance_U2_R1_gt_10.json | 55 ++++++++++++++++ .../gen_case_44_char_count_H_eq_12.c | 41 ++++++++++++ .../gen_case_44_char_count_H_eq_12.json | 53 +++++++++++++++ .../gen_case_49_char_balance_Z1_A1_gt_15.c | 43 ++++++++++++ .../gen_case_49_char_balance_Z1_A1_gt_15.json | 55 ++++++++++++++++ .../gen_case_50_char_count_R_gt_12.c | 41 ++++++++++++ .../gen_case_50_char_count_R_gt_12.json | 53 +++++++++++++++ ...n_case_51_char_multi_state_R_gt_P_plus_2.c | 44 +++++++++++++ ...ase_51_char_multi_state_R_gt_P_plus_2.json | 55 ++++++++++++++++ .../gen_case_53_char_two_loops_dep_PQ_ge_7.c | 44 +++++++++++++ ...en_case_53_char_two_loops_dep_PQ_ge_7.json | 57 ++++++++++++++++ .../gen_case_54_char_count_S_ge_5.c | 41 ++++++++++++ .../gen_case_54_char_count_S_ge_5.json | 53 +++++++++++++++ 60 files changed, 2983 insertions(+) create mode 100644 benchmarks/iid_testing/gen_case_01_char_count_eq_20.c create mode 100644 benchmarks/iid_testing/gen_case_01_char_count_eq_20.json create mode 100644 benchmarks/iid_testing/gen_case_02_char_count_gt_15.c create mode 100644 benchmarks/iid_testing/gen_case_02_char_count_gt_15.json create mode 100644 benchmarks/iid_testing/gen_case_04_char_balance_AB_eq_13.c create mode 100644 benchmarks/iid_testing/gen_case_04_char_balance_AB_eq_13.json create mode 100644 benchmarks/iid_testing/gen_case_05_char_range_58_eq_3.c create mode 100644 benchmarks/iid_testing/gen_case_05_char_range_58_eq_3.json create mode 100644 benchmarks/iid_testing/gen_case_07_char_continue_S_count_F_ge_10.c create mode 100644 benchmarks/iid_testing/gen_case_07_char_continue_S_count_F_ge_10.json create mode 100644 benchmarks/iid_testing/gen_case_08_char_break_Q_count_P_eq_12.c create mode 100644 benchmarks/iid_testing/gen_case_08_char_break_Q_count_P_eq_12.json create mode 100644 benchmarks/iid_testing/gen_case_12_char_multi_state_A_gt_B_plus_5.c create mode 100644 benchmarks/iid_testing/gen_case_12_char_multi_state_A_gt_B_plus_5.json create mode 100644 benchmarks/iid_testing/gen_case_13_char_multi_state_XYZ_combo_eq_10.c create mode 100644 benchmarks/iid_testing/gen_case_13_char_multi_state_XYZ_combo_eq_10.json create mode 100644 benchmarks/iid_testing/gen_case_14_char_multi_state_firstF_lastL_dist_gt_10.c create mode 100644 benchmarks/iid_testing/gen_case_14_char_multi_state_firstF_lastL_dist_gt_10.json create mode 100644 benchmarks/iid_testing/gen_case_15_char_two_loops_A_or_B_eq_10.c create mode 100644 benchmarks/iid_testing/gen_case_15_char_two_loops_A_or_B_eq_10.json create mode 100644 benchmarks/iid_testing/gen_case_16_char_two_loops_dependent_ABCD_eq_7.c create mode 100644 benchmarks/iid_testing/gen_case_16_char_two_loops_dependent_ABCD_eq_7.json create mode 100644 benchmarks/iid_testing/gen_case_18_int_count_pos_gt_20.c create mode 100644 benchmarks/iid_testing/gen_case_18_int_count_pos_gt_20.json create mode 100644 benchmarks/iid_testing/gen_case_24_struct_item_status_A_ge_10.c create mode 100644 benchmarks/iid_testing/gen_case_24_struct_item_status_A_ge_10.json create mode 100644 benchmarks/iid_testing/gen_case_26_struct_dataval_balance_PN_eq_20.c create mode 100644 benchmarks/iid_testing/gen_case_26_struct_dataval_balance_PN_eq_20.json create mode 100644 benchmarks/iid_testing/gen_case_28_char_count_X_nested_check_eq_8_11.c create mode 100644 benchmarks/iid_testing/gen_case_28_char_count_X_nested_check_eq_8_11.json create mode 100644 benchmarks/iid_testing/gen_case_29_char_count_A_deep_nested_check.c create mode 100644 benchmarks/iid_testing/gen_case_29_char_count_A_deep_nested_check.json create mode 100644 benchmarks/iid_testing/gen_case_30_char_count_A_mult_check_eq_30.c create mode 100644 benchmarks/iid_testing/gen_case_30_char_count_A_mult_check_eq_30.json create mode 100644 benchmarks/iid_testing/gen_case_31_char_state_dependent_inc_gt_50.c create mode 100644 benchmarks/iid_testing/gen_case_31_char_state_dependent_inc_gt_50.json create mode 100644 benchmarks/iid_testing/gen_case_32_two_char_arrays_match_ST_gt_5.c create mode 100644 benchmarks/iid_testing/gen_case_32_two_char_arrays_match_ST_gt_5.json create mode 100644 benchmarks/iid_testing/gen_case_33_char_state_machine_XYZ_count2_gt_3.c create mode 100644 benchmarks/iid_testing/gen_case_33_char_state_machine_XYZ_count2_gt_3.json create mode 100644 benchmarks/iid_testing/gen_case_35_char_pointer_loop_count_P_eq_5.c create mode 100644 benchmarks/iid_testing/gen_case_35_char_pointer_loop_count_P_eq_5.json create mode 100644 benchmarks/iid_testing/gen_case_39_char_input_cycle_nested_A_eq_10.c create mode 100644 benchmarks/iid_testing/gen_case_39_char_input_cycle_nested_A_eq_10.json create mode 100644 benchmarks/iid_testing/gen_case_40_char_count_K_lt_10.c create mode 100644 benchmarks/iid_testing/gen_case_40_char_count_K_lt_10.json create mode 100644 benchmarks/iid_testing/gen_case_42_char_balance_U2_R1_gt_10.c create mode 100644 benchmarks/iid_testing/gen_case_42_char_balance_U2_R1_gt_10.json create mode 100644 benchmarks/iid_testing/gen_case_44_char_count_H_eq_12.c create mode 100644 benchmarks/iid_testing/gen_case_44_char_count_H_eq_12.json create mode 100644 benchmarks/iid_testing/gen_case_49_char_balance_Z1_A1_gt_15.c create mode 100644 benchmarks/iid_testing/gen_case_49_char_balance_Z1_A1_gt_15.json create mode 100644 benchmarks/iid_testing/gen_case_50_char_count_R_gt_12.c create mode 100644 benchmarks/iid_testing/gen_case_50_char_count_R_gt_12.json create mode 100644 benchmarks/iid_testing/gen_case_51_char_multi_state_R_gt_P_plus_2.c create mode 100644 benchmarks/iid_testing/gen_case_51_char_multi_state_R_gt_P_plus_2.json create mode 100644 benchmarks/iid_testing/gen_case_53_char_two_loops_dep_PQ_ge_7.c create mode 100644 benchmarks/iid_testing/gen_case_53_char_two_loops_dep_PQ_ge_7.json create mode 100644 benchmarks/iid_testing/gen_case_54_char_count_S_ge_5.c create mode 100644 benchmarks/iid_testing/gen_case_54_char_count_S_ge_5.json diff --git a/benchmarks/iid_testing/gen_case_01_char_count_eq_20.c b/benchmarks/iid_testing/gen_case_01_char_count_eq_20.c new file mode 100644 index 00000000..58b24266 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_01_char_count_eq_20.c @@ -0,0 +1,42 @@ +/* File: gen_case_01_char_count_eq_10.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data[MAX_SIZE]; + short size; + int count = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + // 3. Modify internal state based on data/condition + if (data[i] == 'X') { + count++; // 6. Static change (+1) + } + } + // 4. Final condition based on internal variable(s) + if (count == 10) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_01_char_count_eq_20.json b/benchmarks/iid_testing/gen_case_01_char_count_eq_20.json new file mode 100644 index 00000000..0a7146b6 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_01_char_count_eq_20.json @@ -0,0 +1,53 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 3114, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_02_char_count_gt_15.c b/benchmarks/iid_testing/gen_case_02_char_count_gt_15.c new file mode 100644 index 00000000..757121c5 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_02_char_count_gt_15.c @@ -0,0 +1,41 @@ +/* File: gen_case_02_char_count_gt_15.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data[MAX_SIZE]; + short size; + int y_count = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'Y') { + y_count++; + } + } + // 4. Final condition based on internal variable(s) + if (y_count > 15) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_02_char_count_gt_15.json b/benchmarks/iid_testing/gen_case_02_char_count_gt_15.json new file mode 100644 index 00000000..7d7904ba --- /dev/null +++ b/benchmarks/iid_testing/gen_case_02_char_count_gt_15.json @@ -0,0 +1,53 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 4683, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_04_char_balance_AB_eq_13.c b/benchmarks/iid_testing/gen_case_04_char_balance_AB_eq_13.c new file mode 100644 index 00000000..ba943632 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_04_char_balance_AB_eq_13.c @@ -0,0 +1,43 @@ +/* File: gen_case_04_char_balance_AB_eq_7.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + int balance = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; // Invalid size + } + + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for ( short i = 0; i < size; ++i ) { + if ( data[ i ] == 'A' ) { + balance++; + } else if ( data[ i ] == 'B' ) { + balance--; + } + } + // 4. Final condition based on internal variable(s) + if ( balance == 13 ) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_04_char_balance_AB_eq_13.json b/benchmarks/iid_testing/gen_case_04_char_balance_AB_eq_13.json new file mode 100644 index 00000000..3dff5e7a --- /dev/null +++ b/benchmarks/iid_testing/gen_case_04_char_balance_AB_eq_13.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 4428, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_05_char_range_58_eq_3.c b/benchmarks/iid_testing/gen_case_05_char_range_58_eq_3.c new file mode 100644 index 00000000..2543d78a --- /dev/null +++ b/benchmarks/iid_testing/gen_case_05_char_range_58_eq_3.c @@ -0,0 +1,41 @@ +/* File: gen_case_05_char_range_58_eq_3.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 80 + +int main() +{ + char data[MAX_SIZE]; + short size; + int special_count = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] > '5' && data[i] < '9') { // Check if digit is 6, 7, or 8 + special_count++; // 6. Static change (+1) + } + } + // 4. Final condition based on internal variable(s) + if (special_count == 3) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_05_char_range_58_eq_3.json b/benchmarks/iid_testing/gen_case_05_char_range_58_eq_3.json new file mode 100644 index 00000000..e62ae8c8 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_05_char_range_58_eq_3.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 2506, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_07_char_continue_S_count_F_ge_10.c b/benchmarks/iid_testing/gen_case_07_char_continue_S_count_F_ge_10.c new file mode 100644 index 00000000..9a17b34f --- /dev/null +++ b/benchmarks/iid_testing/gen_case_07_char_continue_S_count_F_ge_10.c @@ -0,0 +1,44 @@ +/* File: gen_case_07_char_continue_S_count_F_ge_10.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 90 + +int main() +{ + char data[MAX_SIZE]; + short size; + int f_count = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'S') { + continue; // 3. Control flow modification + } + if (data[i] == 'F') { + f_count++; // 6. Static change + } + } + // 4. Final condition based on internal variable(s) + if (f_count >= 10) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_07_char_continue_S_count_F_ge_10.json b/benchmarks/iid_testing/gen_case_07_char_continue_S_count_F_ge_10.json new file mode 100644 index 00000000..4ecf2578 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_07_char_continue_S_count_F_ge_10.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 4222, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_08_char_break_Q_count_P_eq_12.c b/benchmarks/iid_testing/gen_case_08_char_break_Q_count_P_eq_12.c new file mode 100644 index 00000000..3462fd67 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_08_char_break_Q_count_P_eq_12.c @@ -0,0 +1,44 @@ +/* File: gen_case_08_char_break_Q_count_P_eq_12.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data[MAX_SIZE]; + short size; + int p_count = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'Q') { + break; // 3. Control flow modification + } + if (data[i] == 'P') { + p_count++; + } + } + // 4. Final condition based on internal variable(s) + if (p_count == 12) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_08_char_break_Q_count_P_eq_12.json b/benchmarks/iid_testing/gen_case_08_char_break_Q_count_P_eq_12.json new file mode 100644 index 00000000..d926905e --- /dev/null +++ b/benchmarks/iid_testing/gen_case_08_char_break_Q_count_P_eq_12.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 3736, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_12_char_multi_state_A_gt_B_plus_5.c b/benchmarks/iid_testing/gen_case_12_char_multi_state_A_gt_B_plus_5.c new file mode 100644 index 00000000..919f1aba --- /dev/null +++ b/benchmarks/iid_testing/gen_case_12_char_multi_state_A_gt_B_plus_5.c @@ -0,0 +1,44 @@ +/* File: gen_case_12_char_multi_state_A_gt_B_plus_5.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 100 + +int main() +{ + char data[MAX_SIZE]; + short size; + int countA = 0; + int countB = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'A') { + countA++; + } else if (data[i] == 'B') { + countB++; + } + } + // 4. Final condition based on internal variable(s) + if (countA > countB + 5) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_12_char_multi_state_A_gt_B_plus_5.json b/benchmarks/iid_testing/gen_case_12_char_multi_state_A_gt_B_plus_5.json new file mode 100644 index 00000000..f1f5c8cd --- /dev/null +++ b/benchmarks/iid_testing/gen_case_12_char_multi_state_A_gt_B_plus_5.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 2350, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_13_char_multi_state_XYZ_combo_eq_10.c b/benchmarks/iid_testing/gen_case_13_char_multi_state_XYZ_combo_eq_10.c new file mode 100644 index 00000000..4ca83f09 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_13_char_multi_state_XYZ_combo_eq_10.c @@ -0,0 +1,47 @@ +/* File: gen_case_13_char_multi_state_XYZ_combo_eq_10.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data[MAX_SIZE]; + short size; + int countX = 0; + int countY = 0; + int countZ = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'X') { + countX++; + } else if (data[i] == 'Y') { + countY++; + } else if (data[i] == 'Z') { + countZ++; + } + } + // 4. Final condition based on internal variable(s) + if ((countX + countY - countZ) == 10) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_13_char_multi_state_XYZ_combo_eq_10.json b/benchmarks/iid_testing/gen_case_13_char_multi_state_XYZ_combo_eq_10.json new file mode 100644 index 00000000..60854e50 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_13_char_multi_state_XYZ_combo_eq_10.json @@ -0,0 +1,57 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 3452, + "num_covered_branchings": 8, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_14_char_multi_state_firstF_lastL_dist_gt_10.c b/benchmarks/iid_testing/gen_case_14_char_multi_state_firstF_lastL_dist_gt_10.c new file mode 100644 index 00000000..a8e07010 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_14_char_multi_state_firstF_lastL_dist_gt_10.c @@ -0,0 +1,46 @@ +/* File: gen_case_14_char_multi_state_firstF_lastL_dist_gt_10.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 100 + +int main() +{ + char data[MAX_SIZE]; + short size; + short first_F_index = -1; + short last_L_index = -1; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'F') { + if (first_F_index == -1) { // Assign only if not already found + first_F_index = i; // Static change (assignment) + } + } else if (data[i] == 'L') { + last_L_index = i; // Update last known L index (Static change - assignment) + } + } + // 4. Final condition based on internal variable(s) + if (first_F_index != -1 && last_L_index != -1 && last_L_index > first_F_index + 10) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_14_char_multi_state_firstF_lastL_dist_gt_10.json b/benchmarks/iid_testing/gen_case_14_char_multi_state_firstF_lastL_dist_gt_10.json new file mode 100644 index 00000000..5e9a4a9e --- /dev/null +++ b/benchmarks/iid_testing/gen_case_14_char_multi_state_firstF_lastL_dist_gt_10.json @@ -0,0 +1,61 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 5897, + "num_covered_branchings": 10, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_15_char_two_loops_A_or_B_eq_10.c b/benchmarks/iid_testing/gen_case_15_char_two_loops_A_or_B_eq_10.c new file mode 100644 index 00000000..e0b73938 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_15_char_two_loops_A_or_B_eq_10.c @@ -0,0 +1,49 @@ +/* File: gen_case_15_char_two_loops_A_or_B_eq_10.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 100 + +int main() +{ + char data[MAX_SIZE]; + short size; + int countA = 0; + int countB = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + // Loop 1 + for (short i = 0; i < size; ++i) { + if (data[i] == 'A') { + countA++; + } + } + // Loop 2 + for (short i = 0; i < size; ++i) { + if (data[i] == 'B') { + countB++; + } + } + // 4. Final condition based on internal variable(s) + if (countA == 10 || countB == 10) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_15_char_two_loops_A_or_B_eq_10.json b/benchmarks/iid_testing/gen_case_15_char_two_loops_A_or_B_eq_10.json new file mode 100644 index 00000000..e9536b8a --- /dev/null +++ b/benchmarks/iid_testing/gen_case_15_char_two_loops_A_or_B_eq_10.json @@ -0,0 +1,59 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 6061, + "num_covered_branchings": 9, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_16_char_two_loops_dependent_ABCD_eq_7.c b/benchmarks/iid_testing/gen_case_16_char_two_loops_dependent_ABCD_eq_7.c new file mode 100644 index 00000000..e28424fb --- /dev/null +++ b/benchmarks/iid_testing/gen_case_16_char_two_loops_dependent_ABCD_eq_7.c @@ -0,0 +1,46 @@ +/* File: gen_case_16_char_two_loops_dependent_ABCD_eq_7.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data[MAX_SIZE]; + short size; + int k = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + // Loop 1 + for (short i = 0; i < size; ++i) { + if (data[i] == 'A') ++k; + if (data[i] == 'B') --k; + } + // Loop 2 (depends on k from loop 1) + for (short i = 0; i < size; ++i) { + if (data[i] == 'C') ++k; + if (data[i] == 'D') --k; + } + // 4. Final condition based on internal variable(s) + if (k == 7) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_16_char_two_loops_dependent_ABCD_eq_7.json b/benchmarks/iid_testing/gen_case_16_char_two_loops_dependent_ABCD_eq_7.json new file mode 100644 index 00000000..87574f5c --- /dev/null +++ b/benchmarks/iid_testing/gen_case_16_char_two_loops_dependent_ABCD_eq_7.json @@ -0,0 +1,61 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 2045, + "num_covered_branchings": 10, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_18_int_count_pos_gt_20.c b/benchmarks/iid_testing/gen_case_18_int_count_pos_gt_20.c new file mode 100644 index 00000000..3d172d6d --- /dev/null +++ b/benchmarks/iid_testing/gen_case_18_int_count_pos_gt_20.c @@ -0,0 +1,41 @@ +/* File: gen_case_18_int_count_pos_gt_20.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + int data[MAX_SIZE]; + short size; + int pos_count = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_int(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] > 0) { + pos_count++; + } + } + // 4. Final condition based on internal variable(s) + if (pos_count > 20) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_18_int_count_pos_gt_20.json b/benchmarks/iid_testing/gen_case_18_int_count_pos_gt_20.json new file mode 100644 index 00000000..33664200 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_18_int_count_pos_gt_20.json @@ -0,0 +1,48 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 25029, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_24_struct_item_status_A_ge_10.c b/benchmarks/iid_testing/gen_case_24_struct_item_status_A_ge_10.c new file mode 100644 index 00000000..52f69c7a --- /dev/null +++ b/benchmarks/iid_testing/gen_case_24_struct_item_status_A_ge_10.c @@ -0,0 +1,49 @@ +/* File: gen_case_24_struct_item_status_A_ge_10.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +typedef struct { + int id; + char status; // e.g., 'A'ctive, 'I'nactive +} Item; + + +#define MAX_SIZE 50 + +int main() +{ + Item inventory[MAX_SIZE]; + short num_items; + int active_count = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + num_items = __VERIFIER_nondet_short(); + if (num_items <= 0 || num_items > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < num_items; ++i) { + inventory[i].id = __VERIFIER_nondet_int(); + inventory[i].status = __VERIFIER_nondet_char(); + + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < num_items; ++i) { + if (inventory[i].status == 'A') { + active_count++; // Constant change + } + } + // 4. Final condition based on internal variable(s) + if (active_count >= 10) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_24_struct_item_status_A_ge_10.json b/benchmarks/iid_testing/gen_case_24_struct_item_status_A_ge_10.json new file mode 100644 index 00000000..88653fd6 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_24_struct_item_status_A_ge_10.json @@ -0,0 +1,53 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 10219, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_26_struct_dataval_balance_PN_eq_20.c b/benchmarks/iid_testing/gen_case_26_struct_dataval_balance_PN_eq_20.c new file mode 100644 index 00000000..88d56f51 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_26_struct_dataval_balance_PN_eq_20.c @@ -0,0 +1,51 @@ +/* File: gen_case_26_struct_dataval_balance_PN_eq_20.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +typedef struct { + char code; // 'P', 'N' + int value; // irrelevant for this logic, just for struct complexity +} DataVal; + + +#define MAX_SIZE 50 + +int main() +{ + DataVal items[MAX_SIZE]; + short num_items; + int balance = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + num_items = __VERIFIER_nondet_short(); + if (num_items <= 0 || num_items > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < num_items; ++i) { + items[i].code = __VERIFIER_nondet_char(); + items[i].value = __VERIFIER_nondet_int(); + + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < num_items; ++i) { + if (items[i].code == 'P') { + balance++; + } else if (items[i].code == 'N') { + balance--; + } + } + // 4. Final condition based on internal variable(s) + if (balance == 20) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_26_struct_dataval_balance_PN_eq_20.json b/benchmarks/iid_testing/gen_case_26_struct_dataval_balance_PN_eq_20.json new file mode 100644 index 00000000..2ff27bba --- /dev/null +++ b/benchmarks/iid_testing/gen_case_26_struct_dataval_balance_PN_eq_20.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 24945, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_28_char_count_X_nested_check_eq_8_11.c b/benchmarks/iid_testing/gen_case_28_char_count_X_nested_check_eq_8_11.c new file mode 100644 index 00000000..c38370ca --- /dev/null +++ b/benchmarks/iid_testing/gen_case_28_char_count_X_nested_check_eq_8_11.c @@ -0,0 +1,41 @@ +/* File: gen_case_28_char_count_X_nested_check_eq_8_11.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data[MAX_SIZE]; + short size; + int k = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'X') { + k++; + } + } + // 4. Final condition based on internal variable(s) + if (k == 8 || k == 11) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_28_char_count_X_nested_check_eq_8_11.json b/benchmarks/iid_testing/gen_case_28_char_count_X_nested_check_eq_8_11.json new file mode 100644 index 00000000..059a8b62 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_28_char_count_X_nested_check_eq_8_11.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 3131, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_29_char_count_A_deep_nested_check.c b/benchmarks/iid_testing/gen_case_29_char_count_A_deep_nested_check.c new file mode 100644 index 00000000..3666d1b5 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_29_char_count_A_deep_nested_check.c @@ -0,0 +1,42 @@ +/* File: gen_case_29_char_count_A_deep_nested_check.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data[MAX_SIZE]; + short size; + int k = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + int i = 0; + while(i < size) { + if (data[i] == 'A') k++; + i++; + } + + // 4. Final condition based on internal variable(s) + if (k >= 6 && k > 8 && k == 11) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_29_char_count_A_deep_nested_check.json b/benchmarks/iid_testing/gen_case_29_char_count_A_deep_nested_check.json new file mode 100644 index 00000000..8e542881 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_29_char_count_A_deep_nested_check.json @@ -0,0 +1,57 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 7360, + "num_covered_branchings": 8, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 4, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_30_char_count_A_mult_check_eq_30.c b/benchmarks/iid_testing/gen_case_30_char_count_A_mult_check_eq_30.c new file mode 100644 index 00000000..fad86294 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_30_char_count_A_mult_check_eq_30.c @@ -0,0 +1,39 @@ +/* File: gen_case_30_char_count_A_mult_check_eq_30.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data[MAX_SIZE]; + short size; + int k = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'A') k++; + } + // 4. Final condition based on internal variable(s) + if (3 * k == 30) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_30_char_count_A_mult_check_eq_30.json b/benchmarks/iid_testing/gen_case_30_char_count_A_mult_check_eq_30.json new file mode 100644 index 00000000..b87b939f --- /dev/null +++ b/benchmarks/iid_testing/gen_case_30_char_count_A_mult_check_eq_30.json @@ -0,0 +1,53 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 4234, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_31_char_state_dependent_inc_gt_50.c b/benchmarks/iid_testing/gen_case_31_char_state_dependent_inc_gt_50.c new file mode 100644 index 00000000..3f0c10c4 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_31_char_state_dependent_inc_gt_50.c @@ -0,0 +1,45 @@ +/* File: gen_case_31_char_state_dependent_inc_gt_50.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 80 + +int main() +{ + char data[MAX_SIZE]; + short size; + int counter = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'X') { + if (counter % 2 == 0) { + counter += 1; // Static change +1 + } else { + counter += 2; // Static change +2 + } + } + } + // 4. Final condition based on internal variable(s) + if (counter > 50) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_31_char_state_dependent_inc_gt_50.json b/benchmarks/iid_testing/gen_case_31_char_state_dependent_inc_gt_50.json new file mode 100644 index 00000000..ad372155 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_31_char_state_dependent_inc_gt_50.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 14605, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_32_two_char_arrays_match_ST_gt_5.c b/benchmarks/iid_testing/gen_case_32_two_char_arrays_match_ST_gt_5.c new file mode 100644 index 00000000..ad5e13b6 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_32_two_char_arrays_match_ST_gt_5.c @@ -0,0 +1,41 @@ +/* File: gen_case_32_two_char_arrays_match_ST_gt_5.c */ +#include + +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data1[MAX_SIZE]; + char data2[MAX_SIZE]; // Second array + short size; + int match_count = 0; // State variable + + // 1. Fill data structures in loops + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; + } + for (short i = 0; i < size; ++i) { + data1[i] = __VERIFIER_nondet_char(); + data2[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop iterating through data + for (short i = 0; i < size; ++i) { + // 3. Modify internal state based on data from both arrays + if (data1[i] == 'S' && data2[i] == 'T') { + match_count++; // 6. Static change (+1) + } + } + + // 4. Final condition based on internal variable + if (match_count > 5) { // 5. Linear check + return 1; + } else { + return 0; + } +} diff --git a/benchmarks/iid_testing/gen_case_32_two_char_arrays_match_ST_gt_5.json b/benchmarks/iid_testing/gen_case_32_two_char_arrays_match_ST_gt_5.json new file mode 100644 index 00000000..7520dcca --- /dev/null +++ b/benchmarks/iid_testing/gen_case_32_two_char_arrays_match_ST_gt_5.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 7020, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_33_char_state_machine_XYZ_count2_gt_3.c b/benchmarks/iid_testing/gen_case_33_char_state_machine_XYZ_count2_gt_3.c new file mode 100644 index 00000000..8a2235c4 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_33_char_state_machine_XYZ_count2_gt_3.c @@ -0,0 +1,47 @@ +/* File: gen_case_33_char_state_machine_XYZ_count2_gt_3.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 150 + +int main() +{ + char data[MAX_SIZE]; + short size; + int state = 0; + int state2_entries = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (state == 0 && data[i] == 'X') { + state = 1; + } else if (state == 1 && data[i] == 'Y') { + state = 2; + state2_entries++; // Static change +1 + } else if (data[i] == 'R') { // Reset condition + state = 0; + } + } + // 4. Final condition based on internal variable(s) + if (state2_entries > 3) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_33_char_state_machine_XYZ_count2_gt_3.json b/benchmarks/iid_testing/gen_case_33_char_state_machine_XYZ_count2_gt_3.json new file mode 100644 index 00000000..8b25e86e --- /dev/null +++ b/benchmarks/iid_testing/gen_case_33_char_state_machine_XYZ_count2_gt_3.json @@ -0,0 +1,61 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 6945, + "num_covered_branchings": 10, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_35_char_pointer_loop_count_P_eq_5.c b/benchmarks/iid_testing/gen_case_35_char_pointer_loop_count_P_eq_5.c new file mode 100644 index 00000000..d6c170c1 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_35_char_pointer_loop_count_P_eq_5.c @@ -0,0 +1,45 @@ +/* File: gen_case_35_char_pointer_loop_count_P_eq_5.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 60 + +int main() +{ + char data[MAX_SIZE]; + short size; + int p_count = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + char* ptr = data; + char* end_ptr = data + size; // Pointer to one past the last element + + while (ptr < end_ptr) { + if (*ptr == 'P') { + p_count++; // Static change + } + ptr++; // Move pointer to the next element + } + // 4. Final condition based on internal variable(s) + if (p_count == 5) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_35_char_pointer_loop_count_P_eq_5.json b/benchmarks/iid_testing/gen_case_35_char_pointer_loop_count_P_eq_5.json new file mode 100644 index 00000000..b7ce8254 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_35_char_pointer_loop_count_P_eq_5.json @@ -0,0 +1,53 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 1426, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_39_char_input_cycle_nested_A_eq_10.c b/benchmarks/iid_testing/gen_case_39_char_input_cycle_nested_A_eq_10.c new file mode 100644 index 00000000..8f356607 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_39_char_input_cycle_nested_A_eq_10.c @@ -0,0 +1,54 @@ +/* File: gen_case_39_char_input_cycle_nested_A_eq_10.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data[MAX_SIZE]; + short size; + int k = 0; + short loop_count_1 = 0; + short loop_count_2 = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + loop_count_1 = __VERIFIER_nondet_short(); + loop_count_2 = __VERIFIER_nondet_short(); + + if (loop_count_1 < 0 || loop_count_1 > 10) loop_count_1 = 3; // Bound loops + if (loop_count_2 < 0 || loop_count_2 > 10) loop_count_2 = 4; // Bound loops + + for (short index_1 = 0; index_1 < loop_count_1; ++index_1) { + for (short index_2 = 0; index_2 < loop_count_2; ++index_2) { + int i = 0; + while (i < size) { // Assuming non-null terminated for simplicity here + if (data[i] == 'A') ++k; + i++; + } + } + } + + // 4. Final condition based on internal variable(s) + if (k == 10) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_39_char_input_cycle_nested_A_eq_10.json b/benchmarks/iid_testing/gen_case_39_char_input_cycle_nested_A_eq_10.json new file mode 100644 index 00000000..f7c89900 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_39_char_input_cycle_nested_A_eq_10.json @@ -0,0 +1,65 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 28806, + "num_covered_branchings": 12, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0, + 10, + 0, + 11, + 0, + 12, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 8, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_40_char_count_K_lt_10.c b/benchmarks/iid_testing/gen_case_40_char_count_K_lt_10.c new file mode 100644 index 00000000..7ed18e53 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_40_char_count_K_lt_10.c @@ -0,0 +1,41 @@ +/* File: gen_case_40_char_count_K_lt_10.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 100 + +int main() +{ + char data[MAX_SIZE]; + short size; + int count = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'K') { + count++; + } + } + // 4. Final condition based on internal variable(s) + if (count < 10) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_40_char_count_K_lt_10.json b/benchmarks/iid_testing/gen_case_40_char_count_K_lt_10.json new file mode 100644 index 00000000..b4841efc --- /dev/null +++ b/benchmarks/iid_testing/gen_case_40_char_count_K_lt_10.json @@ -0,0 +1,53 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 4216, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_42_char_balance_U2_R1_gt_10.c b/benchmarks/iid_testing/gen_case_42_char_balance_U2_R1_gt_10.c new file mode 100644 index 00000000..74153b12 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_42_char_balance_U2_R1_gt_10.c @@ -0,0 +1,43 @@ +/* File: gen_case_42_char_balance_U2_R1_gt_10.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 100 + +int main() +{ + char data[MAX_SIZE]; + short size; + int balance = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'U') { + balance += 2; + } else if (data[i] == 'R') { + balance -= 1; + } + } + // 4. Final condition based on internal variable(s) + if (balance > 10) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_42_char_balance_U2_R1_gt_10.json b/benchmarks/iid_testing/gen_case_42_char_balance_U2_R1_gt_10.json new file mode 100644 index 00000000..f1f5c8cd --- /dev/null +++ b/benchmarks/iid_testing/gen_case_42_char_balance_U2_R1_gt_10.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 2350, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_44_char_count_H_eq_12.c b/benchmarks/iid_testing/gen_case_44_char_count_H_eq_12.c new file mode 100644 index 00000000..6874fbba --- /dev/null +++ b/benchmarks/iid_testing/gen_case_44_char_count_H_eq_12.c @@ -0,0 +1,41 @@ +/* File: gen_case_44_char_count_H_eq_12.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data[MAX_SIZE]; + short size; + int count = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'H') { + count++; + } + } + // 4. Final condition based on internal variable(s) + if (count == 12) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_44_char_count_H_eq_12.json b/benchmarks/iid_testing/gen_case_44_char_count_H_eq_12.json new file mode 100644 index 00000000..2ab8abc5 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_44_char_count_H_eq_12.json @@ -0,0 +1,53 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 3981, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_49_char_balance_Z1_A1_gt_15.c b/benchmarks/iid_testing/gen_case_49_char_balance_Z1_A1_gt_15.c new file mode 100644 index 00000000..13232609 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_49_char_balance_Z1_A1_gt_15.c @@ -0,0 +1,43 @@ +/* File: gen_case_49_char_balance_Z1_A1_gt_15.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 80 + +int main() +{ + char data[MAX_SIZE]; + short size; + int balance = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'Z') { + balance += 1; + } else if (data[i] == 'A') { + balance -= 1; + } + } + // 4. Final condition based on internal variable(s) + if (balance > 15) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_49_char_balance_Z1_A1_gt_15.json b/benchmarks/iid_testing/gen_case_49_char_balance_Z1_A1_gt_15.json new file mode 100644 index 00000000..811a6fad --- /dev/null +++ b/benchmarks/iid_testing/gen_case_49_char_balance_Z1_A1_gt_15.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 4604, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_50_char_count_R_gt_12.c b/benchmarks/iid_testing/gen_case_50_char_count_R_gt_12.c new file mode 100644 index 00000000..516a913c --- /dev/null +++ b/benchmarks/iid_testing/gen_case_50_char_count_R_gt_12.c @@ -0,0 +1,41 @@ +/* File: gen_case_50_char_count_R_gt_12.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data[MAX_SIZE]; + short size; + int count = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'R') { + count++; + } + } + // 4. Final condition based on internal variable(s) + if (count > 12) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_50_char_count_R_gt_12.json b/benchmarks/iid_testing/gen_case_50_char_count_R_gt_12.json new file mode 100644 index 00000000..1e905873 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_50_char_count_R_gt_12.json @@ -0,0 +1,53 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 4697, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_51_char_multi_state_R_gt_P_plus_2.c b/benchmarks/iid_testing/gen_case_51_char_multi_state_R_gt_P_plus_2.c new file mode 100644 index 00000000..08218c7a --- /dev/null +++ b/benchmarks/iid_testing/gen_case_51_char_multi_state_R_gt_P_plus_2.c @@ -0,0 +1,44 @@ +/* File: gen_case_51_char_multi_state_R_gt_P_plus_2.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 80 + +int main() +{ + char data[MAX_SIZE]; + short size; + int countR = 0; + int countP = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'R') { + countR++; + } else if (data[i] == 'P') { + countP++; + } + } + // 4. Final condition based on internal variable(s) + if (countR > countP + 2) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_51_char_multi_state_R_gt_P_plus_2.json b/benchmarks/iid_testing/gen_case_51_char_multi_state_R_gt_P_plus_2.json new file mode 100644 index 00000000..57e7aa4e --- /dev/null +++ b/benchmarks/iid_testing/gen_case_51_char_multi_state_R_gt_P_plus_2.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 1468, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_53_char_two_loops_dep_PQ_ge_7.c b/benchmarks/iid_testing/gen_case_53_char_two_loops_dep_PQ_ge_7.c new file mode 100644 index 00000000..5378e594 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_53_char_two_loops_dep_PQ_ge_7.c @@ -0,0 +1,44 @@ +/* File: gen_case_53_char_two_loops_dep_PQ_ge_7.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 50 + +int main() +{ + char data[MAX_SIZE]; + short size; + int k = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + // Loop 1 + for (short i = 0; i < size; ++i) { + if (data[i] == 'P') ++k; + } + // Loop 2 + for (short i = 0; i < size; ++i) { + if (data[i] == 'Q') --k; + } + // 4. Final condition based on internal variable(s) + if (k >= 7) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_53_char_two_loops_dep_PQ_ge_7.json b/benchmarks/iid_testing/gen_case_53_char_two_loops_dep_PQ_ge_7.json new file mode 100644 index 00000000..56e8af09 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_53_char_two_loops_dep_PQ_ge_7.json @@ -0,0 +1,57 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 1975, + "num_covered_branchings": 8, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_54_char_count_S_ge_5.c b/benchmarks/iid_testing/gen_case_54_char_count_S_ge_5.c new file mode 100644 index 00000000..4dbc52fe --- /dev/null +++ b/benchmarks/iid_testing/gen_case_54_char_count_S_ge_5.c @@ -0,0 +1,41 @@ +/* File: gen_case_54_char_count_S_ge_5.c */ +#include +#include // For NULL if needed + +// Assume these functions are provided externally +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 80 + +int main() +{ + char data[MAX_SIZE]; + short size; + int count = 0; // State variable(s) + + // 1. Fill data structure in an initial loop + size = __VERIFIER_nondet_short(); + if (size <= 0 || size > MAX_SIZE) { + return -1; // Invalid size + } + + for (short i = 0; i < size; ++i) { + data[i] = __VERIFIER_nondet_char(); + } + + // 2. Core logic in loop(s) + + for (short i = 0; i < size; ++i) { + if (data[i] == 'S') { + count++; + } + } + // 4. Final condition based on internal variable(s) + if (count >= 5) { + return 1; // Condition met + } else { + return 0; // Condition not met + } +} diff --git a/benchmarks/iid_testing/gen_case_54_char_count_S_ge_5.json b/benchmarks/iid_testing/gen_case_54_char_count_S_ge_5.json new file mode 100644 index 00000000..65a0e189 --- /dev/null +++ b/benchmarks/iid_testing/gen_case_54_char_count_S_ge_5.json @@ -0,0 +1,53 @@ +{ + "args": { + "max_executions": 1000000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 2030, + "num_covered_branchings": 6, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file From 3574bc705f88168b28963efae7acba99fa92867e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 10 May 2025 17:16:40 +0200 Subject: [PATCH 138/144] feat: remove bad benchmarks --- benchmarks/iid_testing/ai_generated_19.c | 46 ------------- benchmarks/iid_testing/ai_generated_19.json | 67 ------------------- ...char_multi_state_firstF_lastL_dist_gt_10.c | 46 ------------- ...r_multi_state_firstF_lastL_dist_gt_10.json | 61 ----------------- 4 files changed, 220 deletions(-) delete mode 100644 benchmarks/iid_testing/ai_generated_19.c delete mode 100644 benchmarks/iid_testing/ai_generated_19.json delete mode 100644 benchmarks/iid_testing/gen_case_14_char_multi_state_firstF_lastL_dist_gt_10.c delete mode 100644 benchmarks/iid_testing/gen_case_14_char_multi_state_firstF_lastL_dist_gt_10.json diff --git a/benchmarks/iid_testing/ai_generated_19.c b/benchmarks/iid_testing/ai_generated_19.c deleted file mode 100644 index f3e7aba6..00000000 --- a/benchmarks/iid_testing/ai_generated_19.c +++ /dev/null @@ -1,46 +0,0 @@ -#include - -// Assume these functions are provided externally -extern short __VERIFIER_nondet_short(); -extern char __VERIFIER_nondet_char(); - -#define MAX_SIZE 100 - -int main() -{ - char data[ MAX_SIZE ]; - short size; - short first_F_index = -1; // State variable 1 - short last_L_index = -1; // State variable 2 - - // 1. Fill data structure in a loop - size = __VERIFIER_nondet_short(); - if ( size <= 0 || size > MAX_SIZE ) { - return -1; - } - for ( short i = 0; i < size; ++i ) { - data[ i ] = __VERIFIER_nondet_char(); - } - - // 2. Core logic in loop - for ( short i = 0; i < size; ++i ) { - // 3. Modify internal state variables based on data/condition - // 6. Change is static (assignment of index, happens at most once for first_F) - if ( data[ i ] == 'F' ) { - if ( first_F_index == -1 ) { // Assign only if not already found - first_F_index = i; // Static change (assignment) - } - } else if ( data[ i ] == 'L' ) { - last_L_index = i; // Update last known L index (Static change - assignment) - } - } - - // 4. Final condition based on multiple internal variables - // 5. Condition check is linear (last_L_index - first_F_index > 10) - // Check if both were found and the distance is large enough - if ( first_F_index != -1 && last_L_index != -1 && last_L_index > first_F_index + 10 ) { - return 1; - } else { - return 0; - } -} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_19.json b/benchmarks/iid_testing/ai_generated_19.json deleted file mode 100644 index a4dcb4e7..00000000 --- a/benchmarks/iid_testing/ai_generated_19.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "args": { - "max_executions": 100000, - "max_seconds": 300, - "max_trace_length": 10000, - "max_stack_size": 25, - "max_stdin_bytes": 65536, - "max_exec_milliseconds": 250, - "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_zero", - "stdout_model": "stdout_void", - "optimizer_max_seconds": 10, - "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000, - "m32": false - }, - "results": { - "termination_type": "NORMAL", - "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 5897, - "num_covered_branchings": 10, - "covered_branchings": [ - 1, - 0, - 2, - 0, - 3, - 0, - 4, - 0, - 5, - 0, - 6, - 0, - 7, - 0, - 8, - 0, - 9, - 0, - 10, - 0 - ], - "output_statistics": { - "bitshare_analysis": { - "num_generated_tests": 3, - "num_crashes": 0, - "num_boundary_violations": 0 - }, - "sensitivity_analysis": { - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 - }, - "typed_minimization_analysis": { - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 - }, - "STARTUP": { - "num_generated_tests": 1, - "num_crashes": 0, - "num_boundary_violations": 0 - } - } - } -} \ No newline at end of file diff --git a/benchmarks/iid_testing/gen_case_14_char_multi_state_firstF_lastL_dist_gt_10.c b/benchmarks/iid_testing/gen_case_14_char_multi_state_firstF_lastL_dist_gt_10.c deleted file mode 100644 index a8e07010..00000000 --- a/benchmarks/iid_testing/gen_case_14_char_multi_state_firstF_lastL_dist_gt_10.c +++ /dev/null @@ -1,46 +0,0 @@ -/* File: gen_case_14_char_multi_state_firstF_lastL_dist_gt_10.c */ -#include -#include // For NULL if needed - -// Assume these functions are provided externally -extern short __VERIFIER_nondet_short(); -extern char __VERIFIER_nondet_char(); -extern int __VERIFIER_nondet_int(); - -#define MAX_SIZE 100 - -int main() -{ - char data[MAX_SIZE]; - short size; - short first_F_index = -1; - short last_L_index = -1; // State variable(s) - - // 1. Fill data structure in an initial loop - size = __VERIFIER_nondet_short(); - if (size <= 0 || size > MAX_SIZE) { - return -1; // Invalid size - } - - for (short i = 0; i < size; ++i) { - data[i] = __VERIFIER_nondet_char(); - } - - // 2. Core logic in loop(s) - - for (short i = 0; i < size; ++i) { - if (data[i] == 'F') { - if (first_F_index == -1) { // Assign only if not already found - first_F_index = i; // Static change (assignment) - } - } else if (data[i] == 'L') { - last_L_index = i; // Update last known L index (Static change - assignment) - } - } - // 4. Final condition based on internal variable(s) - if (first_F_index != -1 && last_L_index != -1 && last_L_index > first_F_index + 10) { - return 1; // Condition met - } else { - return 0; // Condition not met - } -} diff --git a/benchmarks/iid_testing/gen_case_14_char_multi_state_firstF_lastL_dist_gt_10.json b/benchmarks/iid_testing/gen_case_14_char_multi_state_firstF_lastL_dist_gt_10.json deleted file mode 100644 index 5e9a4a9e..00000000 --- a/benchmarks/iid_testing/gen_case_14_char_multi_state_firstF_lastL_dist_gt_10.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "args": { - "max_executions": 1000000, - "max_seconds": 300, - "max_trace_length": 10000, - "max_stack_size": 25, - "max_stdin_bytes": 65536, - "max_exec_milliseconds": 250, - "max_exec_megabytes": 1024, - "stdin_model": "stdin_replay_bytes_then_repeat_zero", - "stdout_model": "stdout_void", - "optimizer_max_seconds": 10, - "optimizer_max_trace_length": 1000000, - "optimizer_max_stdin_bytes": 1000000 - }, - "results": { - "termination_type": "NORMAL", - "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 5897, - "num_covered_branchings": 10, - "covered_branchings": [ - 1, - 0, - 2, - 0, - 3, - 0, - 4, - 0, - 5, - 0, - 6, - 0, - 7, - 0, - 8, - 0, - 9, - 0, - 10, - 0 - ], - "output_statistics": { - "sensitivity_analysis": { - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 - }, - "typed_minimization_analysis": { - "num_generated_tests": 2, - "num_crashes": 0, - "num_boundary_violations": 0 - }, - "STARTUP": { - "num_generated_tests": 1, - "num_crashes": 0, - "num_boundary_violations": 0 - } - } - } -} \ No newline at end of file From a1cadd1286ed261ab421061842b5364ad55ec8c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 10 May 2025 17:17:52 +0200 Subject: [PATCH 139/144] feat: select best configuration --- .../include/fuzzing/iid_vector_analysis.hpp | 26 ++++++++++--------- src/fuzzing/src/iid_vector_analysis.cpp | 21 ++++++++++----- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp index c5fc6a56..a1ad3b86 100644 --- a/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp +++ b/src/fuzzing/include/fuzzing/iid_vector_analysis.hpp @@ -465,21 +465,23 @@ struct iid_dependencies { public: inline static std::size_t biggest_node_id = 0; - // Configurations - inline static bool random_nested_loop_counts = false; - inline static bool random_direction_in_path = true; - inline static bool random_node_selection = true; - inline static bool generate_more_data_after_coverage = true; - inline static bool generate_for_bad_nodes = true; - inline static int max_failed_generations_in_row = 2; + // Set configurations inline static int minimal_max_generation_after_covered = 10; inline static int minimal_max_generation_for_other_node = 10; + inline static bool generate_for_bad_nodes = true; inline static int minimal_max_generation_artificial_data = 5; - inline static int maximal_number_of_equations_with_same_branching_value = 250; - inline static int maximal_number_of_branching_values = 250; - inline static float percentage_to_add_to_path = 0.4; - inline static bool create_artificial_data = true; - inline static int biggest_value_in_difference_vector = 10; + + // Configurations + inline static int biggest_value_in_difference_vector = 4; + inline static bool generate_more_data_after_coverage = true; + inline static int max_failed_generations_in_row = 3; + inline static int maximal_number_of_branching_values = 30; + inline static int maximal_number_of_equations_in_matrix = 500; + inline static int maximal_number_of_equations_with_same_branching_value = 100; + inline static float percentage_to_add_to_path = 0.5; + inline static bool random_direction_in_path = false; + inline static bool random_nested_loop_counts = false; + inline static bool random_node_selection = true; inline static bool verbose = false; // inline static bool verbose = true; diff --git a/src/fuzzing/src/iid_vector_analysis.cpp b/src/fuzzing/src/iid_vector_analysis.cpp index d6bae5db..a565cf7e 100644 --- a/src/fuzzing/src/iid_vector_analysis.cpp +++ b/src/fuzzing/src/iid_vector_analysis.cpp @@ -697,10 +697,13 @@ void fuzzing::equation_matrix::process_node( branching_node* end_node, { TMPROF_BLOCK(); + if ( unique_rows.size() >= iid_dependencies::maximal_number_of_equations_in_matrix ) { + return; + } + auto it = branching_values.find( end_node->best_coverage_value ); - if ( it != branching_values.end() && - branching_values.size() >= iid_dependencies::maximal_number_of_branching_values ) { + if ( branching_values.size() >= iid_dependencies::maximal_number_of_branching_values && it == branching_values.end() ) { return; } @@ -724,6 +727,10 @@ void fuzzing::equation_matrix::start_compute_matrix() TMPROF_BLOCK(); for ( branching_node* node : all_paths ) { + if ( unique_rows.size() >= iid_dependencies::maximal_number_of_equations_in_matrix ) { + return; + } + auto [ directions_in_path, max_directions_in_path_index ] = get_directions_in_path( node ); add_path( node, directions_in_path, true, max_directions_in_path_index ); } @@ -1444,14 +1451,14 @@ int fuzzing::iid_node_dependence_props::compute_loading_loop_interation( { TMPROF_BLOCK(); - float average_bits = props.bits_read_by_node.at( id ).average_bits_read.mean; + float average_bits_read = props.bits_read_by_node.at( id ).average_bits_read.mean; natural_32_bit offset = props.bits_read_by_node.at( id ).minimal_bit_offset; - float average_used_bits = props.bits_read_by_node.at( id ).average_bits_used.mean; + float average_bits_used = props.bits_read_by_node.at( id ).average_bits_used.mean; - float loaded_per_loop = std::min( props.loaded_bits_per_loop.mean, average_used_bits ); + float loaded_per_loop = std::min( props.loaded_bits_per_loop.mean, average_bits_used ); loaded_per_loop = std::max( loaded_per_loop, 8.0f ); - if ( offset == natural_32_bit( props.loaded_bits_per_loop.mean - average_used_bits ) ) + if ( offset == natural_32_bit( props.loaded_bits_per_loop.mean - average_bits_used ) ) offset = 0; bool is_loop_head = true; @@ -1467,7 +1474,7 @@ int fuzzing::iid_node_dependence_props::compute_loading_loop_interation( } int total_count = is_loop_head ? path_counts[ id ].get_total_count() : path_counts[ id ].get_max_count(); - float bits_needed = average_bits * total_count + offset; + float bits_needed = average_bits_read * total_count + offset; return std::ceil( bits_needed / loaded_per_loop ); } From 9f469cf67c4cb2b18832cbede9ff918eb39f655c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 10 May 2025 17:25:39 +0200 Subject: [PATCH 140/144] feat: remove old files from development --- .../include/fuzzing/gradient_descent.hpp | 51 - .../gradient_descent_with_convergence.hpp | 68 - .../include/fuzzing/iid_node_dependencies.hpp | 229 ---- src/fuzzing/src/gradient_descent.cpp | 356 ------ .../src/gradient_descent_with_convergence.cpp | 256 ---- src/fuzzing/src/iid_node_dependencies.cpp | 1108 ----------------- 6 files changed, 2068 deletions(-) delete mode 100644 src/fuzzing/include/fuzzing/gradient_descent.hpp delete mode 100644 src/fuzzing/include/fuzzing/gradient_descent_with_convergence.hpp delete mode 100644 src/fuzzing/include/fuzzing/iid_node_dependencies.hpp delete mode 100644 src/fuzzing/src/gradient_descent.cpp delete mode 100644 src/fuzzing/src/gradient_descent_with_convergence.cpp delete mode 100644 src/fuzzing/src/iid_node_dependencies.cpp diff --git a/src/fuzzing/include/fuzzing/gradient_descent.hpp b/src/fuzzing/include/fuzzing/gradient_descent.hpp deleted file mode 100644 index 4fc38efd..00000000 --- a/src/fuzzing/include/fuzzing/gradient_descent.hpp +++ /dev/null @@ -1,51 +0,0 @@ -#pragma once - -#include - -/* - # Improvements - - Learning rate optimization(Adaptive learning rate) - - Feature scaling - - Momentum - - Mini-batch gradient descent -*/ - -class GradientDescent { -public: - GradientDescent( const std::vector< std::vector< float > >& coefficient_matrix, - const std::vector< float >& target_vector, - float learning_rate = 0.01f, - int max_iterations = 1000, - float convergence_threshold = 1e-4, - float momentum = 0.9f ); - - std::tuple< std::vector< float >, bool > optimize(); - - void set_learning_rate( float learning_rate ) { _learning_rate = learning_rate; } - void set_max_iterations( int max_iterations ) { _max_iterations = max_iterations; } - void set_convergence_threshold( float convergence_threshold ) - { - _convergence_threshold = convergence_threshold; - } - void set_momentum( float momentum ) { _momentum = momentum; } - -private: - std::vector< std::vector< float > > _coefficient_matrix; - std::vector< float > _target_vector; - float _learning_rate; - int _max_iterations; - float _convergence_threshold; - float _momentum; - bool _debug = false; - - std::vector< float > compute_gradient( const std::vector< float >& current_solution ); - float compute_mean_squared_error( const std::vector< float >& current_solution ); - static std::vector< float > generate_random_weights( size_t n ); - static float dot_product( const std::vector< float >& a, const std::vector< float >& b ); - static void rescale( std::vector< float >& values, float min_value, float max_value ); - static void add_smallest_value( std::vector< float >& values ); - void min_max_normalize( std::vector< std::vector< float > >& matrix ); - void min_max_normalize_target( std::vector< float >& target ); - - void shuffle_data(); -}; \ No newline at end of file diff --git a/src/fuzzing/include/fuzzing/gradient_descent_with_convergence.hpp b/src/fuzzing/include/fuzzing/gradient_descent_with_convergence.hpp deleted file mode 100644 index 9184a2d7..00000000 --- a/src/fuzzing/include/fuzzing/gradient_descent_with_convergence.hpp +++ /dev/null @@ -1,68 +0,0 @@ -#pragma once - -#include -#include -#include - -struct GradientDescentResult { - std::vector< float > weights; - std::vector< float > errors; - std::vector< float > column_count_weighted; - int iterations; - float error_mean; - float error_square_of_mean; - float error_mean_of_squares; - float error_variance; - float variance_threshold; - float count_threshold; - bool converged; - - bool operator==( const GradientDescentResult& other ) const - { - return weights == other.weights && errors == other.errors && - column_count_weighted == other.column_count_weighted && iterations == other.iterations && - error_mean == other.error_mean && error_square_of_mean == other.error_square_of_mean && - error_mean_of_squares == other.error_mean_of_squares && - error_variance == other.error_variance && variance_threshold == other.variance_threshold && - count_threshold == other.count_threshold && converged == other.converged; - } -}; - - -class GradientDescentNew { -public: - GradientDescentNew( const std::vector< std::vector< float > >& coefficient_matrix, - const std::vector< float >& target_vector, - std::map< size_t, float > locked_columns = {}, - float learning_rate = 0.01f, - int max_iterations = 500, - float convergence_threshold = 1e-4, - float momentum = 0.9f ); - - GradientDescentResult optimize(); - void print_input_matrix(); - -private: - std::vector< std::vector< float > > _coefficient_matrix; - std::vector< float > _target_vector; - std::vector< std::optional< float > > _locked_columns; - float _learning_rate; - int _max_iterations; - float _convergence_threshold; - float _momentum; - bool _debug = false; - - std::vector< float > compute_gradient( const std::vector< float >& current_solution ); - std::vector< float > compute_errors( const std::vector< float >& current_solution ); - float compute_mean_squared_error( const std::vector< float >& errors ); - static std::vector< float > generate_random_weights( size_t n ); - static float dot_product( const std::vector< float >& a, const std::vector< float >& b ); - static float compute_variance( const std::vector< float >& errors ); - static bool compute_convergence( const std::vector< float >& counts_per_column_weighted, - float error_variance, - float error_mean, - float& variance_threshold, - float& count_threshold ); - static float compute_mean( const std::vector< float >& errors ); - std::vector< float > compute_column_count( const std::vector< float >& current_solution ); -}; \ No newline at end of file diff --git a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp b/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp deleted file mode 100644 index 15386943..00000000 --- a/src/fuzzing/include/fuzzing/iid_node_dependencies.hpp +++ /dev/null @@ -1,229 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -using loop_to_bodies_t = std::unordered_map< location_id, std::unordered_set< location_id > >; -using loading_loops_t = std::unordered_map< location_id, std::tuple< natural_32_bit, natural_32_bit > >; - -struct DirectionVector { - DirectionVector( std::vector< float > vector, float value ) - : DirectionVector( std::move( vector ), value, 0 ) - {} - - DirectionVector( std::vector< float > vector, float value, int compare_hits ) - : vector( std::move( vector ) ) - , value( value ) - , compare_hits( compare_hits ) - {} - - std::vector< float > vector; - float value; - int compare_hits; - - bool operator<( const DirectionVector& other ) const - { - // if ( value != other.value ) { - // return value < other.value; - // } - - return get_vector_length() < other.get_vector_length(); - } - - bool operator==( const DirectionVector& other ) const - { - return std::tie( vector, value ) == std::tie( other.vector, other.value ); - } - - float get_vector_length() const - { - return std::sqrt( std::inner_product( vector.begin(), vector.end(), vector.begin(), 0.0f ) ); - } -}; - -struct TableRow { - TableRow( std::vector< std::optional< float > > weights, GradientDescentResult result ) - : weights( std::move( weights ) ) - , result( std::move( result ) ) - {} - - bool operator==( const TableRow& other ) const - { - return weights == other.weights && result == other.result; - } - - int get_non_null_count() const - { - return std::count_if( weights.begin(), weights.end(), []( const auto& w ) { return w.has_value(); } ); - } - - std::vector< std::optional< float > > weights; - GradientDescentResult result; -}; - -struct FloatComparator { - bool operator()( const float& a, const float& b ) const - { - const float epsilon = 1e-6f; - return std::abs( a - b ) > epsilon && std::abs( a ) < std::abs( b ); - } -}; - - -struct Mean_counter { - float value; - - Mean_counter() - : value( 0 ) - , count( 0 ) - {} - - void add( float new_value ) { value = value + ( new_value - value ) / ++count; } - - operator int() const { return static_cast< int >( value ); } - - friend std::ostream& operator<<( std::ostream& os, Mean_counter const& m ) { return os << m.value; } - -private: - size_t count; -}; - -namespace fuzzing -{ -struct path_decision { - int left_current; - int left_max; - - int right_current; - int right_max; - - path_decision( int left, int right ) - : left_current( 0 ) - , left_max( left ) - , right_current( 0 ) - , right_max( right ) - {} - - path_decision() - : path_decision( 0, 0 ) - {} - - friend std::ostream& operator<<( std::ostream& os, path_decision const& pd ) - { - return os << " left: " << pd.left_current << "/" << pd.left_max << " right: " << pd.right_current - << "/" << pd.right_max; - } - - bool get_next_direction(); -}; - - -struct node_direction { - location_id node_id; - bool branching_direction; - - auto operator<=>( node_direction const& other ) const; - bool operator==( node_direction const& other ) const; - friend std::ostream& operator<<( std::ostream& os, node_direction const& nn ) - { - return os << nn.node_id.id << " " << ( nn.branching_direction ? "right" : "left" ); - } -}; - -struct number_statistics { - int min; - int max; - Mean_counter mean; - - number_statistics() - : min( std::numeric_limits< int >::max() ) - , max( std::numeric_limits< int >::min() ) - {} - - friend std::ostream& operator<<( std::ostream& os, number_statistics const& ds ) - { - return os << "min: " << ds.min << " max: " << ds.max << " mean: " << ds.mean; - } - - void add( int value ); -}; - -struct coverage_value_props { - number_statistics path_depth; - std::map< node_direction, number_statistics > direction_statistics; - - void process_node( branching_node* node ); - -private: - void update_mean_depth( branching_node* node ); - void update_direction_counts( branching_node* node ); -}; - -struct iid_node_dependence_props { - std::vector< branching_node* > all_paths; - std::set< node_direction > interesting_nodes; - std::vector< std::vector< float > > matrix; - std::vector< float > best_values; - - std::unordered_map< std::pair< location_id, bool >, std::set< node_direction > > dependencies_by_loops; - std::unordered_map< location_id, std::set< node_direction > > dependencies_by_loading; - - coverage_value_props all_cov_value_props; - std::map< float, coverage_value_props, FloatComparator > cov_values_to_props; - - bool update_interesting_nodes( branching_node* node ); - void recompute_matrix(); - void add_equation( branching_node* path ); - std::map< location_id, path_decision > generate_path(); - std::unordered_map< location_id::id_type, float > generate_probabilities(); - - void compute_dependencies_by_loading( const loop_to_bodies_t& loop_heads_to_bodies, branching_node* end_node ); - -private: - std::vector< float > approximate_matrix(); - int get_possible_depth() const; - void dependencies_generation(); - void get_best_subset( std::vector< TableRow > const& table, - std::vector< std::set< node_direction > > const& subsets, - std::set< node_direction > const& all_leafs ); - void print_dependencies(); - std::vector< node_direction > get_all_leafs(); - std::map< location_id::id_type, std::pair< float, float > > - compute_counts_from_leaf_counts( std::vector< float > const& leaf_counts, - std::vector< node_direction > all_leafs ); - std::tuple< std::vector< std::vector< float > >, std::vector< float > > - get_unique_matrix_and_values( std::vector< std::vector< float > > const& full_matrix ); - std::set< DirectionVector > vector_computation(); - void print_subsets( std::set< node_direction > const& subset, - GradientDescentResult const& result, - std::vector< float > const& node_counts ); - void print_table( std::set< node_direction > const& all_leafs, std::vector< TableRow > const& table ); - void compute_dependencies_by_loading( loading_loops_t& loading_loops, branching_node* end_node ); - - std::vector< std::vector< float > > get_matrix( std::vector< node_direction > const& subset ) const; - std::vector< std::vector< float > > get_matrix( std::set< node_direction > const& subset ) const; - std::vector< std::set< node_direction > > get_subsets( std::set< node_direction > const& all_leafs ); -}; - -struct iid_dependencies { - std::unordered_map< location_id, iid_node_dependence_props > id_to_equation_map; - std::set< location_id > non_iid_nodes; - - void update_ignored_nodes( sensitivity_analysis& sensitivity ); - void process_node_dependence( branching_node* node ); -}; - -std::vector< fuzzing::node_direction > get_path( branching_node* node ); -int linear_interpolation( int x1, int y1, int x2, int y2, int x ); -} // namespace fuzzing \ No newline at end of file diff --git a/src/fuzzing/src/gradient_descent.cpp b/src/fuzzing/src/gradient_descent.cpp deleted file mode 100644 index 3e9b6699..00000000 --- a/src/fuzzing/src/gradient_descent.cpp +++ /dev/null @@ -1,356 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/** - * @brief Constructs a GradientDescent object. - * - * @param coefficient_matrix The matrix of coefficients for the linear system. - * @param target_vector The target vector for the linear system. - * @param learning_rate The learning rate for gradient descent (default: 0.001). - * @param max_iterations The maximum number of iterations (default: 10000). - * @param convergence_threshold The threshold for convergence (default: 1e-6). - * @param momentum The momentum factor for gradient descent (default: 0.9). - * @throws `std::invalid_argument` if input dimensions are invalid. - */ -GradientDescent::GradientDescent( const std::vector< std::vector< float > >& coefficient_matrix, - const std::vector< float >& target_vector, - float learning_rate, - int max_iterations, - float convergence_threshold, - float momentum, - int stagnant_threshold, - float improvement_threshold, - float error_threshold ) - : _coefficient_matrix( coefficient_matrix ) - , _target_vector( target_vector ) - , _learning_rate( learning_rate ) - , _max_iterations( max_iterations ) - , _convergence_threshold( convergence_threshold ) - , _momentum( momentum ) - , _stagnant_threshold( stagnant_threshold ) - , _improvement_threshold( improvement_threshold ) - , _error_threshold( error_threshold ) -{ - std::set< std::vector< float > > unique_rows; - std::vector< float > new_target_vector; - std::vector< std::vector< float > > new_coefficient_matrix; - - for ( size_t i = 0; i < _coefficient_matrix.size(); ++i ) { - std::vector< float > row = _coefficient_matrix[ i ]; - row.push_back( _target_vector[ i ] ); - - if ( unique_rows.insert( row ).second ) { - new_target_vector.push_back( _target_vector[ i ] ); - new_coefficient_matrix.push_back( _coefficient_matrix[ i ] ); - } - } - - _coefficient_matrix = new_coefficient_matrix; - _target_vector = new_target_vector; - - if ( _coefficient_matrix.empty() || _target_vector.empty() || - _coefficient_matrix.size() != _target_vector.size() ) { - throw std::invalid_argument( "Invalid input dimensions" ); - } - - for ( auto& row : _coefficient_matrix ) { - row.push_back( 1.0f ); - } -} - -/** - * @brief Performs gradient descent optimization with momentum. - * - * @return std::vector The optimized solution vector. - */ -std::tuple< std::vector< float >, bool > GradientDescent::optimize() -{ - if ( false ) { - std::cout << "### Coefficient Matrix and Target Vector:" << std::endl; - for ( size_t i = 0; i < _coefficient_matrix.size(); ++i ) { - for ( const auto& val : _coefficient_matrix[ i ] ) { - std::cout << val << " "; - } - std::cout << "| " << _target_vector[ i ] << std::endl; - } - } - - if ( true ) { - std::cout << "### Equation Matrix:" << std::endl; - std::cout << "$$\\begin{bmatrix}" << std::endl; - for ( size_t i = 0; i < _coefficient_matrix.size(); ++i ) { - for ( size_t j = 0; j < _coefficient_matrix[ i ].size(); ++j ) { - std::cout << _coefficient_matrix[ i ][ j ]; - if ( j < _coefficient_matrix[ i ].size() - 1 ) { - std::cout << " & "; - } - } - std::cout << " & " << _target_vector[ i ]; - if ( i < _coefficient_matrix.size() - 1 ) { - std::cout << " \\\\"; - } - std::cout << std::endl; - } - std::cout << "\\end{bmatrix}$$" << std::endl; - } - - std::vector< float > current_solution = generate_random_weights( _coefficient_matrix[ 0 ].size() ); - if ( _debug ) { - std::cout << "Current solution: "; - for ( const auto& val : current_solution ) { - std::cout << val << " "; - } - std::cout << std::endl; - } - std::vector< float > velocity( current_solution.size(), 0.0f ); - bool converged = false; - int stagnant_iterations = 0; - - float prev_cost = std::numeric_limits< float >::max(); - - for ( int iteration = 0; iteration < _max_iterations; ++iteration ) { - std::vector< float > gradient = compute_gradient( current_solution ); - - for ( size_t i = 0; i < current_solution.size(); ++i ) { - velocity[ i ] = _momentum * velocity[ i ] - _learning_rate * gradient[ i ]; - current_solution[ i ] += velocity[ i ]; - } - - float current_cost = compute_mean_squared_error( current_solution ); - - if ( _debug && iteration % 25 == 0 ) { - std::cout << "Iteration " << iteration << ", Cost: " << current_cost << std::endl; - } - - if ( std::abs( current_cost ) < std::abs( prev_cost ) && - std::abs( current_cost - prev_cost ) < _improvement_threshold ) { - stagnant_iterations = 0; - } else if ( std::abs( current_cost - prev_cost ) < _improvement_threshold ) { - ++stagnant_iterations; - if ( stagnant_iterations >= _stagnant_threshold ) { - if ( true ) { - std::cout << "Stopped early due to lack of significant improvement after " - << iteration << " iterations." << std::endl; - } - break; - } - } - - if ( std::abs( current_cost - prev_cost ) < _convergence_threshold ) { - if ( _debug ) { - std::cout << "Converged after " << iteration << " iterations." << std::endl; - } - - bool high_error = false; - for ( size_t i = 0; i < _coefficient_matrix.size(); ++i ) { - float predicted = dot_product( current_solution, _coefficient_matrix[ i ] ); - float error = predicted - _target_vector[ i ]; - if ( std::abs( error ) > 0.1 ) { - high_error = true; - break; - } - } - - if ( !high_error ) { - converged = true; - } - - break; - } - prev_cost = current_cost; - } - - return { current_solution, converged }; -} - -/** - * @brief Shuffles the dataset (coefficient matrix and target vector). - */ -void GradientDescent::shuffle_data() -{ - std::random_device rd; - std::mt19937 g( rd() ); - - // Shuffle the coefficient matrix and target vector together - std::vector< size_t > indices( _coefficient_matrix.size() ); - std::iota( indices.begin(), indices.end(), 0 ); - std::shuffle( indices.begin(), indices.end(), g ); - - std::vector< std::vector< float > > shuffled_matrix = _coefficient_matrix; - std::vector< float > shuffled_target = _target_vector; - - for ( size_t i = 0; i < indices.size(); ++i ) { - shuffled_matrix[ i ] = _coefficient_matrix[ indices[ i ] ]; - shuffled_target[ i ] = _target_vector[ indices[ i ] ]; - } - - _coefficient_matrix = shuffled_matrix; - _target_vector = shuffled_target; -} - -/** - * @brief Computes the gradient for the current solution. - * - * @param current_solution The current solution vector. - * @return std::vector The computed gradient vector. - */ -std::vector< float > GradientDescent::compute_gradient( const std::vector< float >& current_solution ) -{ - std::vector< float > gradient( current_solution.size(), 0.0f ); - - for ( size_t i = 0; i < _coefficient_matrix.size(); ++i ) { - float predicted = dot_product( current_solution, _coefficient_matrix[ i ] ); - float error = predicted - _target_vector[ i ]; - - for ( size_t j = 0; j < current_solution.size(); ++j ) { - gradient[ j ] += 2 * error * _coefficient_matrix[ i ][ j ]; - } - } - - for ( float& grad : gradient ) { - grad /= _coefficient_matrix.size(); - } - - return gradient; -} - -/** - * @brief Computes the mean squared error for the current solution. - * - * @param current_solution The current solution vector. - * @return float The computed mean squared error. - */ -float GradientDescent::compute_mean_squared_error( const std::vector< float >& current_solution ) -{ - float mse = 0.0f; - - for ( size_t i = 0; i < _coefficient_matrix.size(); ++i ) { - float predicted = dot_product( current_solution, _coefficient_matrix[ i ] ); - float error = predicted - _target_vector[ i ]; - mse += error * error; - } - - return mse / _coefficient_matrix.size(); -} - -/** - * @brief Computes the dot product of two vectors. - * - * @param a The first vector. - * @param b The second vector. - * @return float The dot product of the two vectors. - * @throws std::invalid_argument if the vectors have different sizes. - */ -float GradientDescent::dot_product( const std::vector< float >& a, const std::vector< float >& b ) -{ - if ( a.size() != b.size() ) { - throw std::invalid_argument( "Vectors must have the same size for dot product" ); - } - - return std::inner_product( a.begin(), a.end(), b.begin(), 0.0f ); -} - -/** - * @brief Generates a vector of random weights. - * - * @param n The size of the weight vector to generate. - * @return std::vector A vector of random weights. - */ -std::vector< float > GradientDescent::generate_random_weights( size_t n ) -{ - std::vector< float > weights( n ); - std::random_device rd; - std::mt19937 gen( rd() ); - std::uniform_real_distribution<> d( -0.01, 0.01 ); - - std::generate( weights.begin(), weights.end(), [ & ]() { return d( gen ); } ); - - return weights; -} - -/** - * @brief Rescales the values in a vector to a specified range. - * - * @param values The vector of values to rescale. - * @param min_value The minimum value of the new range. - * @param max_value The maximum value of the new range. - */ -void GradientDescent::rescale( std::vector< float >& values, float min_value, float max_value ) -{ - if ( values.empty() ) { - return; - } - - auto [ min_elem, max_elem ] = std::minmax_element( values.begin(), values.end() ); - - if ( *min_elem != *max_elem ) { - std::transform( values.begin(), values.end(), values.begin(), [ & ]( float value ) { - return min_value + - ( value - *min_elem ) * ( max_value - min_value ) / ( *max_elem - *min_elem ); - } ); - } -} - -/** - * @brief Adds the smallest value in the vector to each element of the vector. - * - * This function finds the smallest value in the given vector and adds its absolute value - * to each element of the vector. If the vector is empty, the function does nothing. - * - * @param values A reference to a vector of floats to be modified. - */ -void GradientDescent::add_smallest_value( std::vector< float >& values ) -{ - auto min_elem = std::min_element( values.begin(), values.end() ); - - if ( min_elem != values.end() ) { - float min_value = *min_elem; - std::transform( values.begin(), values.end(), values.begin(), [ & ]( float value ) { - return value + std::abs( min_value ); - } ); - } -} - -void GradientDescent::min_max_normalize_target( std::vector< float >& target ) -{ - float min_value = *std::min_element( target.begin(), target.end() ); - float max_value = *std::max_element( target.begin(), target.end() ); - - for ( size_t i = 0; i < target.size(); ++i ) { - if ( max_value - min_value != 0 ) { - target[ i ] = ( target[ i ] - min_value ) / ( max_value - min_value ); - } else { - target[ i ] = 0.0f; // Handle case where all values are the same - } - } -} - -void GradientDescent::min_max_normalize( std::vector< std::vector< float > >& matrix ) -{ - for ( size_t col = 0; col < matrix[ 0 ].size(); ++col ) { - // Find the min and max for the column - float min_value = std::numeric_limits< float >::max(); - float max_value = std::numeric_limits< float >::lowest(); - - for ( size_t row = 0; row < matrix.size(); ++row ) { - min_value = std::min( min_value, matrix[ row ][ col ] ); - max_value = std::max( max_value, matrix[ row ][ col ] ); - } - - // Normalize the column values - for ( size_t row = 0; row < matrix.size(); ++row ) { - if ( max_value - min_value != 0 ) { - matrix[ row ][ col ] = ( matrix[ row ][ col ] - min_value ) / ( max_value - min_value ); - } else { - matrix[ row ][ col ] = 0.0f; // Handle case where all values are the same - } - } - } -} \ No newline at end of file diff --git a/src/fuzzing/src/gradient_descent_with_convergence.cpp b/src/fuzzing/src/gradient_descent_with_convergence.cpp deleted file mode 100644 index f70e804b..00000000 --- a/src/fuzzing/src/gradient_descent_with_convergence.cpp +++ /dev/null @@ -1,256 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -GradientDescentNew::GradientDescentNew( const std::vector< std::vector< float > >& coefficient_matrix, - const std::vector< float >& target_vector, - std::map< size_t, float > locked_columns, - float learning_rate, - int max_iterations, - float convergence_threshold, - float momentum ) - : _coefficient_matrix( coefficient_matrix ) - , _target_vector( target_vector ) - , _learning_rate( learning_rate ) - , _max_iterations( max_iterations ) - , _convergence_threshold( convergence_threshold ) - , _momentum( momentum ) -{ - std::set< std::vector< float > > unique_rows; - std::vector< float > new_target_vector; - std::vector< std::vector< float > > new_coefficient_matrix; - - for ( size_t i = 0; i < _coefficient_matrix.size(); ++i ) { - std::vector< float > row = _coefficient_matrix[ i ]; - row.push_back( _target_vector[ i ] ); - - if ( unique_rows.insert( row ).second ) { - new_target_vector.push_back( _target_vector[ i ] ); - new_coefficient_matrix.push_back( _coefficient_matrix[ i ] ); - } - } - - _coefficient_matrix = new_coefficient_matrix; - _target_vector = new_target_vector; - - if ( _coefficient_matrix.empty() || _target_vector.empty() || - _coefficient_matrix.size() != _target_vector.size() ) { - throw std::invalid_argument( "Invalid input dimensions" ); - } - - for ( auto& row : _coefficient_matrix ) { - row.push_back( 1.0f ); - } - - for ( int i = 0; i < _coefficient_matrix[ 0 ].size(); ++i ) { - if ( locked_columns.find( i ) != locked_columns.end() ) { - _locked_columns.push_back( locked_columns.at( i ) ); - } else { - _locked_columns.push_back( std::nullopt ); - } - } -} - -void GradientDescentNew::print_input_matrix() -{ - std::cout << "### Equation Matrix:" << std::endl; - std::cout << "$$\\begin{bmatrix}" << std::endl; - for ( size_t i = 0; i < _coefficient_matrix.size(); ++i ) { - for ( size_t j = 0; j < _coefficient_matrix[ i ].size(); ++j ) { - std::cout << _coefficient_matrix[ i ][ j ]; - if ( j < _coefficient_matrix[ i ].size() - 1 ) { - std::cout << " & "; - } - } - std::cout << " & " << _target_vector[ i ]; - if ( i < _coefficient_matrix.size() - 1 ) { - std::cout << " \\\\"; - } - std::cout << std::endl; - } - std::cout << "\\end{bmatrix}$$" << std::endl; -} - -GradientDescentResult GradientDescentNew::optimize() -{ - std::vector< float > current_solution = generate_random_weights( _coefficient_matrix[ 0 ].size() ); - std::vector< float > prev_errors = compute_errors( current_solution ); - std::vector< float > velocity( current_solution.size(), 0.0f ); - float prev_cost = std::numeric_limits< float >::max(); - - bool converged = false; - int iterations = 0; - - for ( ; iterations < _max_iterations; ++iterations ) { - std::vector< float > gradient = compute_gradient( current_solution ); - - for ( size_t i = 0; i < current_solution.size(); ++i ) { - if ( _locked_columns[ i ] ) { - current_solution[ i ] = _locked_columns[ i ].value(); - } else { - velocity[ i ] = _momentum * velocity[ i ] - _learning_rate * gradient[ i ]; - current_solution[ i ] += velocity[ i ]; - } - } - - std::vector< float > errors = compute_errors( current_solution ); - float current_cost = compute_mean_squared_error( errors ); - - // if ( std::abs( current_cost - prev_cost ) < _convergence_threshold ) { - // break; - // } - - prev_cost = current_cost; - prev_errors = errors; - } - - float error_variance = compute_variance( prev_errors ); - float error_mean = std::abs( compute_mean( prev_errors ) ); - std::vector< float > counts_per_column = compute_column_count( current_solution ); - - float variance_threshold = 0.0f; - float count_threshold = 0.0f; - - converged = - compute_convergence( counts_per_column, error_variance, error_mean, variance_threshold, count_threshold ); - - return GradientDescentResult{ .weights = current_solution, - .errors = prev_errors, - .column_count_weighted = counts_per_column, - .iterations = iterations, - .error_mean = error_mean, - .error_square_of_mean = float( std::pow( error_mean, 2 ) ), - .error_mean_of_squares = prev_cost, - .error_variance = error_variance, - .variance_threshold = variance_threshold, - .count_threshold = count_threshold, - .converged = converged }; -} - -std::vector< float > GradientDescentNew::compute_column_count( const std::vector< float >& current_solution ) -{ - std::vector< float > counts_per_column( current_solution.size(), 0.0f ); - - for ( size_t i = 0; i < _coefficient_matrix.size(); ++i ) { - for ( size_t j = 0; j < _coefficient_matrix[ i ].size(); ++j ) { - counts_per_column[ j ] += std::abs( _coefficient_matrix[ i ][ j ] * current_solution[ j ] ); - } - } - - return counts_per_column; -} - -bool GradientDescentNew::compute_convergence( const std::vector< float >& counts_per_column, - float error_variance, - float error_mean, - float& variance_threshold, - float& count_threshold ) -{ - constexpr float VARIANCE_SCALING_FACTOR = 10.0f; - - variance_threshold = std::abs( error_mean * VARIANCE_SCALING_FACTOR ); - - float column_count_sum = std::abs( - std::accumulate( counts_per_column.begin(), counts_per_column.end() - 1, 0.0f ) ); - count_threshold = column_count_sum * 0.01f; - - if ( error_variance > variance_threshold ) { - return false; - } - - for ( size_t i = 0; i < counts_per_column.size() - 1; ++i ) { - if ( std::abs( counts_per_column[ i ] ) < count_threshold ) { - return false; - } - } - - return true; -} - -std::vector< float > GradientDescentNew::compute_gradient( const std::vector< float >& current_solution ) -{ - std::vector< float > gradient( current_solution.size(), 0.0f ); - - for ( size_t i = 0; i < _coefficient_matrix.size(); ++i ) { - float predicted = dot_product( current_solution, _coefficient_matrix[ i ] ); - float error = predicted - _target_vector[ i ]; - - for ( size_t j = 0; j < current_solution.size(); ++j ) { - gradient[ j ] += 2 * error * _coefficient_matrix[ i ][ j ]; - } - } - - for ( float& grad : gradient ) { - grad /= _coefficient_matrix.size(); - } - - return gradient; -} - -std::vector< float > GradientDescentNew::compute_errors( const std::vector< float >& current_solution ) -{ - std::vector< float > errors( _coefficient_matrix.size(), 0.0f ); - - for ( size_t i = 0; i < _coefficient_matrix.size(); ++i ) { - float predicted = dot_product( current_solution, _coefficient_matrix[ i ] ); - errors[ i ] = predicted - _target_vector[ i ]; - } - - return errors; -} - -float GradientDescentNew::compute_mean_squared_error( const std::vector< float >& errors ) -{ - float mse = 0.0f; - - for ( size_t i = 0; i < errors.size(); ++i ) { - mse += std::pow( errors[ i ], 2 ); - } - - return mse / errors.size(); -} - -float GradientDescentNew::dot_product( const std::vector< float >& a, const std::vector< float >& b ) -{ - if ( a.size() != b.size() ) { - throw std::invalid_argument( "Vectors must have the same size for dot product" ); - } - - return std::inner_product( a.begin(), a.end(), b.begin(), 0.0f ); -} - -std::vector< float > GradientDescentNew::generate_random_weights( size_t n ) -{ - std::vector< float > weights( n ); - std::random_device rd; - std::mt19937 gen( rd() ); - std::uniform_real_distribution<> d( -0.01, 0.01 ); - - std::generate( weights.begin(), weights.end(), [ & ]() { return d( gen ); } ); - - return weights; -} - - -float GradientDescentNew::compute_variance( const std::vector< float >& errors ) -{ - float mean = compute_mean( errors ); - float error_variance = 0.0f; - - for ( const auto& error : errors ) { - error_variance += std::pow( error - mean, 2 ); - } - - return error_variance / errors.size(); -} - -float GradientDescentNew::compute_mean( const std::vector< float >& errors ) -{ - return std::accumulate( errors.begin(), errors.end(), 0.0f ) / errors.size(); -} \ No newline at end of file diff --git a/src/fuzzing/src/iid_node_dependencies.cpp b/src/fuzzing/src/iid_node_dependencies.cpp deleted file mode 100644 index 6e2450a8..00000000 --- a/src/fuzzing/src/iid_node_dependencies.cpp +++ /dev/null @@ -1,1108 +0,0 @@ -#include -#include -#include -#include -#include -#include - -bool fuzzing::path_decision::get_next_direction() -{ - if ( left_max == 0 ) { - right_current++; - return true; - } - - if ( right_max == 0 ) { - left_current++; - return false; - } - - if ( right_max < left_max ) { - left_current++; - return false; - } - - if ( left_max < right_max ) { - right_current++; - return true; - } - - return false; -} - -/** - * @brief Three-way comparison for node_direction. - * - * @param other The other node_direction object to compare with. - * @return std::strong_ordering Result of the comparison. - */ -auto fuzzing::node_direction::operator<=>( node_direction const& other ) const -{ - if ( auto const cmp = node_id.id <=> other.node_id.id; cmp != 0 ) - return cmp; - - return branching_direction <=> other.branching_direction; -} - - -/** - * @brief Equality operator for node_direction. - * - * Compares two node_direction objects for equality based on their node_id and direction. - * - * @param other The other node_direction object to compare with. - * @return true if both node_id and direction are equal, false otherwise. - */ -bool fuzzing::node_direction::operator==( node_direction const& other ) const -{ - return node_id.id == other.node_id.id && branching_direction == other.branching_direction; -} - - -/** - * @brief Adds a value to the number statistics, updating the minimum, maximum, and mean. - * - * This function updates the minimum and maximum values if the provided value is lower or higher, - * respectively. It also adds the value to the mean calculation. - * - * @param value The integer value to be added to the statistics. - */ -void fuzzing::number_statistics::add( int value ) -{ - min = std::min( min, value ); - max = std::max( max, value ); - mean.add( value ); -} - - -/** - * @brief Processes a branching node to update its properties. - * - * This function updates the mean depth and direction counts of the given branching node. - * - * @param node A pointer to the branching node to be processed. - */ -void fuzzing::coverage_value_props::process_node( branching_node* node ) -{ - update_mean_depth( node ); - update_direction_counts( node ); -} - - -/** - * @brief Updates the mean depth of a branching node. - * - * This function recalculates the mean depth of a branching node using the - * incremental average formula. It retrieves the depth of the provided node - * and updates the mean depth accordingly. - * - * @param node A pointer to the branching_node whose depth is to be used - * for updating the mean depth. - */ -void fuzzing::coverage_value_props::update_mean_depth( branching_node* node ) -{ - int node_depth = node->get_depth(); - path_depth.add( node_depth ); -} - - -/** - * @brief Updates the direction counts for a given branching node. - * - * This function calculates the direction counts for the path of the given branching node - * and updates the minimum and maximum statistics for each direction. - * - * @param node A pointer to the branching node for which the direction counts are to be updated. - */ -void fuzzing::coverage_value_props::update_direction_counts( branching_node* node ) -{ - std::vector< node_direction > path = get_path( node ); - - std::map< node_direction, int > direction_counts; - for ( const node_direction& nav : path ) { - direction_counts[ nav ]++; - } - - for ( auto& [ direction, count ] : direction_counts ) { - auto& stats = direction_statistics[ direction ]; - stats.add( count ); - } -} - - -/** - * @brief Updates the set of interesting nodes based on the given branching node. - * - * This function traverses the paths from the given branching node and updates the set of - * interesting nodes by comparing the paths with all existing paths. If new interesting nodes are - * found, the set is updated and the function returns true. - * - * @param node A pointer to the branching node from which to start the path traversal. - * @return true if the set of interesting nodes was updated, false otherwise. - */ -bool fuzzing::iid_node_dependence_props::update_interesting_nodes( branching_node* node ) -{ - TMPROF_BLOCK(); - - bool set_changed = false; - - auto add_to_interesting = [ this, &set_changed ]( std::vector< node_direction >& nodes, int i ) { - for ( ; i >= 0; --i ) { - number_statistics& stats = all_cov_value_props.direction_statistics[ nodes[ i ] ]; - if ( stats.min == stats.max ) - continue; - - auto result = this->interesting_nodes.emplace( nodes[ i ] ); - if ( result.second ) { - set_changed = true; - } - } - }; - - for ( const auto& end_node : all_paths ) { - std::vector< node_direction > path_1 = get_path( node ); - std::vector< node_direction > path_2 = get_path( end_node ); - - if ( path_1.empty() || path_2.empty() ) - continue; - - ASSUMPTION( path_1.back() == path_2.back() ); - - std::size_t i_1 = path_1.size() - 1; - std::size_t i_2 = path_2.size() - 1; - - while ( i_1 > 0 && i_2 > 0 && path_1[ i_1 ] == path_2[ i_2 ] ) { - --i_1; - --i_2; - } - - add_to_interesting( path_1, i_1 ); - add_to_interesting( path_2, i_2 ); - } - - return set_changed; -} - - -/** - * @brief Recomputes the matrix of IID dependence properties. - * - * This function clears the current matrix and best values, then iterates - * through all paths to add equations to the matrix. If there are no paths, - * the function returns immediately. - * - * @note The matrix clearing could be optimized in the future. - */ -void fuzzing::iid_node_dependence_props::recompute_matrix() -{ - TMPROF_BLOCK(); - - if ( all_paths.empty() ) - return; - - matrix.clear(); - best_values.clear(); - - for ( const auto& path : all_paths ) { - add_equation( path ); - } -} - - -/** - * @brief Adds an equation to the IID dependence properties based on the given branching path. - * - * This function updates the internal matrix and best values with the direction counts of - * interesting nodes encountered in the path. - * - * @param path A pointer to the branching_node representing the path to be processed. - */ -void fuzzing::iid_node_dependence_props::add_equation( branching_node* path ) -{ - TMPROF_BLOCK(); - - std::map< node_direction, int > directions_in_path; - for ( const node_direction& navigation : interesting_nodes ) { - directions_in_path[ navigation ] = 0; - } - - std::vector< node_direction > path_nodes = get_path( path ); - - for ( const node_direction& nav : path_nodes ) { - if ( interesting_nodes.contains( nav ) ) { - directions_in_path[ nav ]++; - } - } - - std::vector< float > values_in_path; - for ( const auto& [ direction, count ] : directions_in_path ) { - values_in_path.push_back( count ); - } - - matrix.push_back( values_in_path ); - best_values.push_back( path->best_coverage_value ); -} - - -std::vector< std::set< fuzzing::node_direction > > -fuzzing::iid_node_dependence_props::get_subsets( std::set< node_direction > const& all_leafs ) -{ - std::vector< std::set< node_direction > > subsets; - std::vector< node_direction > leafs_vector( all_leafs.begin(), all_leafs.end() ); - int n = leafs_vector.size(); - - for ( int i = 1; i < ( 1 << n ); ++i ) { - std::set< node_direction > subset; - for ( int j = 0; j < n; ++j ) { - if ( i & ( 1 << j ) ) { - subset.insert( leafs_vector[ j ] ); - } - } - subsets.push_back( subset ); - } - - std::sort( subsets.begin(), - subsets.end(), - []( const std::set< node_direction >& a, const std::set< node_direction >& b ) { - return a.size() < b.size(); - } ); - - return subsets; -} - -std::vector< std::vector< float > > -fuzzing::iid_node_dependence_props::get_matrix( std::set< node_direction > const& subset ) const -{ - return get_matrix( std::vector< node_direction >( subset.begin(), subset.end() ) ); -} - -std::vector< std::vector< float > > -fuzzing::iid_node_dependence_props::get_matrix( std::vector< node_direction > const& subset ) const -{ - std::vector< std::vector< float > > sub_matrix; - for ( const auto& row : matrix ) { - std::vector< float > sub_row; - - for ( const auto& direction : subset ) { - int idx = std::distance( interesting_nodes.begin(), interesting_nodes.find( direction ) ); - sub_row.push_back( row[ idx ] ); - } - - sub_matrix.push_back( sub_row ); - } - - return sub_matrix; -} - - -void fuzzing::iid_node_dependence_props::print_dependencies() -{ - std::cout << "# Dependencies:" << std::endl; - std::cout << "## Dependencies by loops:" << std::endl; - for ( const auto& [ loop, nodes ] : dependencies_by_loops ) { - for ( const auto& body : nodes ) { - std::cout << "- " << "`(" << body << ") → " << loop.first.id << "`" << std::endl; - } - } - - std::cout << "## Dependencies by sensitivity:" << std::endl; - - std::cout << "## Dependencies by loading:" << std::endl; - for ( const auto& [ loading, nodes ] : dependencies_by_loading ) { - for ( const auto& body : nodes ) { - std::cout << "- " << "`(" << body << ") → " << loading.id << "`" << std::endl; - } - } - - std::cout << "# Subsets:" << std::endl; -} - -void fuzzing::iid_node_dependence_props::print_subsets( std::set< node_direction > const& subset, - GradientDescentResult const& result, - std::vector< float > const& node_counts ) -{ - std::cout << "## Subset of nodes: `{ "; - auto delimeter = ""; - for ( const auto& leaf : subset ) { - std::cout << delimeter << "(" << leaf << ")"; - delimeter = ", "; - } - std::cout << " }`" << std::endl; - - std::cout << "### Weights and Counts:" << std::endl; - std::cout << "- `Iterations: " << result.iterations << "`" << std::endl; - std::cout << "- `Error variance: " << result.error_variance << "`" << std::endl; - std::cout << "- `Error mean: " << result.error_mean << "`" << std::endl; - std::cout << "- `Error square of mean: " << result.error_square_of_mean << "`" << std::endl; - std::cout << "- `Error mean of squares: " << result.error_mean_of_squares << "`" << std::endl; - std::cout << "- `Variance threshold: " << result.variance_threshold << "`" << std::endl; - std::cout << "- `Count threshold: " << result.count_threshold << "`" << std::endl; - std::cout << "- `Converged: " << ( result.converged ? "True" : "False" ) << "`" << std::endl; - - for ( size_t i = 0; i < subset.size(); ++i ) { - const auto& node = *std::next( subset.begin(), i ); - std::cout << "- `(" << node << "):`" << std::endl; - std::cout << " - `Weight: " << result.weights[ i ] << "`" << std::endl; - std::cout << " - `Count: " << static_cast< int >( std::round( node_counts[ i ] ) ) << "`" << std::endl; - std::cout << " - `Column count weighted: " << result.column_count_weighted[ i ] << "`" << std::endl; - } - - std::cout << "- `Increment:`\n - `Weight: " << result.weights.back() - << "`\n - `Column count weighted: " << result.column_count_weighted.back() << "`" << std::endl; -} - -void fuzzing::iid_node_dependence_props::print_table( std::set< node_direction > const& all_leafs, - std::vector< TableRow > const& table ) -{ - std::unordered_map< std::string, int > headers{ - { "Error mean", 0 }, - { "Error variance", 1 }, - // {"Error square of mean", 2}, - // {"Error mean of squares", 3}, - // {"Variance threshold", 4}, - { "Converged", 5 }, - // { "Iterations", 6 }, - // { "Variance threshold", 7 }, - // { "Count threshold", 8 }, - }; - - if ( std::pow( all_leafs.size(), 2 ) - 1 == table.size() ) { - std::cout << "# Result table:" << std::endl; - } else { - std::cout << "# Best result table:" << std::endl; - } - - std::cout << "| "; - for ( const auto& node : all_leafs ) { - std::cout << node << " | "; - } - std::cout << "Increment | "; - for ( const auto& header : headers ) { - std::cout << header.first << " | "; - } - std::cout << std::endl; - - std::cout << "| "; - for ( size_t i = 0; i <= all_leafs.size() + headers.size(); ++i ) { - std::cout << "--- | "; - } - std::cout << std::endl; - - for ( const auto& row : table ) { - std::cout << "| "; - for ( size_t i = 0; i < row.weights.size(); ++i ) { - const auto& value = row.weights[ i ]; - if ( value.has_value() ) { - std::cout << value.value(); - } - std::cout << " | "; - } - - for ( const auto& header : headers ) { - int idx = header.second; - switch ( idx ) { - case 0: std::cout << row.result.error_mean; break; - case 1: std::cout << row.result.error_variance; break; - case 2: std::cout << row.result.error_square_of_mean; break; - case 3: std::cout << row.result.error_mean_of_squares; break; - case 4: std::cout << row.result.variance_threshold; break; - case 5: std::cout << ( row.result.converged ? "True" : "False" ); break; - case 6: std::cout << row.result.iterations; break; - case 7: std::cout << row.result.variance_threshold; break; - case 8: std::cout << row.result.count_threshold; break; - } - std::cout << " | "; - } - - std::cout << std::endl; - } - - std::cout << std::endl; -} - -void fuzzing::iid_node_dependence_props::get_best_subset( std::vector< TableRow > const& table, - std::vector< std::set< node_direction > > const& subsets, - std::set< node_direction > const& all_leafs ) -{ - std::vector< TableRow > table_copy = table; - - for ( const auto& row_locked : table_copy ) { - for ( int subset_idx = 0; subset_idx < subsets.size(); ++subset_idx ) { - auto& row_to_change = table_copy[ subset_idx ]; - - if ( row_locked == row_to_change ) { - continue; - } - - bool can_be_locked = true; - for ( size_t i = 0; i < row_locked.weights.size() - 1; ++i ) { - if ( row_locked.weights[ i ].has_value() && !row_to_change.weights[ i ].has_value() ) { - can_be_locked = false; - break; - } - } - - if ( !can_be_locked ) { - continue; - } - - std::set< node_direction > subset = subsets[ subset_idx ]; - - std::map< size_t, float > locked_columns; - int column_idx = 0; - for ( size_t i = 0; i < row_locked.weights.size() - 1; ++i ) { - if ( row_locked.weights[ i ].has_value() ) { - locked_columns[ column_idx ] = row_locked.weights[ i ].value(); - } - - if ( row_to_change.weights[ i ].has_value() ) { - column_idx++; - } - } - - GradientDescentNew gd( get_matrix( subset ), best_values, locked_columns ); - auto result = gd.optimize(); - - if ( result.error_mean < row_to_change.result.error_mean && result.converged ) { - std::cout << "- Error was reduced from `" << row_to_change.result.error_mean << "` to `" - << result.error_mean - << "`, converged: " << ( result.converged ? "`True`" : "`False`" ) << std::endl; - - for ( size_t i = 0; i < all_leafs.size(); ++i ) { - const auto& node = *std::next( all_leafs.begin(), i ); - - auto it = std::find( subset.begin(), subset.end(), node ); - if ( it != subset.end() ) { - row_to_change.weights[ i ] = result.weights[ std::distance( subset.begin(), it ) ]; - } else { - row_to_change.weights[ i ] = std::nullopt; - } - } - - row_to_change.result.error_mean = result.error_mean; - row_to_change.result.converged = row_to_change.result.converged || result.converged; - } - } - } - - std::sort( table_copy.begin(), table_copy.end(), []( const TableRow& a, const TableRow& b ) { - if ( a.get_non_null_count() != b.get_non_null_count() ) { - return a.get_non_null_count() < b.get_non_null_count(); - } - - return a.result.error_mean < b.result.error_mean; - } ); - - std::erase_if( table_copy, [ & ]( const TableRow& row ) { - return !row.result.converged && row.get_non_null_count() != all_leafs.size() + 1; - } ); - - print_table( all_leafs, table_copy ); -} - -void fuzzing::iid_node_dependence_props::dependencies_generation() -{ - print_dependencies(); - std::set< node_direction > all_leafs; - for ( const auto& [ _, loop_bodies ] : dependencies_by_loops ) { - all_leafs.insert( loop_bodies.begin(), loop_bodies.end() ); - } - - std::vector< std::set< node_direction > > subsets = get_subsets( all_leafs ); - - std::vector< TableRow > table; - - - for ( const auto& subset : subsets ) { - std::vector< std::vector< float > > sub_matrix = get_matrix( subset ); - - GradientDescentNew gd( sub_matrix, best_values ); - auto result = gd.optimize(); - - std::vector< float > const& weights = result.weights; - - float dot_product = std::inner_product( weights.begin(), weights.end() - 1, weights.begin(), 0.0f ); - std::vector< float > node_counts; - for ( auto it = weights.begin(); it != weights.end() - 1; ++it ) { - node_counts.push_back( -weights.back() * ( *it ) / dot_product ); - } - - std::vector< std::optional< float > > weights_with_nulls; - for ( const auto& node : all_leafs ) { - auto it = std::find( subset.begin(), subset.end(), node ); - if ( it != subset.end() ) { - weights_with_nulls.push_back( weights[ std::distance( subset.begin(), it ) ] ); - } else { - weights_with_nulls.push_back( std::nullopt ); - } - } - - weights_with_nulls.push_back( weights.back() ); - TableRow row( weights_with_nulls, result ); - - table.push_back( row ); - - print_subsets( subset, result, node_counts ); - gd.print_input_matrix(); - } - - print_table( all_leafs, table ); - get_best_subset( table, subsets, all_leafs ); -} - -std::vector< fuzzing::node_direction > fuzzing::iid_node_dependence_props::get_all_leafs() -{ - std::set< node_direction > all_leafs; - for ( const auto& [ _, loop_bodies ] : dependencies_by_loops ) { - all_leafs.insert( loop_bodies.begin(), loop_bodies.end() ); - } - - return std::vector< node_direction >( all_leafs.begin(), all_leafs.end() ); -} - -std::tuple< std::vector< std::vector< float > >, std::vector< float > > -fuzzing::iid_node_dependence_props::get_unique_matrix_and_values( std::vector< std::vector< float > > const& full_matrix ) -{ - std::set< std::vector< float > > unique_rows; - std::vector< float > new_best_values; - std::vector< std::vector< float > > matrix; - - for ( size_t i = 0; i < full_matrix.size(); ++i ) { - auto row = full_matrix[ i ]; - row.push_back( best_values[ i ] ); - - if ( unique_rows.insert( row ).second ) { - new_best_values.push_back( best_values[ i ] ); - matrix.push_back( full_matrix[ i ] ); - } - } - - return { matrix, new_best_values }; -} - -std::set< DirectionVector > fuzzing::iid_node_dependence_props::vector_computation() -{ - std::vector< fuzzing::node_direction > all_leafs = get_all_leafs(); - std::vector< std::vector< float > > full_matrix = get_matrix( all_leafs ); - auto [ matrix, new_best_values ] = get_unique_matrix_and_values( full_matrix ); - - std::set< DirectionVector > vectors_with_hits; - - auto is_approximately_equal = []( const auto& a, const auto& b ) { - constexpr float epsilon = 1e-6; - return a.size() == b.size() && std::equal( a.begin(), a.end(), b.begin(), [ epsilon ]( float x, float y ) { - return std::fabs( x - y ) <= epsilon; - } ); - }; - - for ( size_t i = 0; i < matrix.size(); ++i ) { - for ( size_t j = 0; j < matrix.size(); ++j ) { - if ( i == j ) - continue; - - std::vector< float > diff_vector; - std::transform( matrix[ i ].begin(), - matrix[ i ].end(), - matrix[ j ].begin(), - std::back_inserter( diff_vector ), - std::minus<>() ); - - if ( std::any_of( diff_vector.begin(), diff_vector.end(), []( float val ) { return val < 0; } ) ) - continue; - - float best_value_diff = new_best_values[ i ] - new_best_values[ j ]; - if ( best_value_diff != 0 ) { - int compare_hits = 0; - - for ( const auto& row : matrix ) { - std::vector< float > new_row( row.size() ); - std::transform( row.begin(), row.end(), diff_vector.begin(), new_row.begin(), std::plus<>() ); - - if ( std::any_of( matrix.begin(), matrix.end(), [ & ]( const auto& existing_row ) { - return is_approximately_equal( new_row, existing_row ); - } ) ) { - compare_hits++; - } - } - - vectors_with_hits.insert( { diff_vector, best_value_diff, compare_hits } ); - } - } - } - - // std::cout << "Vectors:\n"; - // for ( const auto& vec : vectors_with_hits ) { - // std::cout << "( "; - // for ( size_t i = 0; i < vec.vector.size(); ++i ) { - // std::cout << ( i ? ", " : "" ) << vec.vector[ i ]; - // } - // std::cout << " ) -> " << vec.value << " (" << vec.compare_hits << ")\n"; - // } - - return vectors_with_hits; -} - -std::unordered_map< location_id::id_type, float > fuzzing::iid_node_dependence_props::generate_probabilities() -{ - // dependencies_generation(); - std::vector< node_direction > all_leafs = get_all_leafs(); - auto [ matrix, new_best_values ] = get_unique_matrix_and_values( get_matrix( all_leafs ) ); - std::set< DirectionVector > vectors = vector_computation(); - - float epsilon = 1e-6; - int column_count = matrix[ 0 ].size(); - - std::set< std::vector< float > > paths; - // std::cout << "Path counts:" << std::endl; - for ( const auto& vec : vectors ) { - - for ( int i = 0; i < matrix.size(); ++i ) { - std::vector< float > path_counts( column_count ); - - float curr_best_value = std::abs( new_best_values[ i ] ); - - float counts = curr_best_value / vec.value; - int counts_int = static_cast< int >( std::round( counts ) ); - if ( std::abs( counts - counts_int ) > epsilon ) { - continue; - } - - for ( int j = 0; j < column_count; ++j ) { - path_counts[ j ] = matrix[ i ][ j ] + vec.vector[ j ] * counts_int; - } - - if ( std::any_of( path_counts.begin(), path_counts.end(), []( float val ) { return val < 0; } ) ) { - continue; - } - - // std::cout << "( "; - // for ( size_t j = 0; j < column_count; ++j ) { - // std::cout << ( j ? ", " : "" ) << path_counts[ j ]; - // } - // std::cout << " ) " << std::endl; - - - paths.insert( path_counts ); - } - } - - std::vector< std::vector< float > > sorted_paths( paths.begin(), paths.end() ); - std::sort( sorted_paths.begin(), - sorted_paths.end(), - []( const std::vector< float >& a, const std::vector< float >& b ) { - float a_length = std::inner_product( a.begin(), a.end(), a.begin(), 0.0f ); - float b_length = std::inner_product( b.begin(), b.end(), b.begin(), 0.0f ); - return a_length < b_length; - } ); - - // for ( const auto& path : sorted_paths ) { - // std::cout << "( "; - // for ( size_t i = 0; i < path.size(); ++i ) { - // std::cout << ( i ? ", " : "" ) << path[ i ]; - // } - // std::cout << " )" << std::endl; - // } - - std::map< location_id::id_type, std::pair< float, float > > path_counts = compute_counts_from_leaf_counts( sorted_paths[ 0 ], all_leafs ); - for (const auto& [id, counts] : path_counts) { - std::cout << "Location ID: " << id << ", Left Count: " << counts.first << ", Right Count: " << counts.second << std::endl; - } - - std::unordered_map< location_id::id_type, float > path_probabilities; - for ( int i = 0; i < path_counts.size(); i += 2 ) {} - - // for ( int i = 0; i < all_leafs.size(); i += 2 ) { - // int sum = sorted_paths[ 0 ][ i ] + sorted_paths[ 0 ][ i + 1 ]; - // location_id::id_type id = std::next( all_leafs.begin(), i )->node_id.id; - // if ( sum > 0 ) - // path_probabilities[ id ] = sorted_paths[ 0 ][ i ] / sum; - // } - - // for (const auto& [id, count] : path_probabilities) { - // std::cout << "Location ID: " << id << ", Path Probability: " << count << std::endl; - // } - - return path_probabilities; - return {}; -} - -std::map< location_id::id_type, std::pair< float, float > > -fuzzing::iid_node_dependence_props::compute_counts_from_leaf_counts( std::vector< float > const& leaf_counts, - std::vector< node_direction > all_leafs ) -{ - std::map< location_id::id_type, std::pair< float, float > > path_counts; - for ( int i = 0; i < all_leafs.size(); i += 2 ) { - float leaf_count = leaf_counts[ i ]; - node_direction leaf = all_leafs[ i ]; - - auto& [ left_count, right_count ] = path_counts[ leaf.node_id.id ]; - if ( leaf.branching_direction ) { - right_count = leaf_count; - } else { - left_count = leaf_count; - } - } - - for ( const auto& [ loop_head, dependent_bodies ] : dependencies_by_loops ) { - float loop_count = 0; - for ( const auto& body : dependent_bodies ) { - auto& [ left_count, right_count ] = path_counts[ body.node_id.id ]; - loop_count += right_count; - loop_count += left_count; - } - - location_id head_id = loop_head.first; - bool end_direction = loop_head.second; - - if ( end_direction ) { - path_counts[ head_id.id ] = { 1, loop_count }; - } else { - path_counts[ head_id.id ] = { loop_count, 1 }; - } - } - - return path_counts; -} - - -std::map< location_id, fuzzing::path_decision > fuzzing::iid_node_dependence_props::generate_path() -{ - // // dependencies_generation(); - // std::vector< node_direction > all_leafs = get_all_leafs(); - // auto [ matrix, new_best_values ] = get_unique_matrix_and_values( get_matrix( all_leafs ) ); - // std::set< DirectionVector > vectors = vector_computation(); - - // // auto min_value_it = std::min_element( best_values.begin(), best_values.end() ); - // // int min_index = std::distance( best_values.begin(), min_value_it ); - // // std::vector< float > min_row = matrix[ min_index ]; - - // float epsilon = 1e-6; - // int column_count = matrix[ 0 ].size(); - - // std::set< std::vector< float > > paths; - // std::cout << "Path counts:" << std::endl; - // for ( const auto& vec : vectors ) { - - // for ( int i = 0; i < matrix.size(); ++i ) { - // std::vector< float > path_counts( column_count ); - - // float curr_best_value = std::abs( new_best_values[ i ] ); - - // float counts = curr_best_value / vec.value; - // int counts_int = static_cast< int >( std::round( counts ) ); - // if ( std::abs( counts - counts_int ) > epsilon ) { - // continue; - // } - - // for ( int j = 0; j < column_count; ++j ) { - // path_counts[ j ] = matrix[ i ][ j ] + vec.vector[ j ] * counts_int; - // } - - // if ( std::any_of( path_counts.begin(), path_counts.end(), []( float val ) { return val < 0; } ) ) { - // continue; - // } - - // // std::cout << "( "; - // // for ( size_t j = 0; j < column_count; ++j ) { - // // std::cout << ( j ? ", " : "" ) << path_counts[ j ]; - // // } - // // std::cout << " ) " << std::endl; - - - // paths.insert( path_counts ); - // } - // } - - // std::vector< std::vector< float > > sorted_paths( paths.begin(), paths.end() ); - // std::sort( sorted_paths.begin(), - // sorted_paths.end(), - // []( const std::vector< float >& a, const std::vector< float >& b ) { - // float a_length = std::inner_product( a.begin(), a.end(), a.begin(), 0.0f ); - // float b_length = std::inner_product( b.begin(), b.end(), b.begin(), 0.0f ); - // return a_length < b_length; - // } ); - - // // for ( const auto& path : sorted_paths ) { - // // std::cout << "( "; - // // for ( size_t i = 0; i < path.size(); ++i ) { - // // std::cout << ( i ? ", " : "" ) << path[ i ]; - // // } - // // std::cout << " )" << std::endl; - // // } - - // std::unordered_map< location_id::id_type, float > path_counts; - // INVARIANT( all_leafs.size() % 2 == 0 ); - - // for ( int i = 0; i < all_leafs.size(); i += 2 ) { - // int sum = sorted_paths[ 0 ][ i ] + sorted_paths[ 0 ][ i + 1 ]; - // location_id::id_type id = std::next( all_leafs.begin(), i )->node_id.id; - // if ( sum > 0 ) - // path_counts[ id ] = sorted_paths[ 0 ][ i ] / sum; - // else - // path_counts[ id ] = 0.5; - // } - - // for ( const auto& [ id, count ] : path_counts ) { - // std::cout << "Location ID: " << id << ", Path Count: " << count << std::endl; - // } - return {}; -} - - -/** - * @brief Approximates a matrix using gradient descent optimization. - * - * This function utilizes the GradientDescent class to optimize the given matrix - * and best values, returning a vector of optimized weights. - * - * @return A vector of floats representing the optimized weights. - */ -std::vector< float > fuzzing::iid_node_dependence_props::approximate_matrix() -{ - GradientDescentNew gd( matrix, best_values ); - GradientDescentResult result = gd.optimize(); - - if ( false ) { - for ( size_t i = 0; i < interesting_nodes.size(); ++i ) { - const auto& node = *std::next( interesting_nodes.begin(), i ); - std::cout << "Node ID: " << node.node_id.id << ", Direction: " << node.branching_direction - << ", Weight: " << result.weights[ i ] << std::endl; - } - } - - return result.weights; -} - - -/** - * @brief Computes the possible depth based on the value to mean depth mapping. - * - * This function calculates the possible depth by examining the `cov_values_to_props` map. - * If the map is empty, it returns 0. If the map contains only one element, it returns the mean - * depth of that element. If the map contains more than one element, it calculates the depth using - * linear interpolation based on the first two elements in the map. - * - * @return The computed possible depth. - */ -int fuzzing::iid_node_dependence_props::get_possible_depth() const -{ - if ( cov_values_to_props.empty() ) { - return 0; - } - - if ( cov_values_to_props.size() == 1 ) { - return cov_values_to_props.begin()->second.path_depth.min; - } - - auto it = cov_values_to_props.begin(); - int first_depth = it->second.path_depth.min; - float first_value = it->first; - - ++it; - int second_depth = it->second.path_depth.min; - float second_value = it->first; - - int interpolated_depth = linear_interpolation( first_value, first_depth, second_value, second_depth, 0 ); - - return interpolated_depth; -} - - -/** - * @brief Updates the set of non-IID nodes based on sensitivity analysis. - * - * This function iterates through the nodes that have changed according to the - * provided sensitivity analysis. For each node that has undergone branching, - * it retrieves the node's location ID and attempts to insert it into the set - * of non-IID nodes. If the insertion is successful, it also removes the - * corresponding entry from the ID-to-equation map. - * - * @param sensitivity A reference to the sensitivity analysis object that - * provides the changed nodes. - */ -void fuzzing::iid_dependencies::update_ignored_nodes( sensitivity_analysis& sensitivity ) -{ - for ( branching_node* node : sensitivity.get_changed_nodes() ) { - if ( node->is_did_branching() ) { - auto location_id = node->get_location_id(); - if ( non_iid_nodes.insert( location_id ).second ) { - id_to_equation_map.erase( location_id ); - } - } - } -} - -void fuzzing::iid_node_dependence_props::compute_dependencies_by_loading( loading_loops_t& loading_loops, - branching_node* end_node ) -{ - for ( const auto& bit_index : end_node->sensitive_stdin_bits ) { - for ( const auto& [ loop_head, values ] : loading_loops ) { - auto& [ min, max ] = values; - - if ( bit_index >= min && bit_index <= max ) { - dependencies_by_loading[ loop_head ].insert( { end_node->get_location_id(), true } ); - dependencies_by_loading[ loop_head ].insert( { end_node->get_location_id(), false } ); - } - } - } -} - -void fuzzing::iid_node_dependence_props::compute_dependencies_by_loading( const loop_to_bodies_t& loop_heads_to_bodies, - branching_node* end_node ) -{ - branching_node* node = end_node; - loading_loops_t loading_loops; - - for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { - loading_loops[ loop_head ] = { std::numeric_limits< natural_32_bit >::max(), - std::numeric_limits< natural_32_bit >::min() }; - } - - while ( node != nullptr ) { - for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { - if ( loop_head.id == node->get_location_id().id ) { - natural_32_bit bits_count = node->get_num_stdin_bits(); - - auto& [ min, max ] = loading_loops[ loop_head ]; - min = std::min( min, bits_count ); - max = std::max( max, bits_count ); - } - } - - node = node->predecessor; - } - - node = end_node; - - while ( node != nullptr ) { - for ( const auto& bit_index : node->sensitive_stdin_bits ) { - for ( const auto& [ loop_head, values ] : loading_loops ) { - auto& [ min, max ] = values; - if ( bit_index >= min && bit_index <= max ) { - if ( interesting_nodes.contains( { node->get_location_id(), true } ) ) { - dependencies_by_loading[ loop_head ].insert( { node->get_location_id(), true } ); - } - - if ( interesting_nodes.contains( { node->get_location_id(), false } ) ) { - dependencies_by_loading[ loop_head ].insert( { node->get_location_id(), false } ); - } - } - } - } - - node = node->predecessor; - } -} - -/** - * @brief Processes the dependence of a given branching node. - * - * This function processes the dependence of a branching node by updating - * the internal state and properties associated with the node's location ID. - * It ensures that non-IID nodes are skipped, updates the list of all paths, - * and recomputes the matrix if the node is deemed interesting. Additionally, - * it adds the node to the equation map. - * - * @param node A pointer to the branching node to be processed. - */ -void fuzzing::iid_dependencies::process_node_dependence( branching_node* node ) -{ - TMPROF_BLOCK(); - - if ( non_iid_nodes.contains( node->get_location_id() ) ) - return; - - iid_node_dependence_props& props = id_to_equation_map[ node->get_location_id() ]; - std::vector< node_direction > path = get_path( node ); - for ( const node_direction& nav : path ) { - props.interesting_nodes.insert( nav ); - } - props.recompute_matrix(); - - std::unordered_map< location_id, std::unordered_set< location_id > > loop_heads_to_bodies; - fuzzing::fuzzer::detect_loops_along_path_to_node( node, loop_heads_to_bodies, nullptr ); - props.compute_dependencies_by_loading( loop_heads_to_bodies, node ); - - std::map< location_id, bool > loop_heads_ending; - branching_node* current = node; - while ( current != nullptr ) { - branching_node* predecessor = current->predecessor; - if ( predecessor != nullptr && loop_heads_to_bodies.contains( predecessor->get_location_id() ) && - !loop_heads_ending.contains( predecessor->get_location_id() ) ) { - bool direction = predecessor->successor( true ).pointer->get_location_id() == - current->get_location_id(); - loop_heads_ending[ predecessor->get_location_id() ] = direction; - } - current = predecessor; - } - - for ( const auto& [ loop_head, loop_bodies ] : loop_heads_to_bodies ) { - INVARIANT( loop_heads_ending.contains( loop_head ) ); - bool loop_head_direction = loop_heads_ending[ loop_head ]; - - for ( const auto& body : loop_bodies ) { - if ( props.interesting_nodes.contains( { body, true } ) ) { - props.dependencies_by_loops[ { loop_head, loop_head_direction } ].insert( { body, true } ); - } - - if ( props.interesting_nodes.contains( { body, false } ) ) { - props.dependencies_by_loops[ { loop_head, loop_head_direction } ].insert( { body, false } ); - } - } - } - - props.cov_values_to_props[ node->best_coverage_value ].process_node( node ); - props.all_cov_value_props.process_node( node ); - props.all_paths.push_back( node ); - - - // if ( props.update_interesting_nodes( node ) ) { - // props.recompute_matrix(); - // } else { - // props.add_equation( node ); - // } -} - - -/** - * @brief Retrieves the path of node navigations from the given branching node to the root. - * - * This function constructs a vector of `node_direction` objects representing the path - * from the specified branching node to the root node. Each `node_direction` object - * contains the location ID of the predecessor node and the direction to the current node. - * - * @param node A pointer to the starting branching node. - * @return A vector of `node_direction` objects representing the path from the given node to the - * root. - */ -std::vector< fuzzing::node_direction > fuzzing::get_path( branching_node* node ) -{ - std::vector< node_direction > path; - - branching_node* current = node; - while ( current != nullptr ) { - branching_node* predecessor = current->predecessor; - if ( predecessor != nullptr ) { - node_direction nav = { predecessor->get_location_id(), predecessor->successor_direction( current ) }; - path.push_back( nav ); - } - current = predecessor; - } - - return path; -} - - -int fuzzing::linear_interpolation( int x1, int y1, int x2, int y2, int x ) -{ - if ( x1 == x2 ) { - return y1; - } - - // Perform linear interpolation - double slope = static_cast< double >( y2 - y1 ) / ( x2 - x1 ); - double y = y1 + slope * ( x - x1 ); - - // Round to nearest integer and return - return static_cast< int >( std::round( y ) ); -} \ No newline at end of file From b9870f4b7ab07777b6360f566a545bc75dc7c632 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 10 May 2025 18:22:52 +0200 Subject: [PATCH 141/144] feat: rename and remove benchmarks --- benchmarks/iid_testing/ai_generated_04.c | 47 -------------- benchmarks/iid_testing/ai_generated_11.c | 43 ------------ benchmarks/iid_testing/ai_generated_16.json | 65 +++++++++++++++++++ benchmarks/iid_testing/ai_generated_18.c | 44 ------------- benchmarks/iid_testing/custom_struct.c | 40 ++++++++++++ ...i_generated_04.json => custom_struct.json} | 0 .../{ai_generated_05.c => digit_range.c} | 12 +--- ...{ai_generated_05.json => digit_range.json} | 0 ...n_case_35_char_pointer_loop_count_P_eq_5.c | 45 ------------- ...ount_pos_gt_20.c => greater_in_increase.c} | 0 ...os_gt_20.json => greater_in_increase.json} | 0 .../{ai_generated_21.c => linked_list.c} | 22 +++---- ...{ai_generated_21.json => linked_list.json} | 0 benchmarks/iid_testing/nested_modulo.c | 37 +++++++++++ ...i_generated_18.json => nested_modulo.json} | 0 benchmarks/iid_testing/pointer.c | 40 ++++++++++++ ...er_loop_count_P_eq_5.json => pointer.json} | 0 .../{ai_generated_06.c => two_loops_used.c} | 15 ++--- ..._generated_06.json => two_loops_used.json} | 0 .../iid_testing/two_variables_compared.c | 36 ++++++++++ ...ed_11.json => two_variables_compared.json} | 0 21 files changed, 233 insertions(+), 213 deletions(-) delete mode 100644 benchmarks/iid_testing/ai_generated_04.c delete mode 100644 benchmarks/iid_testing/ai_generated_11.c create mode 100644 benchmarks/iid_testing/ai_generated_16.json delete mode 100644 benchmarks/iid_testing/ai_generated_18.c create mode 100644 benchmarks/iid_testing/custom_struct.c rename benchmarks/iid_testing/{ai_generated_04.json => custom_struct.json} (100%) rename benchmarks/iid_testing/{ai_generated_05.c => digit_range.c} (51%) rename benchmarks/iid_testing/{ai_generated_05.json => digit_range.json} (100%) delete mode 100644 benchmarks/iid_testing/gen_case_35_char_pointer_loop_count_P_eq_5.c rename benchmarks/iid_testing/{gen_case_18_int_count_pos_gt_20.c => greater_in_increase.c} (100%) rename benchmarks/iid_testing/{gen_case_18_int_count_pos_gt_20.json => greater_in_increase.json} (100%) rename benchmarks/iid_testing/{ai_generated_21.c => linked_list.c} (57%) rename benchmarks/iid_testing/{ai_generated_21.json => linked_list.json} (100%) create mode 100644 benchmarks/iid_testing/nested_modulo.c rename benchmarks/iid_testing/{ai_generated_18.json => nested_modulo.json} (100%) create mode 100644 benchmarks/iid_testing/pointer.c rename benchmarks/iid_testing/{gen_case_35_char_pointer_loop_count_P_eq_5.json => pointer.json} (100%) rename benchmarks/iid_testing/{ai_generated_06.c => two_loops_used.c} (53%) rename benchmarks/iid_testing/{ai_generated_06.json => two_loops_used.json} (100%) create mode 100644 benchmarks/iid_testing/two_variables_compared.c rename benchmarks/iid_testing/{ai_generated_11.json => two_variables_compared.json} (100%) diff --git a/benchmarks/iid_testing/ai_generated_04.c b/benchmarks/iid_testing/ai_generated_04.c deleted file mode 100644 index 52a36171..00000000 --- a/benchmarks/iid_testing/ai_generated_04.c +++ /dev/null @@ -1,47 +0,0 @@ -#include - -// Assume these functions are provided externally -extern short __VERIFIER_nondet_short(); -extern char __VERIFIER_nondet_char(); -extern int __VERIFIER_nondet_int(); - -// Simple struct definition -typedef struct { - int id; - char status; // e.g., 'A'ctive, 'I'nactive -} Item; - -#define MAX_ITEMS 40 - -int main() -{ - Item inventory[ MAX_ITEMS ]; // Array of structs - short num_items; - int active_count = 0; // State variable - - // 1. Fill data structure (array of structs) in an initial loop - num_items = __VERIFIER_nondet_short(); - if ( num_items < 0 || num_items > MAX_ITEMS ) { - return -1; - } - for ( short i = 0; i < num_items; ++i ) { - inventory[ i ].id = __VERIFIER_nondet_int(); // Assign some ID - inventory[ i ].status = __VERIFIER_nondet_char(); // Assign status - } - - // 2. Core logic in loop iterating through the array of structs - for ( short i = 0; i < num_items; ++i ) { - // 3. Modify internal state based on struct member - // 5. Change is constant (+1) - if ( inventory[ i ].status == 'A' ) { - active_count++; // Constant change - } - } // Loop always completes - - // 4. Final condition based on internal variable - if ( active_count >= 10 ) { // Check if at least half are active - return 1; - } else { - return 0; - } -} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_11.c b/benchmarks/iid_testing/ai_generated_11.c deleted file mode 100644 index 03508ced..00000000 --- a/benchmarks/iid_testing/ai_generated_11.c +++ /dev/null @@ -1,43 +0,0 @@ -#include - -// Assume these functions are provided externally -extern short __VERIFIER_nondet_short(); -extern char __VERIFIER_nondet_char(); - -#define MAX_SIZE 100 - -int main() -{ - char data[ MAX_SIZE ]; - short size; - int countA = 0; // First state variable - int countB = 0; // Second state variable - - // 1. Fill data structure in a loop - size = __VERIFIER_nondet_short(); - if ( size <= 0 || size > MAX_SIZE ) { - return -1; - } - for ( short i = 0; i < size; ++i ) { - data[ i ] = __VERIFIER_nondet_char(); - } - - // 2. Core logic in loop - for ( short i = 0; i < size; ++i ) { - // 3. Modify internal state variables based on data/condition - // 6. Changes are static (+1) - if ( data[ i ] == 'A' ) { - countA++; // Static change - } else if ( data[ i ] == 'B' ) { - countB++; // Static change - } - } - - // 4. Final condition based on multiple internal variables - // 5. Condition check is linear (countA - countB > 5) - if ( countA > countB + 5 ) { // Check relation between the two counters - return 1; - } else { - return 0; - } -} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_16.json b/benchmarks/iid_testing/ai_generated_16.json new file mode 100644 index 00000000..d3a7a120 --- /dev/null +++ b/benchmarks/iid_testing/ai_generated_16.json @@ -0,0 +1,65 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000, + "m32": false + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 9340, + "num_covered_branchings": 9, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 8, + 0, + 9, + 0 + ], + "output_statistics": { + "bitshare_analysis": { + "num_generated_tests": 3, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "sensitivity_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 2, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_18.c b/benchmarks/iid_testing/ai_generated_18.c deleted file mode 100644 index 2f2d2c3b..00000000 --- a/benchmarks/iid_testing/ai_generated_18.c +++ /dev/null @@ -1,44 +0,0 @@ -#include - -// Assume these functions are provided externally -extern short __VERIFIER_nondet_short(); -extern char __VERIFIER_nondet_char(); - -#define MAX_SIZE 80 - -int main() -{ - char data[ MAX_SIZE ]; - short size; - int counter = 0; // State variable - - // 1. Fill data structure in a loop - size = __VERIFIER_nondet_short(); - if ( size <= 0 || size > MAX_SIZE ) { - return -1; - } - for ( short i = 0; i < size; ++i ) { - data[ i ] = __VERIFIER_nondet_char(); - } - - // 2. Core logic in loop - for ( short i = 0; i < size; ++i ) { - // 3. Modify internal state based on data and current state - // 6. Change is static (+1 or +2) - if ( data[ i ] == 'X' ) { - if ( counter % 2 == 0 ) { // Check if counter is even - counter += 1; // Static change +1 - } else { // Counter is odd - counter += 2; // Static change +2 - } - } - } - - // 4. Final condition based on internal variable - // 5. Condition check is linear - if ( counter > 50 ) { - return 1; - } else { - return 0; - } -} \ No newline at end of file diff --git a/benchmarks/iid_testing/custom_struct.c b/benchmarks/iid_testing/custom_struct.c new file mode 100644 index 00000000..11676e9c --- /dev/null +++ b/benchmarks/iid_testing/custom_struct.c @@ -0,0 +1,40 @@ +#include + +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +typedef struct { + int id; + char status; +} Item; + +#define MAX_ITEMS 40 + +int main() +{ + Item inventory[ MAX_ITEMS ]; + short num_items; + int active_count = 0; + + num_items = __VERIFIER_nondet_short(); + if ( num_items < 0 || num_items > MAX_ITEMS ) { + return -1; + } + for ( short i = 0; i < num_items; ++i ) { + inventory[ i ].id = __VERIFIER_nondet_int(); + inventory[ i ].status = __VERIFIER_nondet_char(); + } + + for ( short i = 0; i < num_items; ++i ) { + if ( inventory[ i ].status == 'A' ) { + active_count++; + } + } + + if ( active_count >= 10 ) { + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_04.json b/benchmarks/iid_testing/custom_struct.json similarity index 100% rename from benchmarks/iid_testing/ai_generated_04.json rename to benchmarks/iid_testing/custom_struct.json diff --git a/benchmarks/iid_testing/ai_generated_05.c b/benchmarks/iid_testing/digit_range.c similarity index 51% rename from benchmarks/iid_testing/ai_generated_05.c rename to benchmarks/iid_testing/digit_range.c index 13a00c97..cac65be4 100644 --- a/benchmarks/iid_testing/ai_generated_05.c +++ b/benchmarks/iid_testing/digit_range.c @@ -1,6 +1,5 @@ #include -// Assume these functions are provided externally extern short __VERIFIER_nondet_short(); extern char __VERIFIER_nondet_char(); @@ -11,7 +10,6 @@ int main() char data[ MAX_SIZE ]; short size; - // 1. Fill data structure in a loop size = __VERIFIER_nondet_short(); if ( size <= 0 || size > MAX_SIZE ) { return -1; @@ -20,19 +18,13 @@ int main() data[ i ] = __VERIFIER_nondet_char(); } - // 2. Core logic in loop int special_count = 0; for ( short i = 0; i < size; ++i ) { - // 3. Modify internal state based on data/condition - // Condition involves two checks on the same element - // 6. Change is static (+1) - if ( data[ i ] > '5' && data[ i ] < '9' ) { // Check if digit is 6, 7, or 8 - special_count++; // Static change + if ( data[ i ] > '5' && data[ i ] < '9' ) { + special_count++; } } - // 4. Final condition based on internal variable - // 5. Condition check is linear if ( special_count == 3 ) { return 1; } else { diff --git a/benchmarks/iid_testing/ai_generated_05.json b/benchmarks/iid_testing/digit_range.json similarity index 100% rename from benchmarks/iid_testing/ai_generated_05.json rename to benchmarks/iid_testing/digit_range.json diff --git a/benchmarks/iid_testing/gen_case_35_char_pointer_loop_count_P_eq_5.c b/benchmarks/iid_testing/gen_case_35_char_pointer_loop_count_P_eq_5.c deleted file mode 100644 index d6c170c1..00000000 --- a/benchmarks/iid_testing/gen_case_35_char_pointer_loop_count_P_eq_5.c +++ /dev/null @@ -1,45 +0,0 @@ -/* File: gen_case_35_char_pointer_loop_count_P_eq_5.c */ -#include -#include // For NULL if needed - -// Assume these functions are provided externally -extern short __VERIFIER_nondet_short(); -extern char __VERIFIER_nondet_char(); -extern int __VERIFIER_nondet_int(); - -#define MAX_SIZE 60 - -int main() -{ - char data[MAX_SIZE]; - short size; - int p_count = 0; // State variable(s) - - // 1. Fill data structure in an initial loop - size = __VERIFIER_nondet_short(); - if (size <= 0 || size > MAX_SIZE) { - return -1; // Invalid size - } - - for (short i = 0; i < size; ++i) { - data[i] = __VERIFIER_nondet_char(); - } - - // 2. Core logic in loop(s) - - char* ptr = data; - char* end_ptr = data + size; // Pointer to one past the last element - - while (ptr < end_ptr) { - if (*ptr == 'P') { - p_count++; // Static change - } - ptr++; // Move pointer to the next element - } - // 4. Final condition based on internal variable(s) - if (p_count == 5) { - return 1; // Condition met - } else { - return 0; // Condition not met - } -} diff --git a/benchmarks/iid_testing/gen_case_18_int_count_pos_gt_20.c b/benchmarks/iid_testing/greater_in_increase.c similarity index 100% rename from benchmarks/iid_testing/gen_case_18_int_count_pos_gt_20.c rename to benchmarks/iid_testing/greater_in_increase.c diff --git a/benchmarks/iid_testing/gen_case_18_int_count_pos_gt_20.json b/benchmarks/iid_testing/greater_in_increase.json similarity index 100% rename from benchmarks/iid_testing/gen_case_18_int_count_pos_gt_20.json rename to benchmarks/iid_testing/greater_in_increase.json diff --git a/benchmarks/iid_testing/ai_generated_21.c b/benchmarks/iid_testing/linked_list.c similarity index 57% rename from benchmarks/iid_testing/ai_generated_21.c rename to benchmarks/iid_testing/linked_list.c index b56346dd..a0d59dac 100644 --- a/benchmarks/iid_testing/ai_generated_21.c +++ b/benchmarks/iid_testing/linked_list.c @@ -15,15 +15,14 @@ typedef struct Node { int main() { - Node nodes[ MAX_NODES ]; // Pre-allocate nodes for simplicity + Node nodes[ MAX_NODES ]; Node* head = NULL; Node* current = NULL; short size; - int count_gt_10 = 0; // State variable + int count_gt_10 = 0; - // 1. Fill data structure (linked list) in an initial loop size = __VERIFIER_nondet_short(); - if ( size < 0 || size > MAX_NODES ) { // Allow size 0 + if ( size < 0 || size > MAX_NODES ) { return -1; } @@ -39,19 +38,16 @@ int main() } } - // 2. Core logic in loop iterating through the data structure (linked list) - current = head; // Start iteration from head + current = head; while ( current != NULL ) { - // 3. Modify internal state based on data elements - // 5. Change is constant (+1) if ( current->value == 10 ) { - count_gt_10++; // Constant change + count_gt_10++; } - current = current->next; // Move to next node - } // Loop completes when end of list is reached - // 4. Final condition based on internal variable - if ( count_gt_10 >= 10 ) { // Non-trivial check + current = current->next; + } + + if ( count_gt_10 >= 10 ) { return 1; } else { return 0; diff --git a/benchmarks/iid_testing/ai_generated_21.json b/benchmarks/iid_testing/linked_list.json similarity index 100% rename from benchmarks/iid_testing/ai_generated_21.json rename to benchmarks/iid_testing/linked_list.json diff --git a/benchmarks/iid_testing/nested_modulo.c b/benchmarks/iid_testing/nested_modulo.c new file mode 100644 index 00000000..02494ed1 --- /dev/null +++ b/benchmarks/iid_testing/nested_modulo.c @@ -0,0 +1,37 @@ +#include + +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 80 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + int counter = 0; + + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + for ( short i = 0; i < size; ++i ) { + if ( data[ i ] == 'X' ) { + if ( counter % 2 == 0 ) { + counter += 1; + } else { + counter += 2; + } + } + } + + if ( counter > 50 ) { + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_18.json b/benchmarks/iid_testing/nested_modulo.json similarity index 100% rename from benchmarks/iid_testing/ai_generated_18.json rename to benchmarks/iid_testing/nested_modulo.json diff --git a/benchmarks/iid_testing/pointer.c b/benchmarks/iid_testing/pointer.c new file mode 100644 index 00000000..ce76348f --- /dev/null +++ b/benchmarks/iid_testing/pointer.c @@ -0,0 +1,40 @@ +#include +#include + +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); +extern int __VERIFIER_nondet_int(); + +#define MAX_SIZE 60 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + int p_count = 0; + + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + + char* ptr = data; + char* end_ptr = data + size; + + while ( ptr < end_ptr ) { + if ( *ptr == 'P' ) { + p_count++; + } + ptr++; + } + if ( p_count == 5 ) { + return 1; + } else { + return 0; + } +} diff --git a/benchmarks/iid_testing/gen_case_35_char_pointer_loop_count_P_eq_5.json b/benchmarks/iid_testing/pointer.json similarity index 100% rename from benchmarks/iid_testing/gen_case_35_char_pointer_loop_count_P_eq_5.json rename to benchmarks/iid_testing/pointer.json diff --git a/benchmarks/iid_testing/ai_generated_06.c b/benchmarks/iid_testing/two_loops_used.c similarity index 53% rename from benchmarks/iid_testing/ai_generated_06.c rename to benchmarks/iid_testing/two_loops_used.c index aed7d04b..f4d4fc98 100644 --- a/benchmarks/iid_testing/ai_generated_06.c +++ b/benchmarks/iid_testing/two_loops_used.c @@ -9,12 +9,10 @@ extern char __VERIFIER_nondet_char(); int main() { char data1[ MAX_SIZE ]; - char data2[ MAX_SIZE ]; // Second array + char data2[ MAX_SIZE ]; short size; - int match_count = 0; // State variable + int match_count = 0; - // 1. Fill data structures in loops - size = __VERIFIER_nondet_short(); if ( size <= 0 || size > MAX_SIZE ) { return -1; } @@ -23,18 +21,13 @@ int main() data2[ i ] = __VERIFIER_nondet_char(); } - // 2. Core logic in loop iterating through data for ( short i = 0; i < size; ++i ) { - // 3. Modify internal state based on data from both arrays - // 6. Change is static (+1) if ( data1[ i ] == 'S' && data2[ i ] == 'T' ) { - match_count++; // Static change + match_count++; } } - // 4. Final condition based on internal variable - // 5. Condition check is linear - if ( match_count > 10 ) { // Check if the S/T pair occurred more than once + if ( match_count > 10 ) { return 1; } else { return 0; diff --git a/benchmarks/iid_testing/ai_generated_06.json b/benchmarks/iid_testing/two_loops_used.json similarity index 100% rename from benchmarks/iid_testing/ai_generated_06.json rename to benchmarks/iid_testing/two_loops_used.json diff --git a/benchmarks/iid_testing/two_variables_compared.c b/benchmarks/iid_testing/two_variables_compared.c new file mode 100644 index 00000000..b8b85d56 --- /dev/null +++ b/benchmarks/iid_testing/two_variables_compared.c @@ -0,0 +1,36 @@ +#include + +extern short __VERIFIER_nondet_short(); +extern char __VERIFIER_nondet_char(); + +#define MAX_SIZE 100 + +int main() +{ + char data[ MAX_SIZE ]; + short size; + int countA = 0; + int countB = 0; + + size = __VERIFIER_nondet_short(); + if ( size <= 0 || size > MAX_SIZE ) { + return -1; + } + for ( short i = 0; i < size; ++i ) { + data[ i ] = __VERIFIER_nondet_char(); + } + + for ( short i = 0; i < size; ++i ) { + if ( data[ i ] == 'A' ) { + countA++; + } else if ( data[ i ] == 'B' ) { + countB++; + } + } + + if ( countA > countB + 5 ) { + return 1; + } else { + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/ai_generated_11.json b/benchmarks/iid_testing/two_variables_compared.json similarity index 100% rename from benchmarks/iid_testing/ai_generated_11.json rename to benchmarks/iid_testing/two_variables_compared.json From 72ad3b8e848ba8a5c129832eb2843ad4bb793250 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 10 May 2025 18:52:06 +0200 Subject: [PATCH 142/144] feat: rename benchmarks for clear description --- ...rated_01.c => aigen_01_char_countX_gt15.c} | 0 ...01.json => aigen_01_char_countX_gt15.json} | 0 ...ed_02.c => aigen_02_char_balancePN_eq15.c} | 0 ...json => aigen_02_char_balancePN_eq15.json} | 0 ...d_03.c => aigen_03_char_sum_AB_val_eq25.c} | 0 ...son => aigen_03_char_sum_AB_val_eq25.json} | 0 ...igen_04_char_nonvowel_upper_decr_target.c} | 0 ...n_04_char_nonvowel_upper_decr_target.json} | 0 ...5_char_statemachine_XY_resetR_count_gt3.c} | 0 ...har_statemachine_XY_resetR_count_gt3.json} | 0 ...=> aigen_06_char_bounded_counter_UD_eq5.c} | 0 ...aigen_06_char_bounded_counter_UD_eq5.json} | 0 ...=> aigen_07_char_two_arrays_matchST_gt5.c} | 0 ...aigen_07_char_two_arrays_matchST_gt5.json} | 0 ...ed_17.c => aigen_08_char_ptr_countP_eq5.c} | 0 ...json => aigen_08_char_ptr_countP_eq5.json} | 0 ...> aigen_09_char_odd_ascii_ratio_gt_half.c} | 0 ...igen_09_char_odd_ascii_ratio_gt_half.json} | 0 ...ar_two_loops_countA_eq10_or_countB_eq10.c} | 0 ...two_loops_countA_eq10_or_countB_eq10.json} | 0 ... => aigen_11_char_continueS_countF_ge10.c} | 0 ... aigen_11_char_continueS_countF_ge10.json} | 0 ...aigen_12_int_array_pos_gt20_or_neg_gt20.c} | 0 ...en_12_int_array_pos_gt20_or_neg_gt20.json} | 0 ...rated_25.c => aigen_13_char_strlen_gt40.c} | 0 ...25.json => aigen_13_char_strlen_gt40.json} | 0 ...ed_26.c => aigen_14_struct_statusA_ge10.c} | 0 ...json => aigen_14_struct_statusA_ge10.json} | 0 ...nt_eq_20.c => aigen_15_char_countX_eq10.c} | 0 ...20.json => aigen_15_char_countX_eq10.json} | 0 ...nt_gt_15.c => aigen_16_char_countY_gt15.c} | 0 ...15.json => aigen_16_char_countY_gt15.json} | 0 ...eq_13.c => aigen_17_char_balanceAB_eq13.c} | 0 ...json => aigen_17_char_balanceAB_eq13.json} | 0 ..._3.c => aigen_18_char_digitrange678_eq3.c} | 0 ...n => aigen_18_char_digitrange678_eq3.json} | 0 ... aigen_19_char_continue_S_count_F_ge_10.c} | 0 ...gen_19_char_continue_S_count_F_ge_10.json} | 0 ... => aigen_20_char_break_Q_count_P_eq_12.c} | 0 ... aigen_20_char_break_Q_count_P_eq_12.json} | 0 ...aigen_21_char_multi_state_A_gt_B_plus_5.c} | 0 ...en_21_char_multi_state_A_gt_B_plus_5.json} | 0 ...10.c => aigen_22_char_countXYZ_sum_eq10.c} | 0 ...n => aigen_22_char_countXYZ_sum_eq10.json} | 0 ...=> aigen_23_char_two_loops_A_or_B_eq_10.c} | 0 ...aigen_23_char_two_loops_A_or_B_eq_10.json} | 0 ...n_24_char_two_loops_dependent_ABCD_eq_7.c} | 0 ...4_char_two_loops_dependent_ABCD_eq_7.json} | 0 ... => aigen_25_struct_item_status_A_ge_10.c} | 0 ... aigen_25_struct_item_status_A_ge_10.json} | 0 ...igen_26_struct_dataval_balance_PN_eq_20.c} | 0 ...n_26_struct_dataval_balance_PN_eq_20.json} | 0 ...en_27_char_count_X_nested_check_eq_8_11.c} | 0 ...27_char_count_X_nested_check_eq_8_11.json} | 0 ...k.c => aigen_28_char_countA_nested_eq11.c} | 0 ... => aigen_28_char_countA_nested_eq11.json} | 0 ..._30.c => aigen_29_char_countA_mul3_eq30.c} | 0 ...on => aigen_29_char_countA_mul3_eq30.json} | 0 ...aigen_30_char_state_dependent_inc_gt_50.c} | 0 ...en_30_char_state_dependent_inc_gt_50.json} | 0 ... aigen_31_two_char_arrays_match_ST_gt_5.c} | 0 ...gen_31_two_char_arrays_match_ST_gt_5.json} | 0 ...n_32_char_state_machine_XYZ_count2_gt_3.c} | 0 ...2_char_state_machine_XYZ_count2_gt_3.json} | 0 ...igen_33_char_input_cycle_nested_A_eq_10.c} | 0 ...n_33_char_input_cycle_nested_A_eq_10.json} | 0 ..._lt_10.c => aigen_34_char_count_K_lt_10.c} | 0 ....json => aigen_34_char_count_K_lt_10.json} | 0 ... aigen_35_char_balance_Uinc2_Rdec1_gt10.c} | 0 ...gen_35_char_balance_Uinc2_Rdec1_gt10.json} | 0 ..._eq_12.c => aigen_36_char_count_H_eq_12.c} | 0 ....json => aigen_36_char_count_H_eq_12.json} | 0 ....c => aigen_37_char_balance_Z1_A1_gt_15.c} | 0 ...=> aigen_37_char_balance_Z1_A1_gt_15.json} | 0 ..._gt_12.c => aigen_38_char_count_R_gt_12.c} | 0 ....json => aigen_38_char_count_R_gt_12.json} | 0 ...aigen_39_char_multi_state_R_gt_P_plus_2.c} | 0 ...en_39_char_multi_state_R_gt_P_plus_2.json} | 0 ...> aigen_40_char_two_loops_balancePQ_ge7.c} | 0 ...igen_40_char_two_loops_balancePQ_ge7.json} | 0 ..._S_ge_5.c => aigen_41_char_count_S_ge_5.c} | 0 ...5.json => aigen_41_char_count_S_ge_5.json} | 0 ...ent_on_b.c => char_array_balanceAB_eq10.c} | 0 ..._b.json => char_array_balanceAB_eq10.json} | 0 ...\275\320\276\320\263\320\276_chars_eq10.c" | 0 ...eeded.c => char_array_balance_A2_B1_eq5.c} | 0 ...json => char_array_balance_A2_B1_eq5.json} | 0 ...5\320\276\320\263\320\276_chars_eq10.json" | 0 ... char_array_conditional_AB_balance_eq10.c} | 0 ...ar_array_conditional_AB_balance_eq10.json} | 0 ...> char_array_conditional_balanceAB_eq10.c} | 0 ...har_array_conditional_balanceAB_eq10.json} | 0 ...> char_array_countA_deep_nested_complex.c} | 0 ...har_array_countA_deep_nested_complex.json} | 0 .../{iid.c => char_array_countA_eq10.c} | 0 .../{iid.json => char_array_countA_eq10.json} | 0 ...har_array_countA_eq10_OR_countB_eq10_v2.c} | 0 ..._array_countA_eq10_OR_countB_eq10_v2.json} | 0 ....c => char_array_countA_eq8_or_10_or_11.c} | 0 ...=> char_array_countA_eq8_or_10_or_11.json} | 0 ...e_predicate.c => char_array_countA_gt10.c} | 0 ...icate.json => char_array_countA_gt10.json} | 0 ...in_iid.c => char_array_countA_mul4_eq20.c} | 0 ....json => char_array_countA_mul4_eq20.json} | 0 ... => char_array_countA_nested_check_eq11.c} | 0 ... char_array_countA_nested_check_eq11.json} | 0 ....c => char_array_countA_vs_countB_plus5.c} | 0 ...=> char_array_countA_vs_countB_plus5.json} | 0 ... => char_array_doublenested_countA_eq10.c} | 0 ... char_array_doublenested_countA_eq10.json} | 0 ...d.c => char_array_dummyload_countA_eq10.c} | 0 ... => char_array_dummyload_countA_eq10.json} | 0 ...else.c => char_array_ifelse_balance_eq5.c} | 0 ...son => char_array_ifelse_balance_eq5.json} | 0 ...ons.c => char_array_indexed_charA_check.c} | 0 ...on => char_array_indexed_charA_check.json} | 0 ...ble_loop.c => char_array_kAB_lXY_k_eq10.c} | 0 ...op.json => char_array_kAB_lXY_k_eq10.json} | 0 ...ndent.c => char_array_kAB_lXY_k_eq10_v2.c} | 0 ...json => char_array_kAB_lXY_k_eq10_v2.json} | 0 ...me.c => char_array_load2by2_countA_eq10.c} | 0 ...n => char_array_load2by2_countA_eq10.json} | 0 ...d.c => char_array_multi_predicate_loops.c} | 0 ... => char_array_multi_predicate_loops.json} | 0 ....c => char_array_multibreak_countA_eq10.c} | 0 ...=> char_array_multibreak_countA_eq10.json} | 0 ... char_array_multinested_AB_balance_eq10.c} | 0 ...ar_array_multinested_AB_balance_eq10.json} | 0 ...> char_array_nested_loop_balanceAB_eq10.c} | 0 ...har_array_nested_loop_balanceAB_eq10.json} | 0 ...c => char_array_nested_loop_countA_eq10.c} | 0 ...> char_array_nested_loop_countA_eq10.json} | 0 ...20.c => char_array_offset20_countA_eq10.c} | 0 ...n => char_array_offset20_countA_eq10.json} | 0 ...ion_in_loop.c => char_array_strlen_eq19.c} | 0 ..._loop.json => char_array_strlen_eq19.json} | 0 ...of_lopp.c => char_array_sum_chars_eq500.c} | 0 .../char_array_sum_chars_eq500.json | 55 +++++++++++++++++++ ... => char_array_two_pass_balanceABCD_eq7.c} | 0 ... char_array_two_pass_balanceABCD_eq7.json} | 0 ...c => char_array_two_pass_balanceAB_eq10.c} | 0 ...> char_array_two_pass_balanceAB_eq10.json} | 0 ..._used.c => char_two_arrays_matchST_gt10.c} | 0 ...json => char_two_arrays_matchST_gt10.json} | 0 144 files changed, 55 insertions(+) rename benchmarks/iid_testing/{ai_generated_01.c => aigen_01_char_countX_gt15.c} (100%) rename benchmarks/iid_testing/{ai_generated_01.json => aigen_01_char_countX_gt15.json} (100%) rename benchmarks/iid_testing/{ai_generated_02.c => aigen_02_char_balancePN_eq15.c} (100%) rename benchmarks/iid_testing/{ai_generated_02.json => aigen_02_char_balancePN_eq15.json} (100%) rename benchmarks/iid_testing/{ai_generated_03.c => aigen_03_char_sum_AB_val_eq25.c} (100%) rename benchmarks/iid_testing/{ai_generated_03.json => aigen_03_char_sum_AB_val_eq25.json} (100%) rename benchmarks/iid_testing/{ai_generated_07.c => aigen_04_char_nonvowel_upper_decr_target.c} (100%) rename benchmarks/iid_testing/{ai_generated_07.json => aigen_04_char_nonvowel_upper_decr_target.json} (100%) rename benchmarks/iid_testing/{ai_generated_12.c => aigen_05_char_statemachine_XY_resetR_count_gt3.c} (100%) rename benchmarks/iid_testing/{ai_generated_12.json => aigen_05_char_statemachine_XY_resetR_count_gt3.json} (100%) rename benchmarks/iid_testing/{ai_generated_14.c => aigen_06_char_bounded_counter_UD_eq5.c} (100%) rename benchmarks/iid_testing/{ai_generated_14.json => aigen_06_char_bounded_counter_UD_eq5.json} (100%) rename benchmarks/iid_testing/{ai_generated_16.c => aigen_07_char_two_arrays_matchST_gt5.c} (100%) rename benchmarks/iid_testing/{ai_generated_16.json => aigen_07_char_two_arrays_matchST_gt5.json} (100%) rename benchmarks/iid_testing/{ai_generated_17.c => aigen_08_char_ptr_countP_eq5.c} (100%) rename benchmarks/iid_testing/{ai_generated_17.json => aigen_08_char_ptr_countP_eq5.json} (100%) rename benchmarks/iid_testing/{ai_generated_20.c => aigen_09_char_odd_ascii_ratio_gt_half.c} (100%) rename benchmarks/iid_testing/{ai_generated_20.json => aigen_09_char_odd_ascii_ratio_gt_half.json} (100%) rename benchmarks/iid_testing/{ai_generated_22.c => aigen_10_char_two_loops_countA_eq10_or_countB_eq10.c} (100%) rename benchmarks/iid_testing/{ai_generated_22.json => aigen_10_char_two_loops_countA_eq10_or_countB_eq10.json} (100%) rename benchmarks/iid_testing/{ai_generated_23.c => aigen_11_char_continueS_countF_ge10.c} (100%) rename benchmarks/iid_testing/{ai_generated_23.json => aigen_11_char_continueS_countF_ge10.json} (100%) rename benchmarks/iid_testing/{ai_generated_24.c => aigen_12_int_array_pos_gt20_or_neg_gt20.c} (100%) rename benchmarks/iid_testing/{ai_generated_24.json => aigen_12_int_array_pos_gt20_or_neg_gt20.json} (100%) rename benchmarks/iid_testing/{ai_generated_25.c => aigen_13_char_strlen_gt40.c} (100%) rename benchmarks/iid_testing/{ai_generated_25.json => aigen_13_char_strlen_gt40.json} (100%) rename benchmarks/iid_testing/{ai_generated_26.c => aigen_14_struct_statusA_ge10.c} (100%) rename benchmarks/iid_testing/{ai_generated_26.json => aigen_14_struct_statusA_ge10.json} (100%) rename benchmarks/iid_testing/{gen_case_01_char_count_eq_20.c => aigen_15_char_countX_eq10.c} (100%) rename benchmarks/iid_testing/{gen_case_01_char_count_eq_20.json => aigen_15_char_countX_eq10.json} (100%) rename benchmarks/iid_testing/{gen_case_02_char_count_gt_15.c => aigen_16_char_countY_gt15.c} (100%) rename benchmarks/iid_testing/{gen_case_02_char_count_gt_15.json => aigen_16_char_countY_gt15.json} (100%) rename benchmarks/iid_testing/{gen_case_04_char_balance_AB_eq_13.c => aigen_17_char_balanceAB_eq13.c} (100%) rename benchmarks/iid_testing/{gen_case_04_char_balance_AB_eq_13.json => aigen_17_char_balanceAB_eq13.json} (100%) rename benchmarks/iid_testing/{gen_case_05_char_range_58_eq_3.c => aigen_18_char_digitrange678_eq3.c} (100%) rename benchmarks/iid_testing/{gen_case_05_char_range_58_eq_3.json => aigen_18_char_digitrange678_eq3.json} (100%) rename benchmarks/iid_testing/{gen_case_07_char_continue_S_count_F_ge_10.c => aigen_19_char_continue_S_count_F_ge_10.c} (100%) rename benchmarks/iid_testing/{gen_case_07_char_continue_S_count_F_ge_10.json => aigen_19_char_continue_S_count_F_ge_10.json} (100%) rename benchmarks/iid_testing/{gen_case_08_char_break_Q_count_P_eq_12.c => aigen_20_char_break_Q_count_P_eq_12.c} (100%) rename benchmarks/iid_testing/{gen_case_08_char_break_Q_count_P_eq_12.json => aigen_20_char_break_Q_count_P_eq_12.json} (100%) rename benchmarks/iid_testing/{gen_case_12_char_multi_state_A_gt_B_plus_5.c => aigen_21_char_multi_state_A_gt_B_plus_5.c} (100%) rename benchmarks/iid_testing/{gen_case_12_char_multi_state_A_gt_B_plus_5.json => aigen_21_char_multi_state_A_gt_B_plus_5.json} (100%) rename benchmarks/iid_testing/{gen_case_13_char_multi_state_XYZ_combo_eq_10.c => aigen_22_char_countXYZ_sum_eq10.c} (100%) rename benchmarks/iid_testing/{gen_case_13_char_multi_state_XYZ_combo_eq_10.json => aigen_22_char_countXYZ_sum_eq10.json} (100%) rename benchmarks/iid_testing/{gen_case_15_char_two_loops_A_or_B_eq_10.c => aigen_23_char_two_loops_A_or_B_eq_10.c} (100%) rename benchmarks/iid_testing/{gen_case_15_char_two_loops_A_or_B_eq_10.json => aigen_23_char_two_loops_A_or_B_eq_10.json} (100%) rename benchmarks/iid_testing/{gen_case_16_char_two_loops_dependent_ABCD_eq_7.c => aigen_24_char_two_loops_dependent_ABCD_eq_7.c} (100%) rename benchmarks/iid_testing/{gen_case_16_char_two_loops_dependent_ABCD_eq_7.json => aigen_24_char_two_loops_dependent_ABCD_eq_7.json} (100%) rename benchmarks/iid_testing/{gen_case_24_struct_item_status_A_ge_10.c => aigen_25_struct_item_status_A_ge_10.c} (100%) rename benchmarks/iid_testing/{gen_case_24_struct_item_status_A_ge_10.json => aigen_25_struct_item_status_A_ge_10.json} (100%) rename benchmarks/iid_testing/{gen_case_26_struct_dataval_balance_PN_eq_20.c => aigen_26_struct_dataval_balance_PN_eq_20.c} (100%) rename benchmarks/iid_testing/{gen_case_26_struct_dataval_balance_PN_eq_20.json => aigen_26_struct_dataval_balance_PN_eq_20.json} (100%) rename benchmarks/iid_testing/{gen_case_28_char_count_X_nested_check_eq_8_11.c => aigen_27_char_count_X_nested_check_eq_8_11.c} (100%) rename benchmarks/iid_testing/{gen_case_28_char_count_X_nested_check_eq_8_11.json => aigen_27_char_count_X_nested_check_eq_8_11.json} (100%) rename benchmarks/iid_testing/{gen_case_29_char_count_A_deep_nested_check.c => aigen_28_char_countA_nested_eq11.c} (100%) rename benchmarks/iid_testing/{gen_case_29_char_count_A_deep_nested_check.json => aigen_28_char_countA_nested_eq11.json} (100%) rename benchmarks/iid_testing/{gen_case_30_char_count_A_mult_check_eq_30.c => aigen_29_char_countA_mul3_eq30.c} (100%) rename benchmarks/iid_testing/{gen_case_30_char_count_A_mult_check_eq_30.json => aigen_29_char_countA_mul3_eq30.json} (100%) rename benchmarks/iid_testing/{gen_case_31_char_state_dependent_inc_gt_50.c => aigen_30_char_state_dependent_inc_gt_50.c} (100%) rename benchmarks/iid_testing/{gen_case_31_char_state_dependent_inc_gt_50.json => aigen_30_char_state_dependent_inc_gt_50.json} (100%) rename benchmarks/iid_testing/{gen_case_32_two_char_arrays_match_ST_gt_5.c => aigen_31_two_char_arrays_match_ST_gt_5.c} (100%) rename benchmarks/iid_testing/{gen_case_32_two_char_arrays_match_ST_gt_5.json => aigen_31_two_char_arrays_match_ST_gt_5.json} (100%) rename benchmarks/iid_testing/{gen_case_33_char_state_machine_XYZ_count2_gt_3.c => aigen_32_char_state_machine_XYZ_count2_gt_3.c} (100%) rename benchmarks/iid_testing/{gen_case_33_char_state_machine_XYZ_count2_gt_3.json => aigen_32_char_state_machine_XYZ_count2_gt_3.json} (100%) rename benchmarks/iid_testing/{gen_case_39_char_input_cycle_nested_A_eq_10.c => aigen_33_char_input_cycle_nested_A_eq_10.c} (100%) rename benchmarks/iid_testing/{gen_case_39_char_input_cycle_nested_A_eq_10.json => aigen_33_char_input_cycle_nested_A_eq_10.json} (100%) rename benchmarks/iid_testing/{gen_case_40_char_count_K_lt_10.c => aigen_34_char_count_K_lt_10.c} (100%) rename benchmarks/iid_testing/{gen_case_40_char_count_K_lt_10.json => aigen_34_char_count_K_lt_10.json} (100%) rename benchmarks/iid_testing/{gen_case_42_char_balance_U2_R1_gt_10.c => aigen_35_char_balance_Uinc2_Rdec1_gt10.c} (100%) rename benchmarks/iid_testing/{gen_case_42_char_balance_U2_R1_gt_10.json => aigen_35_char_balance_Uinc2_Rdec1_gt10.json} (100%) rename benchmarks/iid_testing/{gen_case_44_char_count_H_eq_12.c => aigen_36_char_count_H_eq_12.c} (100%) rename benchmarks/iid_testing/{gen_case_44_char_count_H_eq_12.json => aigen_36_char_count_H_eq_12.json} (100%) rename benchmarks/iid_testing/{gen_case_49_char_balance_Z1_A1_gt_15.c => aigen_37_char_balance_Z1_A1_gt_15.c} (100%) rename benchmarks/iid_testing/{gen_case_49_char_balance_Z1_A1_gt_15.json => aigen_37_char_balance_Z1_A1_gt_15.json} (100%) rename benchmarks/iid_testing/{gen_case_50_char_count_R_gt_12.c => aigen_38_char_count_R_gt_12.c} (100%) rename benchmarks/iid_testing/{gen_case_50_char_count_R_gt_12.json => aigen_38_char_count_R_gt_12.json} (100%) rename benchmarks/iid_testing/{gen_case_51_char_multi_state_R_gt_P_plus_2.c => aigen_39_char_multi_state_R_gt_P_plus_2.c} (100%) rename benchmarks/iid_testing/{gen_case_51_char_multi_state_R_gt_P_plus_2.json => aigen_39_char_multi_state_R_gt_P_plus_2.json} (100%) rename benchmarks/iid_testing/{gen_case_53_char_two_loops_dep_PQ_ge_7.c => aigen_40_char_two_loops_balancePQ_ge7.c} (100%) rename benchmarks/iid_testing/{gen_case_53_char_two_loops_dep_PQ_ge_7.json => aigen_40_char_two_loops_balancePQ_ge7.json} (100%) rename benchmarks/iid_testing/{gen_case_54_char_count_S_ge_5.c => aigen_41_char_count_S_ge_5.c} (100%) rename benchmarks/iid_testing/{gen_case_54_char_count_S_ge_5.json => aigen_41_char_count_S_ge_5.json} (100%) rename benchmarks/iid_testing/{decrement_on_b.c => char_array_balanceAB_eq10.c} (100%) rename benchmarks/iid_testing/{decrement_on_b.json => char_array_balanceAB_eq10.json} (100%) rename benchmarks/iid_testing/iid_multiple.c => "benchmarks/iid_testing/char_array_balance_ \320\274\320\275\320\276\320\263\320\276_chars_eq10.c" (100%) rename benchmarks/iid_testing/{combination_needed.c => char_array_balance_A2_B1_eq5.c} (100%) rename benchmarks/iid_testing/{combination_needed.json => char_array_balance_A2_B1_eq5.json} (100%) rename benchmarks/iid_testing/iid_multiple.json => "benchmarks/iid_testing/char_array_balance_\320\274\320\275\320\276\320\263\320\276_chars_eq10.json" (100%) rename benchmarks/iid_testing/{decrement_in_condition.c => char_array_conditional_AB_balance_eq10.c} (100%) rename benchmarks/iid_testing/{decrement_in_condition.json => char_array_conditional_AB_balance_eq10.json} (100%) rename benchmarks/iid_testing/{iid_in_condition.c => char_array_conditional_balanceAB_eq10.c} (100%) rename benchmarks/iid_testing/{iid_in_condition.json => char_array_conditional_balanceAB_eq10.json} (100%) rename benchmarks/iid_testing/{more_iid_condition_nested_big_values.c => char_array_countA_deep_nested_complex.c} (100%) rename benchmarks/iid_testing/{more_iid_condition_nested_big_values.json => char_array_countA_deep_nested_complex.json} (100%) rename benchmarks/iid_testing/{iid.c => char_array_countA_eq10.c} (100%) rename benchmarks/iid_testing/{iid.json => char_array_countA_eq10.json} (100%) rename benchmarks/iid_testing/{iid_twice.c => char_array_countA_eq10_OR_countB_eq10_v2.c} (100%) rename benchmarks/iid_testing/{iid_twice.json => char_array_countA_eq10_OR_countB_eq10_v2.json} (100%) rename benchmarks/iid_testing/{more_iid_condition.c => char_array_countA_eq8_or_10_or_11.c} (100%) rename benchmarks/iid_testing/{more_iid_condition.json => char_array_countA_eq8_or_10_or_11.json} (100%) rename benchmarks/iid_testing/{iid_reverse_predicate.c => char_array_countA_gt10.c} (100%) rename benchmarks/iid_testing/{iid_reverse_predicate.json => char_array_countA_gt10.json} (100%) rename benchmarks/iid_testing/{multiplication_in_iid.c => char_array_countA_mul4_eq20.c} (100%) rename benchmarks/iid_testing/{decrement_in_else.json => char_array_countA_mul4_eq20.json} (100%) rename benchmarks/iid_testing/{more_iid_condition_nested.c => char_array_countA_nested_check_eq11.c} (100%) rename benchmarks/iid_testing/{more_iid_condition_nested.json => char_array_countA_nested_check_eq11.json} (100%) rename benchmarks/iid_testing/{two_variables_compared.c => char_array_countA_vs_countB_plus5.c} (100%) rename benchmarks/iid_testing/{two_variables_compared.json => char_array_countA_vs_countB_plus5.json} (100%) rename benchmarks/iid_testing/{input_cycle_2_loops.c => char_array_doublenested_countA_eq10.c} (100%) rename benchmarks/iid_testing/{input_cycle_2_loops.json => char_array_doublenested_countA_eq10.json} (100%) rename benchmarks/iid_testing/{iid_with_dumy_load.c => char_array_dummyload_countA_eq10.c} (100%) rename benchmarks/iid_testing/{iid_with_dumy_load.json => char_array_dummyload_countA_eq10.json} (100%) rename benchmarks/iid_testing/{decrement_in_else.c => char_array_ifelse_balance_eq5.c} (100%) rename benchmarks/iid_testing/{multiplication_in_iid.json => char_array_ifelse_balance_eq5.json} (100%) rename benchmarks/iid_testing/{lots_of_non_iid_conditions.c => char_array_indexed_charA_check.c} (100%) rename benchmarks/iid_testing/{lots_of_non_iid_conditions.json => char_array_indexed_charA_check.json} (100%) rename benchmarks/iid_testing/{double_loop.c => char_array_kAB_lXY_k_eq10.c} (100%) rename benchmarks/iid_testing/{double_loop.json => char_array_kAB_lXY_k_eq10.json} (100%) rename benchmarks/iid_testing/{same_loop_independent.c => char_array_kAB_lXY_k_eq10_v2.c} (100%) rename benchmarks/iid_testing/{same_loop_independent.json => char_array_kAB_lXY_k_eq10_v2.json} (100%) rename benchmarks/iid_testing/{loading_loop_2_at_a_time.c => char_array_load2by2_countA_eq10.c} (100%) rename benchmarks/iid_testing/{loading_loop_2_at_a_time.json => char_array_load2by2_countA_eq10.json} (100%) rename benchmarks/iid_testing/{different_predicates_in_iid.c => char_array_multi_predicate_loops.c} (100%) rename benchmarks/iid_testing/{different_predicates_in_iid.json => char_array_multi_predicate_loops.json} (100%) rename benchmarks/iid_testing/{multiple_loop_heads.c => char_array_multibreak_countA_eq10.c} (100%) rename benchmarks/iid_testing/{multiple_loop_heads.json => char_array_multibreak_countA_eq10.json} (100%) rename benchmarks/iid_testing/{input_cycle_2_loops_combination.c => char_array_multinested_AB_balance_eq10.c} (100%) rename benchmarks/iid_testing/{input_cycle_2_loops_combination.json => char_array_multinested_AB_balance_eq10.json} (100%) rename benchmarks/iid_testing/{input_cycle_dec_b.c => char_array_nested_loop_balanceAB_eq10.c} (100%) rename benchmarks/iid_testing/{input_cycle_dec_b.json => char_array_nested_loop_balanceAB_eq10.json} (100%) rename benchmarks/iid_testing/{input_cycle.c => char_array_nested_loop_countA_eq10.c} (100%) rename benchmarks/iid_testing/{input_cycle.json => char_array_nested_loop_countA_eq10.json} (100%) rename benchmarks/iid_testing/{loading_loop_plus_20.c => char_array_offset20_countA_eq10.c} (100%) rename benchmarks/iid_testing/{loading_loop_plus_20.json => char_array_offset20_countA_eq10.json} (100%) rename benchmarks/iid_testing/{computation_in_loop.c => char_array_strlen_eq19.c} (100%) rename benchmarks/iid_testing/{computation_in_loop.json => char_array_strlen_eq19.json} (100%) rename benchmarks/iid_testing/{sum_of_lopp.c => char_array_sum_chars_eq500.c} (100%) create mode 100644 benchmarks/iid_testing/char_array_sum_chars_eq500.json rename benchmarks/iid_testing/{same_loop_dependent_2.c => char_array_two_pass_balanceABCD_eq7.c} (100%) rename benchmarks/iid_testing/{same_loop_dependent_2.json => char_array_two_pass_balanceABCD_eq7.json} (100%) rename benchmarks/iid_testing/{same_loop_dependent.c => char_array_two_pass_balanceAB_eq10.c} (100%) rename benchmarks/iid_testing/{same_loop_dependent.json => char_array_two_pass_balanceAB_eq10.json} (100%) rename benchmarks/iid_testing/{two_loops_used.c => char_two_arrays_matchST_gt10.c} (100%) rename benchmarks/iid_testing/{two_loops_used.json => char_two_arrays_matchST_gt10.json} (100%) diff --git a/benchmarks/iid_testing/ai_generated_01.c b/benchmarks/iid_testing/aigen_01_char_countX_gt15.c similarity index 100% rename from benchmarks/iid_testing/ai_generated_01.c rename to benchmarks/iid_testing/aigen_01_char_countX_gt15.c diff --git a/benchmarks/iid_testing/ai_generated_01.json b/benchmarks/iid_testing/aigen_01_char_countX_gt15.json similarity index 100% rename from benchmarks/iid_testing/ai_generated_01.json rename to benchmarks/iid_testing/aigen_01_char_countX_gt15.json diff --git a/benchmarks/iid_testing/ai_generated_02.c b/benchmarks/iid_testing/aigen_02_char_balancePN_eq15.c similarity index 100% rename from benchmarks/iid_testing/ai_generated_02.c rename to benchmarks/iid_testing/aigen_02_char_balancePN_eq15.c diff --git a/benchmarks/iid_testing/ai_generated_02.json b/benchmarks/iid_testing/aigen_02_char_balancePN_eq15.json similarity index 100% rename from benchmarks/iid_testing/ai_generated_02.json rename to benchmarks/iid_testing/aigen_02_char_balancePN_eq15.json diff --git a/benchmarks/iid_testing/ai_generated_03.c b/benchmarks/iid_testing/aigen_03_char_sum_AB_val_eq25.c similarity index 100% rename from benchmarks/iid_testing/ai_generated_03.c rename to benchmarks/iid_testing/aigen_03_char_sum_AB_val_eq25.c diff --git a/benchmarks/iid_testing/ai_generated_03.json b/benchmarks/iid_testing/aigen_03_char_sum_AB_val_eq25.json similarity index 100% rename from benchmarks/iid_testing/ai_generated_03.json rename to benchmarks/iid_testing/aigen_03_char_sum_AB_val_eq25.json diff --git a/benchmarks/iid_testing/ai_generated_07.c b/benchmarks/iid_testing/aigen_04_char_nonvowel_upper_decr_target.c similarity index 100% rename from benchmarks/iid_testing/ai_generated_07.c rename to benchmarks/iid_testing/aigen_04_char_nonvowel_upper_decr_target.c diff --git a/benchmarks/iid_testing/ai_generated_07.json b/benchmarks/iid_testing/aigen_04_char_nonvowel_upper_decr_target.json similarity index 100% rename from benchmarks/iid_testing/ai_generated_07.json rename to benchmarks/iid_testing/aigen_04_char_nonvowel_upper_decr_target.json diff --git a/benchmarks/iid_testing/ai_generated_12.c b/benchmarks/iid_testing/aigen_05_char_statemachine_XY_resetR_count_gt3.c similarity index 100% rename from benchmarks/iid_testing/ai_generated_12.c rename to benchmarks/iid_testing/aigen_05_char_statemachine_XY_resetR_count_gt3.c diff --git a/benchmarks/iid_testing/ai_generated_12.json b/benchmarks/iid_testing/aigen_05_char_statemachine_XY_resetR_count_gt3.json similarity index 100% rename from benchmarks/iid_testing/ai_generated_12.json rename to benchmarks/iid_testing/aigen_05_char_statemachine_XY_resetR_count_gt3.json diff --git a/benchmarks/iid_testing/ai_generated_14.c b/benchmarks/iid_testing/aigen_06_char_bounded_counter_UD_eq5.c similarity index 100% rename from benchmarks/iid_testing/ai_generated_14.c rename to benchmarks/iid_testing/aigen_06_char_bounded_counter_UD_eq5.c diff --git a/benchmarks/iid_testing/ai_generated_14.json b/benchmarks/iid_testing/aigen_06_char_bounded_counter_UD_eq5.json similarity index 100% rename from benchmarks/iid_testing/ai_generated_14.json rename to benchmarks/iid_testing/aigen_06_char_bounded_counter_UD_eq5.json diff --git a/benchmarks/iid_testing/ai_generated_16.c b/benchmarks/iid_testing/aigen_07_char_two_arrays_matchST_gt5.c similarity index 100% rename from benchmarks/iid_testing/ai_generated_16.c rename to benchmarks/iid_testing/aigen_07_char_two_arrays_matchST_gt5.c diff --git a/benchmarks/iid_testing/ai_generated_16.json b/benchmarks/iid_testing/aigen_07_char_two_arrays_matchST_gt5.json similarity index 100% rename from benchmarks/iid_testing/ai_generated_16.json rename to benchmarks/iid_testing/aigen_07_char_two_arrays_matchST_gt5.json diff --git a/benchmarks/iid_testing/ai_generated_17.c b/benchmarks/iid_testing/aigen_08_char_ptr_countP_eq5.c similarity index 100% rename from benchmarks/iid_testing/ai_generated_17.c rename to benchmarks/iid_testing/aigen_08_char_ptr_countP_eq5.c diff --git a/benchmarks/iid_testing/ai_generated_17.json b/benchmarks/iid_testing/aigen_08_char_ptr_countP_eq5.json similarity index 100% rename from benchmarks/iid_testing/ai_generated_17.json rename to benchmarks/iid_testing/aigen_08_char_ptr_countP_eq5.json diff --git a/benchmarks/iid_testing/ai_generated_20.c b/benchmarks/iid_testing/aigen_09_char_odd_ascii_ratio_gt_half.c similarity index 100% rename from benchmarks/iid_testing/ai_generated_20.c rename to benchmarks/iid_testing/aigen_09_char_odd_ascii_ratio_gt_half.c diff --git a/benchmarks/iid_testing/ai_generated_20.json b/benchmarks/iid_testing/aigen_09_char_odd_ascii_ratio_gt_half.json similarity index 100% rename from benchmarks/iid_testing/ai_generated_20.json rename to benchmarks/iid_testing/aigen_09_char_odd_ascii_ratio_gt_half.json diff --git a/benchmarks/iid_testing/ai_generated_22.c b/benchmarks/iid_testing/aigen_10_char_two_loops_countA_eq10_or_countB_eq10.c similarity index 100% rename from benchmarks/iid_testing/ai_generated_22.c rename to benchmarks/iid_testing/aigen_10_char_two_loops_countA_eq10_or_countB_eq10.c diff --git a/benchmarks/iid_testing/ai_generated_22.json b/benchmarks/iid_testing/aigen_10_char_two_loops_countA_eq10_or_countB_eq10.json similarity index 100% rename from benchmarks/iid_testing/ai_generated_22.json rename to benchmarks/iid_testing/aigen_10_char_two_loops_countA_eq10_or_countB_eq10.json diff --git a/benchmarks/iid_testing/ai_generated_23.c b/benchmarks/iid_testing/aigen_11_char_continueS_countF_ge10.c similarity index 100% rename from benchmarks/iid_testing/ai_generated_23.c rename to benchmarks/iid_testing/aigen_11_char_continueS_countF_ge10.c diff --git a/benchmarks/iid_testing/ai_generated_23.json b/benchmarks/iid_testing/aigen_11_char_continueS_countF_ge10.json similarity index 100% rename from benchmarks/iid_testing/ai_generated_23.json rename to benchmarks/iid_testing/aigen_11_char_continueS_countF_ge10.json diff --git a/benchmarks/iid_testing/ai_generated_24.c b/benchmarks/iid_testing/aigen_12_int_array_pos_gt20_or_neg_gt20.c similarity index 100% rename from benchmarks/iid_testing/ai_generated_24.c rename to benchmarks/iid_testing/aigen_12_int_array_pos_gt20_or_neg_gt20.c diff --git a/benchmarks/iid_testing/ai_generated_24.json b/benchmarks/iid_testing/aigen_12_int_array_pos_gt20_or_neg_gt20.json similarity index 100% rename from benchmarks/iid_testing/ai_generated_24.json rename to benchmarks/iid_testing/aigen_12_int_array_pos_gt20_or_neg_gt20.json diff --git a/benchmarks/iid_testing/ai_generated_25.c b/benchmarks/iid_testing/aigen_13_char_strlen_gt40.c similarity index 100% rename from benchmarks/iid_testing/ai_generated_25.c rename to benchmarks/iid_testing/aigen_13_char_strlen_gt40.c diff --git a/benchmarks/iid_testing/ai_generated_25.json b/benchmarks/iid_testing/aigen_13_char_strlen_gt40.json similarity index 100% rename from benchmarks/iid_testing/ai_generated_25.json rename to benchmarks/iid_testing/aigen_13_char_strlen_gt40.json diff --git a/benchmarks/iid_testing/ai_generated_26.c b/benchmarks/iid_testing/aigen_14_struct_statusA_ge10.c similarity index 100% rename from benchmarks/iid_testing/ai_generated_26.c rename to benchmarks/iid_testing/aigen_14_struct_statusA_ge10.c diff --git a/benchmarks/iid_testing/ai_generated_26.json b/benchmarks/iid_testing/aigen_14_struct_statusA_ge10.json similarity index 100% rename from benchmarks/iid_testing/ai_generated_26.json rename to benchmarks/iid_testing/aigen_14_struct_statusA_ge10.json diff --git a/benchmarks/iid_testing/gen_case_01_char_count_eq_20.c b/benchmarks/iid_testing/aigen_15_char_countX_eq10.c similarity index 100% rename from benchmarks/iid_testing/gen_case_01_char_count_eq_20.c rename to benchmarks/iid_testing/aigen_15_char_countX_eq10.c diff --git a/benchmarks/iid_testing/gen_case_01_char_count_eq_20.json b/benchmarks/iid_testing/aigen_15_char_countX_eq10.json similarity index 100% rename from benchmarks/iid_testing/gen_case_01_char_count_eq_20.json rename to benchmarks/iid_testing/aigen_15_char_countX_eq10.json diff --git a/benchmarks/iid_testing/gen_case_02_char_count_gt_15.c b/benchmarks/iid_testing/aigen_16_char_countY_gt15.c similarity index 100% rename from benchmarks/iid_testing/gen_case_02_char_count_gt_15.c rename to benchmarks/iid_testing/aigen_16_char_countY_gt15.c diff --git a/benchmarks/iid_testing/gen_case_02_char_count_gt_15.json b/benchmarks/iid_testing/aigen_16_char_countY_gt15.json similarity index 100% rename from benchmarks/iid_testing/gen_case_02_char_count_gt_15.json rename to benchmarks/iid_testing/aigen_16_char_countY_gt15.json diff --git a/benchmarks/iid_testing/gen_case_04_char_balance_AB_eq_13.c b/benchmarks/iid_testing/aigen_17_char_balanceAB_eq13.c similarity index 100% rename from benchmarks/iid_testing/gen_case_04_char_balance_AB_eq_13.c rename to benchmarks/iid_testing/aigen_17_char_balanceAB_eq13.c diff --git a/benchmarks/iid_testing/gen_case_04_char_balance_AB_eq_13.json b/benchmarks/iid_testing/aigen_17_char_balanceAB_eq13.json similarity index 100% rename from benchmarks/iid_testing/gen_case_04_char_balance_AB_eq_13.json rename to benchmarks/iid_testing/aigen_17_char_balanceAB_eq13.json diff --git a/benchmarks/iid_testing/gen_case_05_char_range_58_eq_3.c b/benchmarks/iid_testing/aigen_18_char_digitrange678_eq3.c similarity index 100% rename from benchmarks/iid_testing/gen_case_05_char_range_58_eq_3.c rename to benchmarks/iid_testing/aigen_18_char_digitrange678_eq3.c diff --git a/benchmarks/iid_testing/gen_case_05_char_range_58_eq_3.json b/benchmarks/iid_testing/aigen_18_char_digitrange678_eq3.json similarity index 100% rename from benchmarks/iid_testing/gen_case_05_char_range_58_eq_3.json rename to benchmarks/iid_testing/aigen_18_char_digitrange678_eq3.json diff --git a/benchmarks/iid_testing/gen_case_07_char_continue_S_count_F_ge_10.c b/benchmarks/iid_testing/aigen_19_char_continue_S_count_F_ge_10.c similarity index 100% rename from benchmarks/iid_testing/gen_case_07_char_continue_S_count_F_ge_10.c rename to benchmarks/iid_testing/aigen_19_char_continue_S_count_F_ge_10.c diff --git a/benchmarks/iid_testing/gen_case_07_char_continue_S_count_F_ge_10.json b/benchmarks/iid_testing/aigen_19_char_continue_S_count_F_ge_10.json similarity index 100% rename from benchmarks/iid_testing/gen_case_07_char_continue_S_count_F_ge_10.json rename to benchmarks/iid_testing/aigen_19_char_continue_S_count_F_ge_10.json diff --git a/benchmarks/iid_testing/gen_case_08_char_break_Q_count_P_eq_12.c b/benchmarks/iid_testing/aigen_20_char_break_Q_count_P_eq_12.c similarity index 100% rename from benchmarks/iid_testing/gen_case_08_char_break_Q_count_P_eq_12.c rename to benchmarks/iid_testing/aigen_20_char_break_Q_count_P_eq_12.c diff --git a/benchmarks/iid_testing/gen_case_08_char_break_Q_count_P_eq_12.json b/benchmarks/iid_testing/aigen_20_char_break_Q_count_P_eq_12.json similarity index 100% rename from benchmarks/iid_testing/gen_case_08_char_break_Q_count_P_eq_12.json rename to benchmarks/iid_testing/aigen_20_char_break_Q_count_P_eq_12.json diff --git a/benchmarks/iid_testing/gen_case_12_char_multi_state_A_gt_B_plus_5.c b/benchmarks/iid_testing/aigen_21_char_multi_state_A_gt_B_plus_5.c similarity index 100% rename from benchmarks/iid_testing/gen_case_12_char_multi_state_A_gt_B_plus_5.c rename to benchmarks/iid_testing/aigen_21_char_multi_state_A_gt_B_plus_5.c diff --git a/benchmarks/iid_testing/gen_case_12_char_multi_state_A_gt_B_plus_5.json b/benchmarks/iid_testing/aigen_21_char_multi_state_A_gt_B_plus_5.json similarity index 100% rename from benchmarks/iid_testing/gen_case_12_char_multi_state_A_gt_B_plus_5.json rename to benchmarks/iid_testing/aigen_21_char_multi_state_A_gt_B_plus_5.json diff --git a/benchmarks/iid_testing/gen_case_13_char_multi_state_XYZ_combo_eq_10.c b/benchmarks/iid_testing/aigen_22_char_countXYZ_sum_eq10.c similarity index 100% rename from benchmarks/iid_testing/gen_case_13_char_multi_state_XYZ_combo_eq_10.c rename to benchmarks/iid_testing/aigen_22_char_countXYZ_sum_eq10.c diff --git a/benchmarks/iid_testing/gen_case_13_char_multi_state_XYZ_combo_eq_10.json b/benchmarks/iid_testing/aigen_22_char_countXYZ_sum_eq10.json similarity index 100% rename from benchmarks/iid_testing/gen_case_13_char_multi_state_XYZ_combo_eq_10.json rename to benchmarks/iid_testing/aigen_22_char_countXYZ_sum_eq10.json diff --git a/benchmarks/iid_testing/gen_case_15_char_two_loops_A_or_B_eq_10.c b/benchmarks/iid_testing/aigen_23_char_two_loops_A_or_B_eq_10.c similarity index 100% rename from benchmarks/iid_testing/gen_case_15_char_two_loops_A_or_B_eq_10.c rename to benchmarks/iid_testing/aigen_23_char_two_loops_A_or_B_eq_10.c diff --git a/benchmarks/iid_testing/gen_case_15_char_two_loops_A_or_B_eq_10.json b/benchmarks/iid_testing/aigen_23_char_two_loops_A_or_B_eq_10.json similarity index 100% rename from benchmarks/iid_testing/gen_case_15_char_two_loops_A_or_B_eq_10.json rename to benchmarks/iid_testing/aigen_23_char_two_loops_A_or_B_eq_10.json diff --git a/benchmarks/iid_testing/gen_case_16_char_two_loops_dependent_ABCD_eq_7.c b/benchmarks/iid_testing/aigen_24_char_two_loops_dependent_ABCD_eq_7.c similarity index 100% rename from benchmarks/iid_testing/gen_case_16_char_two_loops_dependent_ABCD_eq_7.c rename to benchmarks/iid_testing/aigen_24_char_two_loops_dependent_ABCD_eq_7.c diff --git a/benchmarks/iid_testing/gen_case_16_char_two_loops_dependent_ABCD_eq_7.json b/benchmarks/iid_testing/aigen_24_char_two_loops_dependent_ABCD_eq_7.json similarity index 100% rename from benchmarks/iid_testing/gen_case_16_char_two_loops_dependent_ABCD_eq_7.json rename to benchmarks/iid_testing/aigen_24_char_two_loops_dependent_ABCD_eq_7.json diff --git a/benchmarks/iid_testing/gen_case_24_struct_item_status_A_ge_10.c b/benchmarks/iid_testing/aigen_25_struct_item_status_A_ge_10.c similarity index 100% rename from benchmarks/iid_testing/gen_case_24_struct_item_status_A_ge_10.c rename to benchmarks/iid_testing/aigen_25_struct_item_status_A_ge_10.c diff --git a/benchmarks/iid_testing/gen_case_24_struct_item_status_A_ge_10.json b/benchmarks/iid_testing/aigen_25_struct_item_status_A_ge_10.json similarity index 100% rename from benchmarks/iid_testing/gen_case_24_struct_item_status_A_ge_10.json rename to benchmarks/iid_testing/aigen_25_struct_item_status_A_ge_10.json diff --git a/benchmarks/iid_testing/gen_case_26_struct_dataval_balance_PN_eq_20.c b/benchmarks/iid_testing/aigen_26_struct_dataval_balance_PN_eq_20.c similarity index 100% rename from benchmarks/iid_testing/gen_case_26_struct_dataval_balance_PN_eq_20.c rename to benchmarks/iid_testing/aigen_26_struct_dataval_balance_PN_eq_20.c diff --git a/benchmarks/iid_testing/gen_case_26_struct_dataval_balance_PN_eq_20.json b/benchmarks/iid_testing/aigen_26_struct_dataval_balance_PN_eq_20.json similarity index 100% rename from benchmarks/iid_testing/gen_case_26_struct_dataval_balance_PN_eq_20.json rename to benchmarks/iid_testing/aigen_26_struct_dataval_balance_PN_eq_20.json diff --git a/benchmarks/iid_testing/gen_case_28_char_count_X_nested_check_eq_8_11.c b/benchmarks/iid_testing/aigen_27_char_count_X_nested_check_eq_8_11.c similarity index 100% rename from benchmarks/iid_testing/gen_case_28_char_count_X_nested_check_eq_8_11.c rename to benchmarks/iid_testing/aigen_27_char_count_X_nested_check_eq_8_11.c diff --git a/benchmarks/iid_testing/gen_case_28_char_count_X_nested_check_eq_8_11.json b/benchmarks/iid_testing/aigen_27_char_count_X_nested_check_eq_8_11.json similarity index 100% rename from benchmarks/iid_testing/gen_case_28_char_count_X_nested_check_eq_8_11.json rename to benchmarks/iid_testing/aigen_27_char_count_X_nested_check_eq_8_11.json diff --git a/benchmarks/iid_testing/gen_case_29_char_count_A_deep_nested_check.c b/benchmarks/iid_testing/aigen_28_char_countA_nested_eq11.c similarity index 100% rename from benchmarks/iid_testing/gen_case_29_char_count_A_deep_nested_check.c rename to benchmarks/iid_testing/aigen_28_char_countA_nested_eq11.c diff --git a/benchmarks/iid_testing/gen_case_29_char_count_A_deep_nested_check.json b/benchmarks/iid_testing/aigen_28_char_countA_nested_eq11.json similarity index 100% rename from benchmarks/iid_testing/gen_case_29_char_count_A_deep_nested_check.json rename to benchmarks/iid_testing/aigen_28_char_countA_nested_eq11.json diff --git a/benchmarks/iid_testing/gen_case_30_char_count_A_mult_check_eq_30.c b/benchmarks/iid_testing/aigen_29_char_countA_mul3_eq30.c similarity index 100% rename from benchmarks/iid_testing/gen_case_30_char_count_A_mult_check_eq_30.c rename to benchmarks/iid_testing/aigen_29_char_countA_mul3_eq30.c diff --git a/benchmarks/iid_testing/gen_case_30_char_count_A_mult_check_eq_30.json b/benchmarks/iid_testing/aigen_29_char_countA_mul3_eq30.json similarity index 100% rename from benchmarks/iid_testing/gen_case_30_char_count_A_mult_check_eq_30.json rename to benchmarks/iid_testing/aigen_29_char_countA_mul3_eq30.json diff --git a/benchmarks/iid_testing/gen_case_31_char_state_dependent_inc_gt_50.c b/benchmarks/iid_testing/aigen_30_char_state_dependent_inc_gt_50.c similarity index 100% rename from benchmarks/iid_testing/gen_case_31_char_state_dependent_inc_gt_50.c rename to benchmarks/iid_testing/aigen_30_char_state_dependent_inc_gt_50.c diff --git a/benchmarks/iid_testing/gen_case_31_char_state_dependent_inc_gt_50.json b/benchmarks/iid_testing/aigen_30_char_state_dependent_inc_gt_50.json similarity index 100% rename from benchmarks/iid_testing/gen_case_31_char_state_dependent_inc_gt_50.json rename to benchmarks/iid_testing/aigen_30_char_state_dependent_inc_gt_50.json diff --git a/benchmarks/iid_testing/gen_case_32_two_char_arrays_match_ST_gt_5.c b/benchmarks/iid_testing/aigen_31_two_char_arrays_match_ST_gt_5.c similarity index 100% rename from benchmarks/iid_testing/gen_case_32_two_char_arrays_match_ST_gt_5.c rename to benchmarks/iid_testing/aigen_31_two_char_arrays_match_ST_gt_5.c diff --git a/benchmarks/iid_testing/gen_case_32_two_char_arrays_match_ST_gt_5.json b/benchmarks/iid_testing/aigen_31_two_char_arrays_match_ST_gt_5.json similarity index 100% rename from benchmarks/iid_testing/gen_case_32_two_char_arrays_match_ST_gt_5.json rename to benchmarks/iid_testing/aigen_31_two_char_arrays_match_ST_gt_5.json diff --git a/benchmarks/iid_testing/gen_case_33_char_state_machine_XYZ_count2_gt_3.c b/benchmarks/iid_testing/aigen_32_char_state_machine_XYZ_count2_gt_3.c similarity index 100% rename from benchmarks/iid_testing/gen_case_33_char_state_machine_XYZ_count2_gt_3.c rename to benchmarks/iid_testing/aigen_32_char_state_machine_XYZ_count2_gt_3.c diff --git a/benchmarks/iid_testing/gen_case_33_char_state_machine_XYZ_count2_gt_3.json b/benchmarks/iid_testing/aigen_32_char_state_machine_XYZ_count2_gt_3.json similarity index 100% rename from benchmarks/iid_testing/gen_case_33_char_state_machine_XYZ_count2_gt_3.json rename to benchmarks/iid_testing/aigen_32_char_state_machine_XYZ_count2_gt_3.json diff --git a/benchmarks/iid_testing/gen_case_39_char_input_cycle_nested_A_eq_10.c b/benchmarks/iid_testing/aigen_33_char_input_cycle_nested_A_eq_10.c similarity index 100% rename from benchmarks/iid_testing/gen_case_39_char_input_cycle_nested_A_eq_10.c rename to benchmarks/iid_testing/aigen_33_char_input_cycle_nested_A_eq_10.c diff --git a/benchmarks/iid_testing/gen_case_39_char_input_cycle_nested_A_eq_10.json b/benchmarks/iid_testing/aigen_33_char_input_cycle_nested_A_eq_10.json similarity index 100% rename from benchmarks/iid_testing/gen_case_39_char_input_cycle_nested_A_eq_10.json rename to benchmarks/iid_testing/aigen_33_char_input_cycle_nested_A_eq_10.json diff --git a/benchmarks/iid_testing/gen_case_40_char_count_K_lt_10.c b/benchmarks/iid_testing/aigen_34_char_count_K_lt_10.c similarity index 100% rename from benchmarks/iid_testing/gen_case_40_char_count_K_lt_10.c rename to benchmarks/iid_testing/aigen_34_char_count_K_lt_10.c diff --git a/benchmarks/iid_testing/gen_case_40_char_count_K_lt_10.json b/benchmarks/iid_testing/aigen_34_char_count_K_lt_10.json similarity index 100% rename from benchmarks/iid_testing/gen_case_40_char_count_K_lt_10.json rename to benchmarks/iid_testing/aigen_34_char_count_K_lt_10.json diff --git a/benchmarks/iid_testing/gen_case_42_char_balance_U2_R1_gt_10.c b/benchmarks/iid_testing/aigen_35_char_balance_Uinc2_Rdec1_gt10.c similarity index 100% rename from benchmarks/iid_testing/gen_case_42_char_balance_U2_R1_gt_10.c rename to benchmarks/iid_testing/aigen_35_char_balance_Uinc2_Rdec1_gt10.c diff --git a/benchmarks/iid_testing/gen_case_42_char_balance_U2_R1_gt_10.json b/benchmarks/iid_testing/aigen_35_char_balance_Uinc2_Rdec1_gt10.json similarity index 100% rename from benchmarks/iid_testing/gen_case_42_char_balance_U2_R1_gt_10.json rename to benchmarks/iid_testing/aigen_35_char_balance_Uinc2_Rdec1_gt10.json diff --git a/benchmarks/iid_testing/gen_case_44_char_count_H_eq_12.c b/benchmarks/iid_testing/aigen_36_char_count_H_eq_12.c similarity index 100% rename from benchmarks/iid_testing/gen_case_44_char_count_H_eq_12.c rename to benchmarks/iid_testing/aigen_36_char_count_H_eq_12.c diff --git a/benchmarks/iid_testing/gen_case_44_char_count_H_eq_12.json b/benchmarks/iid_testing/aigen_36_char_count_H_eq_12.json similarity index 100% rename from benchmarks/iid_testing/gen_case_44_char_count_H_eq_12.json rename to benchmarks/iid_testing/aigen_36_char_count_H_eq_12.json diff --git a/benchmarks/iid_testing/gen_case_49_char_balance_Z1_A1_gt_15.c b/benchmarks/iid_testing/aigen_37_char_balance_Z1_A1_gt_15.c similarity index 100% rename from benchmarks/iid_testing/gen_case_49_char_balance_Z1_A1_gt_15.c rename to benchmarks/iid_testing/aigen_37_char_balance_Z1_A1_gt_15.c diff --git a/benchmarks/iid_testing/gen_case_49_char_balance_Z1_A1_gt_15.json b/benchmarks/iid_testing/aigen_37_char_balance_Z1_A1_gt_15.json similarity index 100% rename from benchmarks/iid_testing/gen_case_49_char_balance_Z1_A1_gt_15.json rename to benchmarks/iid_testing/aigen_37_char_balance_Z1_A1_gt_15.json diff --git a/benchmarks/iid_testing/gen_case_50_char_count_R_gt_12.c b/benchmarks/iid_testing/aigen_38_char_count_R_gt_12.c similarity index 100% rename from benchmarks/iid_testing/gen_case_50_char_count_R_gt_12.c rename to benchmarks/iid_testing/aigen_38_char_count_R_gt_12.c diff --git a/benchmarks/iid_testing/gen_case_50_char_count_R_gt_12.json b/benchmarks/iid_testing/aigen_38_char_count_R_gt_12.json similarity index 100% rename from benchmarks/iid_testing/gen_case_50_char_count_R_gt_12.json rename to benchmarks/iid_testing/aigen_38_char_count_R_gt_12.json diff --git a/benchmarks/iid_testing/gen_case_51_char_multi_state_R_gt_P_plus_2.c b/benchmarks/iid_testing/aigen_39_char_multi_state_R_gt_P_plus_2.c similarity index 100% rename from benchmarks/iid_testing/gen_case_51_char_multi_state_R_gt_P_plus_2.c rename to benchmarks/iid_testing/aigen_39_char_multi_state_R_gt_P_plus_2.c diff --git a/benchmarks/iid_testing/gen_case_51_char_multi_state_R_gt_P_plus_2.json b/benchmarks/iid_testing/aigen_39_char_multi_state_R_gt_P_plus_2.json similarity index 100% rename from benchmarks/iid_testing/gen_case_51_char_multi_state_R_gt_P_plus_2.json rename to benchmarks/iid_testing/aigen_39_char_multi_state_R_gt_P_plus_2.json diff --git a/benchmarks/iid_testing/gen_case_53_char_two_loops_dep_PQ_ge_7.c b/benchmarks/iid_testing/aigen_40_char_two_loops_balancePQ_ge7.c similarity index 100% rename from benchmarks/iid_testing/gen_case_53_char_two_loops_dep_PQ_ge_7.c rename to benchmarks/iid_testing/aigen_40_char_two_loops_balancePQ_ge7.c diff --git a/benchmarks/iid_testing/gen_case_53_char_two_loops_dep_PQ_ge_7.json b/benchmarks/iid_testing/aigen_40_char_two_loops_balancePQ_ge7.json similarity index 100% rename from benchmarks/iid_testing/gen_case_53_char_two_loops_dep_PQ_ge_7.json rename to benchmarks/iid_testing/aigen_40_char_two_loops_balancePQ_ge7.json diff --git a/benchmarks/iid_testing/gen_case_54_char_count_S_ge_5.c b/benchmarks/iid_testing/aigen_41_char_count_S_ge_5.c similarity index 100% rename from benchmarks/iid_testing/gen_case_54_char_count_S_ge_5.c rename to benchmarks/iid_testing/aigen_41_char_count_S_ge_5.c diff --git a/benchmarks/iid_testing/gen_case_54_char_count_S_ge_5.json b/benchmarks/iid_testing/aigen_41_char_count_S_ge_5.json similarity index 100% rename from benchmarks/iid_testing/gen_case_54_char_count_S_ge_5.json rename to benchmarks/iid_testing/aigen_41_char_count_S_ge_5.json diff --git a/benchmarks/iid_testing/decrement_on_b.c b/benchmarks/iid_testing/char_array_balanceAB_eq10.c similarity index 100% rename from benchmarks/iid_testing/decrement_on_b.c rename to benchmarks/iid_testing/char_array_balanceAB_eq10.c diff --git a/benchmarks/iid_testing/decrement_on_b.json b/benchmarks/iid_testing/char_array_balanceAB_eq10.json similarity index 100% rename from benchmarks/iid_testing/decrement_on_b.json rename to benchmarks/iid_testing/char_array_balanceAB_eq10.json diff --git a/benchmarks/iid_testing/iid_multiple.c "b/benchmarks/iid_testing/char_array_balance_ \320\274\320\275\320\276\320\263\320\276_chars_eq10.c" similarity index 100% rename from benchmarks/iid_testing/iid_multiple.c rename to "benchmarks/iid_testing/char_array_balance_ \320\274\320\275\320\276\320\263\320\276_chars_eq10.c" diff --git a/benchmarks/iid_testing/combination_needed.c b/benchmarks/iid_testing/char_array_balance_A2_B1_eq5.c similarity index 100% rename from benchmarks/iid_testing/combination_needed.c rename to benchmarks/iid_testing/char_array_balance_A2_B1_eq5.c diff --git a/benchmarks/iid_testing/combination_needed.json b/benchmarks/iid_testing/char_array_balance_A2_B1_eq5.json similarity index 100% rename from benchmarks/iid_testing/combination_needed.json rename to benchmarks/iid_testing/char_array_balance_A2_B1_eq5.json diff --git a/benchmarks/iid_testing/iid_multiple.json "b/benchmarks/iid_testing/char_array_balance_\320\274\320\275\320\276\320\263\320\276_chars_eq10.json" similarity index 100% rename from benchmarks/iid_testing/iid_multiple.json rename to "benchmarks/iid_testing/char_array_balance_\320\274\320\275\320\276\320\263\320\276_chars_eq10.json" diff --git a/benchmarks/iid_testing/decrement_in_condition.c b/benchmarks/iid_testing/char_array_conditional_AB_balance_eq10.c similarity index 100% rename from benchmarks/iid_testing/decrement_in_condition.c rename to benchmarks/iid_testing/char_array_conditional_AB_balance_eq10.c diff --git a/benchmarks/iid_testing/decrement_in_condition.json b/benchmarks/iid_testing/char_array_conditional_AB_balance_eq10.json similarity index 100% rename from benchmarks/iid_testing/decrement_in_condition.json rename to benchmarks/iid_testing/char_array_conditional_AB_balance_eq10.json diff --git a/benchmarks/iid_testing/iid_in_condition.c b/benchmarks/iid_testing/char_array_conditional_balanceAB_eq10.c similarity index 100% rename from benchmarks/iid_testing/iid_in_condition.c rename to benchmarks/iid_testing/char_array_conditional_balanceAB_eq10.c diff --git a/benchmarks/iid_testing/iid_in_condition.json b/benchmarks/iid_testing/char_array_conditional_balanceAB_eq10.json similarity index 100% rename from benchmarks/iid_testing/iid_in_condition.json rename to benchmarks/iid_testing/char_array_conditional_balanceAB_eq10.json diff --git a/benchmarks/iid_testing/more_iid_condition_nested_big_values.c b/benchmarks/iid_testing/char_array_countA_deep_nested_complex.c similarity index 100% rename from benchmarks/iid_testing/more_iid_condition_nested_big_values.c rename to benchmarks/iid_testing/char_array_countA_deep_nested_complex.c diff --git a/benchmarks/iid_testing/more_iid_condition_nested_big_values.json b/benchmarks/iid_testing/char_array_countA_deep_nested_complex.json similarity index 100% rename from benchmarks/iid_testing/more_iid_condition_nested_big_values.json rename to benchmarks/iid_testing/char_array_countA_deep_nested_complex.json diff --git a/benchmarks/iid_testing/iid.c b/benchmarks/iid_testing/char_array_countA_eq10.c similarity index 100% rename from benchmarks/iid_testing/iid.c rename to benchmarks/iid_testing/char_array_countA_eq10.c diff --git a/benchmarks/iid_testing/iid.json b/benchmarks/iid_testing/char_array_countA_eq10.json similarity index 100% rename from benchmarks/iid_testing/iid.json rename to benchmarks/iid_testing/char_array_countA_eq10.json diff --git a/benchmarks/iid_testing/iid_twice.c b/benchmarks/iid_testing/char_array_countA_eq10_OR_countB_eq10_v2.c similarity index 100% rename from benchmarks/iid_testing/iid_twice.c rename to benchmarks/iid_testing/char_array_countA_eq10_OR_countB_eq10_v2.c diff --git a/benchmarks/iid_testing/iid_twice.json b/benchmarks/iid_testing/char_array_countA_eq10_OR_countB_eq10_v2.json similarity index 100% rename from benchmarks/iid_testing/iid_twice.json rename to benchmarks/iid_testing/char_array_countA_eq10_OR_countB_eq10_v2.json diff --git a/benchmarks/iid_testing/more_iid_condition.c b/benchmarks/iid_testing/char_array_countA_eq8_or_10_or_11.c similarity index 100% rename from benchmarks/iid_testing/more_iid_condition.c rename to benchmarks/iid_testing/char_array_countA_eq8_or_10_or_11.c diff --git a/benchmarks/iid_testing/more_iid_condition.json b/benchmarks/iid_testing/char_array_countA_eq8_or_10_or_11.json similarity index 100% rename from benchmarks/iid_testing/more_iid_condition.json rename to benchmarks/iid_testing/char_array_countA_eq8_or_10_or_11.json diff --git a/benchmarks/iid_testing/iid_reverse_predicate.c b/benchmarks/iid_testing/char_array_countA_gt10.c similarity index 100% rename from benchmarks/iid_testing/iid_reverse_predicate.c rename to benchmarks/iid_testing/char_array_countA_gt10.c diff --git a/benchmarks/iid_testing/iid_reverse_predicate.json b/benchmarks/iid_testing/char_array_countA_gt10.json similarity index 100% rename from benchmarks/iid_testing/iid_reverse_predicate.json rename to benchmarks/iid_testing/char_array_countA_gt10.json diff --git a/benchmarks/iid_testing/multiplication_in_iid.c b/benchmarks/iid_testing/char_array_countA_mul4_eq20.c similarity index 100% rename from benchmarks/iid_testing/multiplication_in_iid.c rename to benchmarks/iid_testing/char_array_countA_mul4_eq20.c diff --git a/benchmarks/iid_testing/decrement_in_else.json b/benchmarks/iid_testing/char_array_countA_mul4_eq20.json similarity index 100% rename from benchmarks/iid_testing/decrement_in_else.json rename to benchmarks/iid_testing/char_array_countA_mul4_eq20.json diff --git a/benchmarks/iid_testing/more_iid_condition_nested.c b/benchmarks/iid_testing/char_array_countA_nested_check_eq11.c similarity index 100% rename from benchmarks/iid_testing/more_iid_condition_nested.c rename to benchmarks/iid_testing/char_array_countA_nested_check_eq11.c diff --git a/benchmarks/iid_testing/more_iid_condition_nested.json b/benchmarks/iid_testing/char_array_countA_nested_check_eq11.json similarity index 100% rename from benchmarks/iid_testing/more_iid_condition_nested.json rename to benchmarks/iid_testing/char_array_countA_nested_check_eq11.json diff --git a/benchmarks/iid_testing/two_variables_compared.c b/benchmarks/iid_testing/char_array_countA_vs_countB_plus5.c similarity index 100% rename from benchmarks/iid_testing/two_variables_compared.c rename to benchmarks/iid_testing/char_array_countA_vs_countB_plus5.c diff --git a/benchmarks/iid_testing/two_variables_compared.json b/benchmarks/iid_testing/char_array_countA_vs_countB_plus5.json similarity index 100% rename from benchmarks/iid_testing/two_variables_compared.json rename to benchmarks/iid_testing/char_array_countA_vs_countB_plus5.json diff --git a/benchmarks/iid_testing/input_cycle_2_loops.c b/benchmarks/iid_testing/char_array_doublenested_countA_eq10.c similarity index 100% rename from benchmarks/iid_testing/input_cycle_2_loops.c rename to benchmarks/iid_testing/char_array_doublenested_countA_eq10.c diff --git a/benchmarks/iid_testing/input_cycle_2_loops.json b/benchmarks/iid_testing/char_array_doublenested_countA_eq10.json similarity index 100% rename from benchmarks/iid_testing/input_cycle_2_loops.json rename to benchmarks/iid_testing/char_array_doublenested_countA_eq10.json diff --git a/benchmarks/iid_testing/iid_with_dumy_load.c b/benchmarks/iid_testing/char_array_dummyload_countA_eq10.c similarity index 100% rename from benchmarks/iid_testing/iid_with_dumy_load.c rename to benchmarks/iid_testing/char_array_dummyload_countA_eq10.c diff --git a/benchmarks/iid_testing/iid_with_dumy_load.json b/benchmarks/iid_testing/char_array_dummyload_countA_eq10.json similarity index 100% rename from benchmarks/iid_testing/iid_with_dumy_load.json rename to benchmarks/iid_testing/char_array_dummyload_countA_eq10.json diff --git a/benchmarks/iid_testing/decrement_in_else.c b/benchmarks/iid_testing/char_array_ifelse_balance_eq5.c similarity index 100% rename from benchmarks/iid_testing/decrement_in_else.c rename to benchmarks/iid_testing/char_array_ifelse_balance_eq5.c diff --git a/benchmarks/iid_testing/multiplication_in_iid.json b/benchmarks/iid_testing/char_array_ifelse_balance_eq5.json similarity index 100% rename from benchmarks/iid_testing/multiplication_in_iid.json rename to benchmarks/iid_testing/char_array_ifelse_balance_eq5.json diff --git a/benchmarks/iid_testing/lots_of_non_iid_conditions.c b/benchmarks/iid_testing/char_array_indexed_charA_check.c similarity index 100% rename from benchmarks/iid_testing/lots_of_non_iid_conditions.c rename to benchmarks/iid_testing/char_array_indexed_charA_check.c diff --git a/benchmarks/iid_testing/lots_of_non_iid_conditions.json b/benchmarks/iid_testing/char_array_indexed_charA_check.json similarity index 100% rename from benchmarks/iid_testing/lots_of_non_iid_conditions.json rename to benchmarks/iid_testing/char_array_indexed_charA_check.json diff --git a/benchmarks/iid_testing/double_loop.c b/benchmarks/iid_testing/char_array_kAB_lXY_k_eq10.c similarity index 100% rename from benchmarks/iid_testing/double_loop.c rename to benchmarks/iid_testing/char_array_kAB_lXY_k_eq10.c diff --git a/benchmarks/iid_testing/double_loop.json b/benchmarks/iid_testing/char_array_kAB_lXY_k_eq10.json similarity index 100% rename from benchmarks/iid_testing/double_loop.json rename to benchmarks/iid_testing/char_array_kAB_lXY_k_eq10.json diff --git a/benchmarks/iid_testing/same_loop_independent.c b/benchmarks/iid_testing/char_array_kAB_lXY_k_eq10_v2.c similarity index 100% rename from benchmarks/iid_testing/same_loop_independent.c rename to benchmarks/iid_testing/char_array_kAB_lXY_k_eq10_v2.c diff --git a/benchmarks/iid_testing/same_loop_independent.json b/benchmarks/iid_testing/char_array_kAB_lXY_k_eq10_v2.json similarity index 100% rename from benchmarks/iid_testing/same_loop_independent.json rename to benchmarks/iid_testing/char_array_kAB_lXY_k_eq10_v2.json diff --git a/benchmarks/iid_testing/loading_loop_2_at_a_time.c b/benchmarks/iid_testing/char_array_load2by2_countA_eq10.c similarity index 100% rename from benchmarks/iid_testing/loading_loop_2_at_a_time.c rename to benchmarks/iid_testing/char_array_load2by2_countA_eq10.c diff --git a/benchmarks/iid_testing/loading_loop_2_at_a_time.json b/benchmarks/iid_testing/char_array_load2by2_countA_eq10.json similarity index 100% rename from benchmarks/iid_testing/loading_loop_2_at_a_time.json rename to benchmarks/iid_testing/char_array_load2by2_countA_eq10.json diff --git a/benchmarks/iid_testing/different_predicates_in_iid.c b/benchmarks/iid_testing/char_array_multi_predicate_loops.c similarity index 100% rename from benchmarks/iid_testing/different_predicates_in_iid.c rename to benchmarks/iid_testing/char_array_multi_predicate_loops.c diff --git a/benchmarks/iid_testing/different_predicates_in_iid.json b/benchmarks/iid_testing/char_array_multi_predicate_loops.json similarity index 100% rename from benchmarks/iid_testing/different_predicates_in_iid.json rename to benchmarks/iid_testing/char_array_multi_predicate_loops.json diff --git a/benchmarks/iid_testing/multiple_loop_heads.c b/benchmarks/iid_testing/char_array_multibreak_countA_eq10.c similarity index 100% rename from benchmarks/iid_testing/multiple_loop_heads.c rename to benchmarks/iid_testing/char_array_multibreak_countA_eq10.c diff --git a/benchmarks/iid_testing/multiple_loop_heads.json b/benchmarks/iid_testing/char_array_multibreak_countA_eq10.json similarity index 100% rename from benchmarks/iid_testing/multiple_loop_heads.json rename to benchmarks/iid_testing/char_array_multibreak_countA_eq10.json diff --git a/benchmarks/iid_testing/input_cycle_2_loops_combination.c b/benchmarks/iid_testing/char_array_multinested_AB_balance_eq10.c similarity index 100% rename from benchmarks/iid_testing/input_cycle_2_loops_combination.c rename to benchmarks/iid_testing/char_array_multinested_AB_balance_eq10.c diff --git a/benchmarks/iid_testing/input_cycle_2_loops_combination.json b/benchmarks/iid_testing/char_array_multinested_AB_balance_eq10.json similarity index 100% rename from benchmarks/iid_testing/input_cycle_2_loops_combination.json rename to benchmarks/iid_testing/char_array_multinested_AB_balance_eq10.json diff --git a/benchmarks/iid_testing/input_cycle_dec_b.c b/benchmarks/iid_testing/char_array_nested_loop_balanceAB_eq10.c similarity index 100% rename from benchmarks/iid_testing/input_cycle_dec_b.c rename to benchmarks/iid_testing/char_array_nested_loop_balanceAB_eq10.c diff --git a/benchmarks/iid_testing/input_cycle_dec_b.json b/benchmarks/iid_testing/char_array_nested_loop_balanceAB_eq10.json similarity index 100% rename from benchmarks/iid_testing/input_cycle_dec_b.json rename to benchmarks/iid_testing/char_array_nested_loop_balanceAB_eq10.json diff --git a/benchmarks/iid_testing/input_cycle.c b/benchmarks/iid_testing/char_array_nested_loop_countA_eq10.c similarity index 100% rename from benchmarks/iid_testing/input_cycle.c rename to benchmarks/iid_testing/char_array_nested_loop_countA_eq10.c diff --git a/benchmarks/iid_testing/input_cycle.json b/benchmarks/iid_testing/char_array_nested_loop_countA_eq10.json similarity index 100% rename from benchmarks/iid_testing/input_cycle.json rename to benchmarks/iid_testing/char_array_nested_loop_countA_eq10.json diff --git a/benchmarks/iid_testing/loading_loop_plus_20.c b/benchmarks/iid_testing/char_array_offset20_countA_eq10.c similarity index 100% rename from benchmarks/iid_testing/loading_loop_plus_20.c rename to benchmarks/iid_testing/char_array_offset20_countA_eq10.c diff --git a/benchmarks/iid_testing/loading_loop_plus_20.json b/benchmarks/iid_testing/char_array_offset20_countA_eq10.json similarity index 100% rename from benchmarks/iid_testing/loading_loop_plus_20.json rename to benchmarks/iid_testing/char_array_offset20_countA_eq10.json diff --git a/benchmarks/iid_testing/computation_in_loop.c b/benchmarks/iid_testing/char_array_strlen_eq19.c similarity index 100% rename from benchmarks/iid_testing/computation_in_loop.c rename to benchmarks/iid_testing/char_array_strlen_eq19.c diff --git a/benchmarks/iid_testing/computation_in_loop.json b/benchmarks/iid_testing/char_array_strlen_eq19.json similarity index 100% rename from benchmarks/iid_testing/computation_in_loop.json rename to benchmarks/iid_testing/char_array_strlen_eq19.json diff --git a/benchmarks/iid_testing/sum_of_lopp.c b/benchmarks/iid_testing/char_array_sum_chars_eq500.c similarity index 100% rename from benchmarks/iid_testing/sum_of_lopp.c rename to benchmarks/iid_testing/char_array_sum_chars_eq500.c diff --git a/benchmarks/iid_testing/char_array_sum_chars_eq500.json b/benchmarks/iid_testing/char_array_sum_chars_eq500.json new file mode 100644 index 00000000..5cd1e344 --- /dev/null +++ b/benchmarks/iid_testing/char_array_sum_chars_eq500.json @@ -0,0 +1,55 @@ +{ + "args": { + "max_executions": 10000, + "max_seconds": 300, + "max_trace_length": 10000, + "max_stack_size": 25, + "max_stdin_bytes": 65536, + "max_exec_milliseconds": 250, + "max_exec_megabytes": 1024, + "stdin_model": "stdin_replay_bytes_then_repeat_zero", + "stdout_model": "stdout_void", + "optimizer_max_seconds": 10, + "optimizer_max_trace_length": 1000000, + "optimizer_max_stdin_bytes": 1000000 + }, + "results": { + "termination_type": "NORMAL", + "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", + "num_executions": 6789, + "num_covered_branchings": 7, + "covered_branchings": [ + 1, + 0, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0 + ], + "output_statistics": { + "sensitivity_analysis": { + "num_generated_tests": 5, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "typed_minimization_analysis": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + }, + "STARTUP": { + "num_generated_tests": 1, + "num_crashes": 0, + "num_boundary_violations": 0 + } + } + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/same_loop_dependent_2.c b/benchmarks/iid_testing/char_array_two_pass_balanceABCD_eq7.c similarity index 100% rename from benchmarks/iid_testing/same_loop_dependent_2.c rename to benchmarks/iid_testing/char_array_two_pass_balanceABCD_eq7.c diff --git a/benchmarks/iid_testing/same_loop_dependent_2.json b/benchmarks/iid_testing/char_array_two_pass_balanceABCD_eq7.json similarity index 100% rename from benchmarks/iid_testing/same_loop_dependent_2.json rename to benchmarks/iid_testing/char_array_two_pass_balanceABCD_eq7.json diff --git a/benchmarks/iid_testing/same_loop_dependent.c b/benchmarks/iid_testing/char_array_two_pass_balanceAB_eq10.c similarity index 100% rename from benchmarks/iid_testing/same_loop_dependent.c rename to benchmarks/iid_testing/char_array_two_pass_balanceAB_eq10.c diff --git a/benchmarks/iid_testing/same_loop_dependent.json b/benchmarks/iid_testing/char_array_two_pass_balanceAB_eq10.json similarity index 100% rename from benchmarks/iid_testing/same_loop_dependent.json rename to benchmarks/iid_testing/char_array_two_pass_balanceAB_eq10.json diff --git a/benchmarks/iid_testing/two_loops_used.c b/benchmarks/iid_testing/char_two_arrays_matchST_gt10.c similarity index 100% rename from benchmarks/iid_testing/two_loops_used.c rename to benchmarks/iid_testing/char_two_arrays_matchST_gt10.c diff --git a/benchmarks/iid_testing/two_loops_used.json b/benchmarks/iid_testing/char_two_arrays_matchST_gt10.json similarity index 100% rename from benchmarks/iid_testing/two_loops_used.json rename to benchmarks/iid_testing/char_two_arrays_matchST_gt10.json From 9786aeed66f1a61737fd1345255ea99c4abf5bad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 10 May 2025 20:48:52 +0200 Subject: [PATCH 143/144] feat: update executions with the final version of IID Vector analysis --- benchmarks/iid_testing/_.c | 36 +++++++++++++++++++ .../aigen_01_char_countX_gt15.json | 2 +- .../aigen_02_char_balancePN_eq15.json | 2 +- .../aigen_03_char_sum_AB_val_eq25.json | 2 +- ...en_04_char_nonvowel_upper_decr_target.json | 2 +- .../aigen_06_char_bounded_counter_UD_eq5.json | 2 +- .../aigen_07_char_two_arrays_matchST_gt5.json | 10 ++---- .../aigen_08_char_ptr_countP_eq5.json | 2 +- ..._two_loops_countA_eq10_or_countB_eq10.json | 2 +- .../aigen_11_char_continueS_countF_ge10.json | 2 +- .../aigen_14_struct_statusA_ge10.json | 2 +- .../aigen_15_char_countX_eq10.json | 2 +- .../aigen_16_char_countY_gt15.json | 2 +- .../aigen_17_char_balanceAB_eq13.json | 2 +- .../aigen_18_char_digitrange678_eq3.json | 2 +- ...igen_19_char_continue_S_count_F_ge_10.json | 2 +- .../aigen_20_char_break_Q_count_P_eq_12.json | 2 +- ...gen_21_char_multi_state_A_gt_B_plus_5.json | 2 +- .../aigen_22_char_countXYZ_sum_eq10.json | 2 +- .../aigen_23_char_two_loops_A_or_B_eq_10.json | 2 +- ...24_char_two_loops_dependent_ABCD_eq_7.json | 2 +- .../aigen_25_struct_item_status_A_ge_10.json | 2 +- ...en_26_struct_dataval_balance_PN_eq_20.json | 2 +- ..._27_char_count_X_nested_check_eq_8_11.json | 2 +- .../aigen_28_char_countA_nested_eq11.json | 4 +-- .../aigen_29_char_countA_mul3_eq30.json | 2 +- ...gen_30_char_state_dependent_inc_gt_50.json | 2 +- ...igen_31_two_char_arrays_match_ST_gt_5.json | 2 +- ...en_33_char_input_cycle_nested_A_eq_10.json | 2 +- .../aigen_34_char_count_K_lt_10.json | 2 +- ...igen_35_char_balance_Uinc2_Rdec1_gt10.json | 2 +- .../aigen_36_char_count_H_eq_12.json | 2 +- .../aigen_37_char_balance_Z1_A1_gt_15.json | 2 +- .../aigen_38_char_count_R_gt_12.json | 2 +- ...gen_39_char_multi_state_R_gt_P_plus_2.json | 2 +- ...aigen_40_char_two_loops_balancePQ_ge7.json | 2 +- .../aigen_41_char_count_S_ge_5.json | 2 +- .../char_array_balance_A2_B1_eq5.json | 2 +- .../char_array_balance_many_chars_eq10.c | 0 .../char_array_balance_many_chars_eq10.json | 0 ...char_array_countA_deep_nested_complex.json | 2 +- .../char_array_countA_nested_check_eq11.json | 2 +- .../char_array_countA_vs_countB_plus5.json | 2 +- .../char_array_doublenested_countA_eq10.json | 2 +- ...har_array_multinested_AB_balance_eq10.json | 2 +- .../iid_testing/char_array_sum_chars_eq500.c | 2 +- .../char_array_sum_chars_eq500.json | 12 +++---- .../char_two_arrays_matchST_gt10.c | 1 + .../char_two_arrays_matchST_gt10.json | 2 +- benchmarks/iid_testing/digit_range.json | 2 +- benchmarks/iid_testing/nested_modulo.json | 2 +- benchmarks/iid_testing/pointer.json | 2 +- 52 files changed, 93 insertions(+), 60 deletions(-) create mode 100644 benchmarks/iid_testing/_.c rename "benchmarks/iid_testing/char_array_balance_ \320\274\320\275\320\276\320\263\320\276_chars_eq10.c" => benchmarks/iid_testing/char_array_balance_many_chars_eq10.c (100%) rename "benchmarks/iid_testing/char_array_balance_\320\274\320\275\320\276\320\263\320\276_chars_eq10.json" => benchmarks/iid_testing/char_array_balance_many_chars_eq10.json (100%) diff --git a/benchmarks/iid_testing/_.c b/benchmarks/iid_testing/_.c new file mode 100644 index 00000000..b70849fc --- /dev/null +++ b/benchmarks/iid_testing/_.c @@ -0,0 +1,36 @@ +#include + +extern char __VERIFIER_nondet_char(); +extern short __VERIFIER_nondet_short(); + + +int main() +{ + char s[ 50 ]; + { + short n; + n = __VERIFIER_nondet_short(); + if ( n <= 0 ) // ID: 1 + return -1; + if ( n >= sizeof( s ) / sizeof( s[ 0 ] ) ) // ID: 2 + return -1; + for ( short i = 0; i < n; ++i ) // ID: 3 + s[ i ] = __VERIFIER_nondet_char(); + if ( s[ n - 1 ] != '\0' ) // ID: 4 + return -1; + } + { + int i = 0, k = 0; + while ( true ) { + if ( s[ i ] == '\0' ) // ID: 5 + break; + k += s[ i ]; + ++i; + } + + if ( k == 500 ) // ID: 6 + return 1; + + return 0; + } +} \ No newline at end of file diff --git a/benchmarks/iid_testing/aigen_01_char_countX_gt15.json b/benchmarks/iid_testing/aigen_01_char_countX_gt15.json index b30659e8..6df0b390 100644 --- a/benchmarks/iid_testing/aigen_01_char_countX_gt15.json +++ b/benchmarks/iid_testing/aigen_01_char_countX_gt15.json @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 5694, + "num_executions": 3880, "num_covered_branchings": 6, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_02_char_balancePN_eq15.json b/benchmarks/iid_testing/aigen_02_char_balancePN_eq15.json index 111ed923..8f3766bc 100644 --- a/benchmarks/iid_testing/aigen_02_char_balancePN_eq15.json +++ b/benchmarks/iid_testing/aigen_02_char_balancePN_eq15.json @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 4658, + "num_executions": 3211, "num_covered_branchings": 7, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_03_char_sum_AB_val_eq25.json b/benchmarks/iid_testing/aigen_03_char_sum_AB_val_eq25.json index c960b594..87aadfe3 100644 --- a/benchmarks/iid_testing/aigen_03_char_sum_AB_val_eq25.json +++ b/benchmarks/iid_testing/aigen_03_char_sum_AB_val_eq25.json @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 11717, + "num_executions": 9524, "num_covered_branchings": 7, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_04_char_nonvowel_upper_decr_target.json b/benchmarks/iid_testing/aigen_04_char_nonvowel_upper_decr_target.json index 9cd8bd82..4e054f12 100644 --- a/benchmarks/iid_testing/aigen_04_char_nonvowel_upper_decr_target.json +++ b/benchmarks/iid_testing/aigen_04_char_nonvowel_upper_decr_target.json @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 2594, + "num_executions": 1799, "num_covered_branchings": 12, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_06_char_bounded_counter_UD_eq5.json b/benchmarks/iid_testing/aigen_06_char_bounded_counter_UD_eq5.json index d3a7a120..da22a0d4 100644 --- a/benchmarks/iid_testing/aigen_06_char_bounded_counter_UD_eq5.json +++ b/benchmarks/iid_testing/aigen_06_char_bounded_counter_UD_eq5.json @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 9340, + "num_executions": 9907, "num_covered_branchings": 9, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_07_char_two_arrays_matchST_gt5.json b/benchmarks/iid_testing/aigen_07_char_two_arrays_matchST_gt5.json index d3a7a120..473e872d 100644 --- a/benchmarks/iid_testing/aigen_07_char_two_arrays_matchST_gt5.json +++ b/benchmarks/iid_testing/aigen_07_char_two_arrays_matchST_gt5.json @@ -17,8 +17,8 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 9340, - "num_covered_branchings": 9, + "num_executions": 3107, + "num_covered_branchings": 7, "covered_branchings": [ 1, 0, @@ -33,15 +33,11 @@ 6, 0, 7, - 0, - 8, - 0, - 9, 0 ], "output_statistics": { "bitshare_analysis": { - "num_generated_tests": 3, + "num_generated_tests": 1, "num_crashes": 0, "num_boundary_violations": 0 }, diff --git a/benchmarks/iid_testing/aigen_08_char_ptr_countP_eq5.json b/benchmarks/iid_testing/aigen_08_char_ptr_countP_eq5.json index f0e39ed5..46a3863e 100644 --- a/benchmarks/iid_testing/aigen_08_char_ptr_countP_eq5.json +++ b/benchmarks/iid_testing/aigen_08_char_ptr_countP_eq5.json @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 1426, + "num_executions": 923, "num_covered_branchings": 6, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_10_char_two_loops_countA_eq10_or_countB_eq10.json b/benchmarks/iid_testing/aigen_10_char_two_loops_countA_eq10_or_countB_eq10.json index cb65b6e6..8288bd4f 100644 --- a/benchmarks/iid_testing/aigen_10_char_two_loops_countA_eq10_or_countB_eq10.json +++ b/benchmarks/iid_testing/aigen_10_char_two_loops_countA_eq10_or_countB_eq10.json @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 6061, + "num_executions": 3652, "num_covered_branchings": 9, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_11_char_continueS_countF_ge10.json b/benchmarks/iid_testing/aigen_11_char_continueS_countF_ge10.json index a8ea38f4..e586b99e 100644 --- a/benchmarks/iid_testing/aigen_11_char_continueS_countF_ge10.json +++ b/benchmarks/iid_testing/aigen_11_char_continueS_countF_ge10.json @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 4050, + "num_executions": 2252, "num_covered_branchings": 7, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_14_struct_statusA_ge10.json b/benchmarks/iid_testing/aigen_14_struct_statusA_ge10.json index c753c534..02c1726d 100644 --- a/benchmarks/iid_testing/aigen_14_struct_statusA_ge10.json +++ b/benchmarks/iid_testing/aigen_14_struct_statusA_ge10.json @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 2190, + "num_executions": 1910, "num_covered_branchings": 6, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_15_char_countX_eq10.json b/benchmarks/iid_testing/aigen_15_char_countX_eq10.json index 0a7146b6..d519fa9b 100644 --- a/benchmarks/iid_testing/aigen_15_char_countX_eq10.json +++ b/benchmarks/iid_testing/aigen_15_char_countX_eq10.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 3114, + "num_executions": 1810, "num_covered_branchings": 6, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_16_char_countY_gt15.json b/benchmarks/iid_testing/aigen_16_char_countY_gt15.json index 7d7904ba..493e7ac3 100644 --- a/benchmarks/iid_testing/aigen_16_char_countY_gt15.json +++ b/benchmarks/iid_testing/aigen_16_char_countY_gt15.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 4683, + "num_executions": 3544, "num_covered_branchings": 6, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_17_char_balanceAB_eq13.json b/benchmarks/iid_testing/aigen_17_char_balanceAB_eq13.json index 3dff5e7a..a2c59a81 100644 --- a/benchmarks/iid_testing/aigen_17_char_balanceAB_eq13.json +++ b/benchmarks/iid_testing/aigen_17_char_balanceAB_eq13.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 4428, + "num_executions": 2593, "num_covered_branchings": 7, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_18_char_digitrange678_eq3.json b/benchmarks/iid_testing/aigen_18_char_digitrange678_eq3.json index e62ae8c8..86072272 100644 --- a/benchmarks/iid_testing/aigen_18_char_digitrange678_eq3.json +++ b/benchmarks/iid_testing/aigen_18_char_digitrange678_eq3.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 2506, + "num_executions": 1561, "num_covered_branchings": 7, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_19_char_continue_S_count_F_ge_10.json b/benchmarks/iid_testing/aigen_19_char_continue_S_count_F_ge_10.json index 4ecf2578..d81f1cfe 100644 --- a/benchmarks/iid_testing/aigen_19_char_continue_S_count_F_ge_10.json +++ b/benchmarks/iid_testing/aigen_19_char_continue_S_count_F_ge_10.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 4222, + "num_executions": 2252, "num_covered_branchings": 7, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_20_char_break_Q_count_P_eq_12.json b/benchmarks/iid_testing/aigen_20_char_break_Q_count_P_eq_12.json index d926905e..892ee80a 100644 --- a/benchmarks/iid_testing/aigen_20_char_break_Q_count_P_eq_12.json +++ b/benchmarks/iid_testing/aigen_20_char_break_Q_count_P_eq_12.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 3736, + "num_executions": 2243, "num_covered_branchings": 7, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_21_char_multi_state_A_gt_B_plus_5.json b/benchmarks/iid_testing/aigen_21_char_multi_state_A_gt_B_plus_5.json index f1f5c8cd..fbe5458f 100644 --- a/benchmarks/iid_testing/aigen_21_char_multi_state_A_gt_B_plus_5.json +++ b/benchmarks/iid_testing/aigen_21_char_multi_state_A_gt_B_plus_5.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 2350, + "num_executions": 1396, "num_covered_branchings": 7, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_22_char_countXYZ_sum_eq10.json b/benchmarks/iid_testing/aigen_22_char_countXYZ_sum_eq10.json index 60854e50..e3074c95 100644 --- a/benchmarks/iid_testing/aigen_22_char_countXYZ_sum_eq10.json +++ b/benchmarks/iid_testing/aigen_22_char_countXYZ_sum_eq10.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 3452, + "num_executions": 1822, "num_covered_branchings": 8, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_23_char_two_loops_A_or_B_eq_10.json b/benchmarks/iid_testing/aigen_23_char_two_loops_A_or_B_eq_10.json index e9536b8a..333c3700 100644 --- a/benchmarks/iid_testing/aigen_23_char_two_loops_A_or_B_eq_10.json +++ b/benchmarks/iid_testing/aigen_23_char_two_loops_A_or_B_eq_10.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 6061, + "num_executions": 3652, "num_covered_branchings": 9, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_24_char_two_loops_dependent_ABCD_eq_7.json b/benchmarks/iid_testing/aigen_24_char_two_loops_dependent_ABCD_eq_7.json index 87574f5c..73888b8c 100644 --- a/benchmarks/iid_testing/aigen_24_char_two_loops_dependent_ABCD_eq_7.json +++ b/benchmarks/iid_testing/aigen_24_char_two_loops_dependent_ABCD_eq_7.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 2045, + "num_executions": 1299, "num_covered_branchings": 10, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_25_struct_item_status_A_ge_10.json b/benchmarks/iid_testing/aigen_25_struct_item_status_A_ge_10.json index 88653fd6..46f9ae80 100644 --- a/benchmarks/iid_testing/aigen_25_struct_item_status_A_ge_10.json +++ b/benchmarks/iid_testing/aigen_25_struct_item_status_A_ge_10.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 10219, + "num_executions": 6834, "num_covered_branchings": 6, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_26_struct_dataval_balance_PN_eq_20.json b/benchmarks/iid_testing/aigen_26_struct_dataval_balance_PN_eq_20.json index 2ff27bba..7dd23d1c 100644 --- a/benchmarks/iid_testing/aigen_26_struct_dataval_balance_PN_eq_20.json +++ b/benchmarks/iid_testing/aigen_26_struct_dataval_balance_PN_eq_20.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 24945, + "num_executions": 20230, "num_covered_branchings": 7, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_27_char_count_X_nested_check_eq_8_11.json b/benchmarks/iid_testing/aigen_27_char_count_X_nested_check_eq_8_11.json index 059a8b62..3fb5b44f 100644 --- a/benchmarks/iid_testing/aigen_27_char_count_X_nested_check_eq_8_11.json +++ b/benchmarks/iid_testing/aigen_27_char_count_X_nested_check_eq_8_11.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 3131, + "num_executions": 2702, "num_covered_branchings": 7, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_28_char_countA_nested_eq11.json b/benchmarks/iid_testing/aigen_28_char_countA_nested_eq11.json index 8e542881..94e96484 100644 --- a/benchmarks/iid_testing/aigen_28_char_countA_nested_eq11.json +++ b/benchmarks/iid_testing/aigen_28_char_countA_nested_eq11.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 7360, + "num_executions": 2145, "num_covered_branchings": 8, "covered_branchings": [ 1, @@ -38,7 +38,7 @@ ], "output_statistics": { "sensitivity_analysis": { - "num_generated_tests": 4, + "num_generated_tests": 2, "num_crashes": 0, "num_boundary_violations": 0 }, diff --git a/benchmarks/iid_testing/aigen_29_char_countA_mul3_eq30.json b/benchmarks/iid_testing/aigen_29_char_countA_mul3_eq30.json index b87b939f..d519fa9b 100644 --- a/benchmarks/iid_testing/aigen_29_char_countA_mul3_eq30.json +++ b/benchmarks/iid_testing/aigen_29_char_countA_mul3_eq30.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 4234, + "num_executions": 1810, "num_covered_branchings": 6, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_30_char_state_dependent_inc_gt_50.json b/benchmarks/iid_testing/aigen_30_char_state_dependent_inc_gt_50.json index ad372155..51f24c49 100644 --- a/benchmarks/iid_testing/aigen_30_char_state_dependent_inc_gt_50.json +++ b/benchmarks/iid_testing/aigen_30_char_state_dependent_inc_gt_50.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 14605, + "num_executions": 11414, "num_covered_branchings": 7, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_31_two_char_arrays_match_ST_gt_5.json b/benchmarks/iid_testing/aigen_31_two_char_arrays_match_ST_gt_5.json index 7520dcca..c0e68500 100644 --- a/benchmarks/iid_testing/aigen_31_two_char_arrays_match_ST_gt_5.json +++ b/benchmarks/iid_testing/aigen_31_two_char_arrays_match_ST_gt_5.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 7020, + "num_executions": 3107, "num_covered_branchings": 7, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_33_char_input_cycle_nested_A_eq_10.json b/benchmarks/iid_testing/aigen_33_char_input_cycle_nested_A_eq_10.json index f7c89900..19710352 100644 --- a/benchmarks/iid_testing/aigen_33_char_input_cycle_nested_A_eq_10.json +++ b/benchmarks/iid_testing/aigen_33_char_input_cycle_nested_A_eq_10.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 28806, + "num_executions": 31124, "num_covered_branchings": 12, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_34_char_count_K_lt_10.json b/benchmarks/iid_testing/aigen_34_char_count_K_lt_10.json index b4841efc..e9f7da97 100644 --- a/benchmarks/iid_testing/aigen_34_char_count_K_lt_10.json +++ b/benchmarks/iid_testing/aigen_34_char_count_K_lt_10.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 4216, + "num_executions": 2246, "num_covered_branchings": 6, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_35_char_balance_Uinc2_Rdec1_gt10.json b/benchmarks/iid_testing/aigen_35_char_balance_Uinc2_Rdec1_gt10.json index f1f5c8cd..fbe5458f 100644 --- a/benchmarks/iid_testing/aigen_35_char_balance_Uinc2_Rdec1_gt10.json +++ b/benchmarks/iid_testing/aigen_35_char_balance_Uinc2_Rdec1_gt10.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 2350, + "num_executions": 1396, "num_covered_branchings": 7, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_36_char_count_H_eq_12.json b/benchmarks/iid_testing/aigen_36_char_count_H_eq_12.json index 2ab8abc5..0df483ec 100644 --- a/benchmarks/iid_testing/aigen_36_char_count_H_eq_12.json +++ b/benchmarks/iid_testing/aigen_36_char_count_H_eq_12.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 3981, + "num_executions": 2308, "num_covered_branchings": 6, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_37_char_balance_Z1_A1_gt_15.json b/benchmarks/iid_testing/aigen_37_char_balance_Z1_A1_gt_15.json index 811a6fad..45057f6e 100644 --- a/benchmarks/iid_testing/aigen_37_char_balance_Z1_A1_gt_15.json +++ b/benchmarks/iid_testing/aigen_37_char_balance_Z1_A1_gt_15.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 4604, + "num_executions": 3897, "num_covered_branchings": 7, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_38_char_count_R_gt_12.json b/benchmarks/iid_testing/aigen_38_char_count_R_gt_12.json index 1e905873..f100021d 100644 --- a/benchmarks/iid_testing/aigen_38_char_count_R_gt_12.json +++ b/benchmarks/iid_testing/aigen_38_char_count_R_gt_12.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 4697, + "num_executions": 2587, "num_covered_branchings": 6, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_39_char_multi_state_R_gt_P_plus_2.json b/benchmarks/iid_testing/aigen_39_char_multi_state_R_gt_P_plus_2.json index 57e7aa4e..30ef0fcf 100644 --- a/benchmarks/iid_testing/aigen_39_char_multi_state_R_gt_P_plus_2.json +++ b/benchmarks/iid_testing/aigen_39_char_multi_state_R_gt_P_plus_2.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 1468, + "num_executions": 1050, "num_covered_branchings": 7, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_40_char_two_loops_balancePQ_ge7.json b/benchmarks/iid_testing/aigen_40_char_two_loops_balancePQ_ge7.json index 56e8af09..acd1cf89 100644 --- a/benchmarks/iid_testing/aigen_40_char_two_loops_balancePQ_ge7.json +++ b/benchmarks/iid_testing/aigen_40_char_two_loops_balancePQ_ge7.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 1975, + "num_executions": 1357, "num_covered_branchings": 8, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/aigen_41_char_count_S_ge_5.json b/benchmarks/iid_testing/aigen_41_char_count_S_ge_5.json index 65a0e189..1b3a9fdf 100644 --- a/benchmarks/iid_testing/aigen_41_char_count_S_ge_5.json +++ b/benchmarks/iid_testing/aigen_41_char_count_S_ge_5.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 2030, + "num_executions": 1312, "num_covered_branchings": 6, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/char_array_balance_A2_B1_eq5.json b/benchmarks/iid_testing/char_array_balance_A2_B1_eq5.json index 2564ccf3..dca556b5 100644 --- a/benchmarks/iid_testing/char_array_balance_A2_B1_eq5.json +++ b/benchmarks/iid_testing/char_array_balance_A2_B1_eq5.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 1654, + "num_executions": 3908, "num_covered_branchings": 8, "covered_branchings": [ 1, diff --git "a/benchmarks/iid_testing/char_array_balance_ \320\274\320\275\320\276\320\263\320\276_chars_eq10.c" b/benchmarks/iid_testing/char_array_balance_many_chars_eq10.c similarity index 100% rename from "benchmarks/iid_testing/char_array_balance_ \320\274\320\275\320\276\320\263\320\276_chars_eq10.c" rename to benchmarks/iid_testing/char_array_balance_many_chars_eq10.c diff --git "a/benchmarks/iid_testing/char_array_balance_\320\274\320\275\320\276\320\263\320\276_chars_eq10.json" b/benchmarks/iid_testing/char_array_balance_many_chars_eq10.json similarity index 100% rename from "benchmarks/iid_testing/char_array_balance_\320\274\320\275\320\276\320\263\320\276_chars_eq10.json" rename to benchmarks/iid_testing/char_array_balance_many_chars_eq10.json diff --git a/benchmarks/iid_testing/char_array_countA_deep_nested_complex.json b/benchmarks/iid_testing/char_array_countA_deep_nested_complex.json index d18aeb41..26037999 100644 --- a/benchmarks/iid_testing/char_array_countA_deep_nested_complex.json +++ b/benchmarks/iid_testing/char_array_countA_deep_nested_complex.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 54514, + "num_executions": 40502, "num_covered_branchings": 12, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/char_array_countA_nested_check_eq11.json b/benchmarks/iid_testing/char_array_countA_nested_check_eq11.json index fcef796c..3cbff683 100644 --- a/benchmarks/iid_testing/char_array_countA_nested_check_eq11.json +++ b/benchmarks/iid_testing/char_array_countA_nested_check_eq11.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 11207, + "num_executions": 7492, "num_covered_branchings": 9, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/char_array_countA_vs_countB_plus5.json b/benchmarks/iid_testing/char_array_countA_vs_countB_plus5.json index f72d9871..ad9eb90b 100644 --- a/benchmarks/iid_testing/char_array_countA_vs_countB_plus5.json +++ b/benchmarks/iid_testing/char_array_countA_vs_countB_plus5.json @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 2350, + "num_executions": 1396, "num_covered_branchings": 7, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/char_array_doublenested_countA_eq10.json b/benchmarks/iid_testing/char_array_doublenested_countA_eq10.json index 0fc5542c..6981c0b7 100644 --- a/benchmarks/iid_testing/char_array_doublenested_countA_eq10.json +++ b/benchmarks/iid_testing/char_array_doublenested_countA_eq10.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 2633, + "num_executions": 3013, "num_covered_branchings": 11, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/char_array_multinested_AB_balance_eq10.json b/benchmarks/iid_testing/char_array_multinested_AB_balance_eq10.json index 144ea353..0554eca6 100644 --- a/benchmarks/iid_testing/char_array_multinested_AB_balance_eq10.json +++ b/benchmarks/iid_testing/char_array_multinested_AB_balance_eq10.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 2855, + "num_executions": 2365, "num_covered_branchings": 13, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/char_array_sum_chars_eq500.c b/benchmarks/iid_testing/char_array_sum_chars_eq500.c index 92c683c7..b70849fc 100644 --- a/benchmarks/iid_testing/char_array_sum_chars_eq500.c +++ b/benchmarks/iid_testing/char_array_sum_chars_eq500.c @@ -33,4 +33,4 @@ int main() return 0; } -} +} \ No newline at end of file diff --git a/benchmarks/iid_testing/char_array_sum_chars_eq500.json b/benchmarks/iid_testing/char_array_sum_chars_eq500.json index 5cd1e344..fa1ac97a 100644 --- a/benchmarks/iid_testing/char_array_sum_chars_eq500.json +++ b/benchmarks/iid_testing/char_array_sum_chars_eq500.json @@ -1,6 +1,6 @@ { "args": { - "max_executions": 10000, + "max_executions": 20000, "max_seconds": 300, "max_trace_length": 10000, "max_stack_size": 25, @@ -16,8 +16,8 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 6789, - "num_covered_branchings": 7, + "num_executions": 72926, + "num_covered_branchings": 6, "covered_branchings": [ 1, 0, @@ -30,13 +30,13 @@ 5, 0, 6, - 0, - 7, 0 ], + "num_uncovered_branchings": 0, + "uncovered_branchings": [], "output_statistics": { "sensitivity_analysis": { - "num_generated_tests": 5, + "num_generated_tests": 4, "num_crashes": 0, "num_boundary_violations": 0 }, diff --git a/benchmarks/iid_testing/char_two_arrays_matchST_gt10.c b/benchmarks/iid_testing/char_two_arrays_matchST_gt10.c index f4d4fc98..6ac68521 100644 --- a/benchmarks/iid_testing/char_two_arrays_matchST_gt10.c +++ b/benchmarks/iid_testing/char_two_arrays_matchST_gt10.c @@ -13,6 +13,7 @@ int main() short size; int match_count = 0; + size = __VERIFIER_nondet_short(); if ( size <= 0 || size > MAX_SIZE ) { return -1; } diff --git a/benchmarks/iid_testing/char_two_arrays_matchST_gt10.json b/benchmarks/iid_testing/char_two_arrays_matchST_gt10.json index 19a9682f..6ffe0f77 100644 --- a/benchmarks/iid_testing/char_two_arrays_matchST_gt10.json +++ b/benchmarks/iid_testing/char_two_arrays_matchST_gt10.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 10743, + "num_executions": 6897, "num_covered_branchings": 7, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/digit_range.json b/benchmarks/iid_testing/digit_range.json index 630af63e..abdff7aa 100644 --- a/benchmarks/iid_testing/digit_range.json +++ b/benchmarks/iid_testing/digit_range.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 2506, + "num_executions": 1561, "num_covered_branchings": 7, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/nested_modulo.json b/benchmarks/iid_testing/nested_modulo.json index 04b041fc..5921b69a 100644 --- a/benchmarks/iid_testing/nested_modulo.json +++ b/benchmarks/iid_testing/nested_modulo.json @@ -17,7 +17,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 14605, + "num_executions": 11414, "num_covered_branchings": 7, "covered_branchings": [ 1, diff --git a/benchmarks/iid_testing/pointer.json b/benchmarks/iid_testing/pointer.json index b7ce8254..bdeead36 100644 --- a/benchmarks/iid_testing/pointer.json +++ b/benchmarks/iid_testing/pointer.json @@ -16,7 +16,7 @@ "results": { "termination_type": "NORMAL", "termination_reason": "ALL_REACHABLE_BRANCHINGS_COVERED", - "num_executions": 1426, + "num_executions": 923, "num_covered_branchings": 6, "covered_branchings": [ 1, From 0cd8051c43f620c2a9c5900de9baf8afa04b4dce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20M=C3=A1ca?= Date: Sat, 10 May 2025 20:49:14 +0200 Subject: [PATCH 144/144] feat: remove include --- src/fuzzing/src/fuzzer.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/fuzzing/src/fuzzer.cpp b/src/fuzzing/src/fuzzer.cpp index b2b87cf3..ee8664a1 100644 --- a/src/fuzzing/src/fuzzer.cpp +++ b/src/fuzzing/src/fuzzer.cpp @@ -8,10 +8,9 @@ #include #include #include -#include -// constexpr bool use_vector_analysis = false; -constexpr bool use_vector_analysis = true; +constexpr bool use_vector_analysis = false; +// constexpr bool use_vector_analysis = true; namespace fuzzing {