From 17a1088d18d61a9ca73b648815b342a05824127c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20B=C3=B6ck?= Date: Mon, 23 Mar 2026 16:05:27 +0100 Subject: [PATCH 1/3] [hls-fuzzer] Add non-functional fuzzer to bitwidth optimizations With the recently landed bitwidth type system we have an initial generator that is capable of generating C programs known to compute under a given bitwidth. The initial PR used this for functional testing, where the programs generated are simply more likely to trigger mis-compilations in the optimization pass. The new non-functional oracle introduced in this PR tests the optimization capabilities of the pass: It checks the output IR for any computations that are done at a higher bitwidth than expected. The logic for doing so is currently simple: All results of operations that aren't operands of an `end` should be done using a bitwidth that is at most equal to the bitwidth the generator used. The logic for this is via a separate oracle executable called `hls-fuzzer-check-bitwidth`. This enables a nice workflow where reproduction can be done through a simple `bash execute.sh` as is done for the functional testing. The mode is currently selected using a `--non-functional` flag. The PR contains following limitations to be fixed later or simple known issues: * It finds bugs very, very quickly due to: https://github.com/EPFL-LAP/dynamatic/issues/798 which is blocked by https://github.com/EPFL-LAP/dynamatic/issues/792 which is blocked by https://github.com/EPFL-LAP/dynamatic/pull/802. I manually verified some bugs it found that the logic ought to work, but it is hard to say until we get more usage. As a drive-by fix, also disable right-shift generation for now (the logic was incorrect and was accidentally included). --- tools/hls-fuzzer/AbstractWorker.h | 2 +- tools/hls-fuzzer/CMakeLists.txt | 2 + tools/hls-fuzzer/Options.h | 10 ++- tools/hls-fuzzer/OptionsParser.cpp | 6 +- tools/hls-fuzzer/Opts.td | 5 ++ tools/hls-fuzzer/hls-fuzzer.cpp | 11 ++- tools/hls-fuzzer/oracles/CMakeLists.txt | 7 ++ tools/hls-fuzzer/oracles/check-bitwidth.cpp | 69 +++++++++++++++++++ .../targets/BitwidthOptimizationsTarget.cpp | 37 ++++++++-- .../hls-fuzzer/targets/BitwidthTypeSystem.cpp | 2 + tools/hls-fuzzer/targets/RandomCTarget.cpp | 7 +- tools/hls-fuzzer/targets/TargetUtils.cpp | 49 ++++++++----- tools/hls-fuzzer/targets/TargetUtils.h | 19 +++++ 13 files changed, 196 insertions(+), 30 deletions(-) create mode 100644 tools/hls-fuzzer/oracles/CMakeLists.txt create mode 100644 tools/hls-fuzzer/oracles/check-bitwidth.cpp diff --git a/tools/hls-fuzzer/AbstractWorker.h b/tools/hls-fuzzer/AbstractWorker.h index 6e60297d91..121a00e8d8 100644 --- a/tools/hls-fuzzer/AbstractWorker.h +++ b/tools/hls-fuzzer/AbstractWorker.h @@ -21,7 +21,7 @@ class AbstractWorker { /// Creates a random C program that should be written to 'os'. /// 'os' writes to a file called 'functionName.c'. virtual void generate(llvm::raw_ostream &os, - llvm::StringRef functionName) const = 0; + llvm::StringRef functionName) = 0; /// Result of verification. enum VerificationResult { diff --git a/tools/hls-fuzzer/CMakeLists.txt b/tools/hls-fuzzer/CMakeLists.txt index 007403cb3e..06f3acdf94 100644 --- a/tools/hls-fuzzer/CMakeLists.txt +++ b/tools/hls-fuzzer/CMakeLists.txt @@ -37,3 +37,5 @@ target_link_libraries(hls-fuzzer PRIVATE MLIRSupport DynamaticHLSFuzzer) target_include_directories(hls-fuzzer PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) # Also keep dynamatic up-to-date such that we are always testing the up-to-date verison. add_dependencies(hls-fuzzer DynamaticHLSFuzzerOptsTableGen dynamatic) + +add_subdirectory(oracles) diff --git a/tools/hls-fuzzer/Options.h b/tools/hls-fuzzer/Options.h index 76b383d60d..c78b1d6b99 100644 --- a/tools/hls-fuzzer/Options.h +++ b/tools/hls-fuzzer/Options.h @@ -5,8 +5,16 @@ namespace dynamatic { +enum class OracleKind { + Functional, + NonFunctional, +}; + struct Options { - std::string dynamaticPath; + // Path of this executable. + std::string executablePath; + std::string dynamaticExecutablePath; + OracleKind kind = OracleKind::Functional; }; } // namespace dynamatic diff --git a/tools/hls-fuzzer/OptionsParser.cpp b/tools/hls-fuzzer/OptionsParser.cpp index c7d439f26f..15df07c0dc 100644 --- a/tools/hls-fuzzer/OptionsParser.cpp +++ b/tools/hls-fuzzer/OptionsParser.cpp @@ -69,7 +69,11 @@ dynamatic::OptionsParser::getPositionalArguments() const { dynamatic::Options dynamatic::OptionsParser::apply(Options defaults) { std::vector arguments = getPositionalArguments(); if (arguments.size() == 1) - defaults.dynamaticPath = getPositionalArguments()[0]; + defaults.dynamaticExecutablePath = getPositionalArguments()[0]; + defaults.kind = args.hasFlag(OPT_functional, OPT_non_functional, + defaults.kind == OracleKind::Functional) + ? OracleKind::Functional + : OracleKind::NonFunctional; return defaults; } diff --git a/tools/hls-fuzzer/Opts.td b/tools/hls-fuzzer/Opts.td index bda2f61c56..b05884d749 100644 --- a/tools/hls-fuzzer/Opts.td +++ b/tools/hls-fuzzer/Opts.td @@ -7,3 +7,8 @@ def num_threads : JoinedOrSeparate<["-"], "j">, MetaVarName<"">, HelpText<"Number of threads to use">; def help : Flag<["--"], "help">, HelpText<"displays this help text">; + +def grp_oracle : OptionGroup<"Oracle options">; + +def functional : Flag<["--"], "functional">, HelpText<"Switch to functional testing">, Group; +def non_functional : Flag<["--"], "non-functional">, HelpText<"Switch to non-functional testing">, Group; diff --git a/tools/hls-fuzzer/hls-fuzzer.cpp b/tools/hls-fuzzer/hls-fuzzer.cpp index 291ffab1fd..9817db0140 100644 --- a/tools/hls-fuzzer/hls-fuzzer.cpp +++ b/tools/hls-fuzzer/hls-fuzzer.cpp @@ -13,6 +13,7 @@ #include "TargetRegistry.h" #include "llvm/DebugInfo/LogicalView/Core/LVOptions.h" +#include "llvm/Support/FileSystem.h" static std::atomic_bool quit = false; static std::mutex errorMutex; @@ -98,7 +99,15 @@ int main(int argc, char **argv) { return -1; } - auto options = optionsParser.apply(dynamatic::Options{}); + dynamatic::Options defaults{}; +#pragma clang diagnostic ignored "-Wmain" + defaults.executablePath = llvm::sys::fs::getMainExecutable( + argv[0], reinterpret_cast(&main)); + defaults.dynamaticExecutablePath = + std::filesystem::path(defaults.executablePath).parent_path() / + "dynamatic"; + + auto options = optionsParser.apply(defaults); std::optional numThreads = optionsParser.getNumThreads(); if (!numThreads) diff --git a/tools/hls-fuzzer/oracles/CMakeLists.txt b/tools/hls-fuzzer/oracles/CMakeLists.txt new file mode 100644 index 0000000000..33cfdb6f19 --- /dev/null +++ b/tools/hls-fuzzer/oracles/CMakeLists.txt @@ -0,0 +1,7 @@ +add_llvm_executable(hls-fuzzer-check-bitwidth + check-bitwidth.cpp +) +llvm_update_compile_flags(hls-fuzzer-check-bitwidth) +target_link_libraries(hls-fuzzer-check-bitwidth PRIVATE DynamaticHandshake MLIRParser) + +add_dependencies(hls-fuzzer hls-fuzzer-check-bitwidth) diff --git a/tools/hls-fuzzer/oracles/check-bitwidth.cpp b/tools/hls-fuzzer/oracles/check-bitwidth.cpp new file mode 100644 index 0000000000..09fb238ccf --- /dev/null +++ b/tools/hls-fuzzer/oracles/check-bitwidth.cpp @@ -0,0 +1,69 @@ + +#include "dynamatic/Dialect/Handshake/HandshakeDialect.h" +#include "mlir/Tools/ParseUtilities.h" + +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/raw_ostream.h" + +#include +#include + +using namespace mlir; +using namespace dynamatic; + +int main(int argc, char **argv) { + if (argc != 3) { + llvm::errs() << "expected exactly two arguments\n"; + return -1; + } + + StringRef mlirFile = argv[1]; + StringRef bitwidthArg = argv[2]; + std::uint32_t bitWidth; + if (bitwidthArg.getAsInteger(0, bitWidth)) { + llvm::errs() << "expected an integer as second argument\n"; + return -1; + } + + llvm::ErrorOr> buffer = + llvm::MemoryBuffer::getFileOrSTDIN(mlirFile, true); + if (!buffer) { + llvm::errs() << "failed to open" << mlirFile << "\n"; + return -1; + } + + auto sourceMgr = std::make_shared(); + sourceMgr->AddNewSourceBuffer(std::move(*buffer), SMLoc()); + DialectRegistry registry; + registry.insert(); + MLIRContext context(registry); + ParserConfig config(&context); + OwningOpRef module = + parseSourceFileForTool(sourceMgr, config, true); + + WalkResult result = module->walk([&](Operation *op) { + for (Value iter : op->getResults()) { + auto channelType = dyn_cast(iter.getType()); + if (!channelType || !isa(channelType.getDataType())) + continue; + + // Results that feed into 'end' ops are allowed to use a higher bitwidth + // as it is required for interface conformance. + if (llvm::any_of(iter.getUsers(), + [](Operation *op) { return isa(op); })) + continue; + + if (channelType.getDataBitWidth() > bitWidth) { + op->emitError("expected computation with at most a bitwidth of '") + << bitWidth << "' rather than '" << channelType.getDataBitWidth() + << "'"; + return WalkResult::interrupt(); + } + } + return WalkResult::advance(); + }); + if (result.wasInterrupted()) + return -1; + + return 0; +} diff --git a/tools/hls-fuzzer/targets/BitwidthOptimizationsTarget.cpp b/tools/hls-fuzzer/targets/BitwidthOptimizationsTarget.cpp index 7ed04f55d9..4024fb8602 100644 --- a/tools/hls-fuzzer/targets/BitwidthOptimizationsTarget.cpp +++ b/tools/hls-fuzzer/targets/BitwidthOptimizationsTarget.cpp @@ -22,11 +22,13 @@ class BitwidthOptimizationsGenerator : public AbstractWorker { Randomly &&random) : AbstractWorker(options, std::move(random)) {} - void generate(llvm::raw_ostream &os, - llvm::StringRef functionName) const override; + void generate(llvm::raw_ostream &os, llvm::StringRef functionName) override; VerificationResult verify(const std::filesystem::path &sourceFile) const override; + +private: + std::uint8_t maxBitwidth; }; } // namespace @@ -38,10 +40,10 @@ BitwidthOptimizationsTarget::createWorker(const Options &options, std::move(randomly)); } -void BitwidthOptimizationsGenerator::generate( - llvm::raw_ostream &os, llvm::StringRef functionName) const { +void BitwidthOptimizationsGenerator::generate(llvm::raw_ostream &os, + llvm::StringRef functionName) { // Enforce a strict bitwidth requirement for the entire program. - auto maxBitwidth = random.getInteger(1, 32); + maxBitwidth = random.getInteger(1, 32); gen::BitwidthTypeSystem bitwidthTypeSystem(maxBitwidth, random); gen::BasicCGenerator generator(random, bitwidthTypeSystem, /*entryContext=*/ @@ -60,5 +62,28 @@ void BitwidthOptimizationsGenerator::generate( AbstractWorker::VerificationResult BitwidthOptimizationsGenerator::verify( const std::filesystem::path &sourceFile) const { - return performDifferentialTesting(sourceFile, options.dynamaticPath); + switch (options.kind) { + case OracleKind::Functional: + return performDifferentialTesting(sourceFile, + options.dynamaticExecutablePath); + case OracleKind::NonFunctional: + break; + } + + std::filesystem::path parentPath = sourceFile.parent_path(); + std::string executeFile = (parentPath / "execute.sh").string(); + llvm::cantFail(llvm::writeToOutput( + executeFile, [&](llvm::raw_ostream &os) -> llvm::Error { + os << "set -e\n"; + outputDynamaticInvocation(os, sourceFile, + options.dynamaticExecutablePath, R"( +compile +)"); + os << (std::filesystem::path(options.executablePath).parent_path() / + "hls-fuzzer-check-bitwidth") + << " ./out/comp/handshake_export.mlir " + << static_cast(maxBitwidth) << '\n'; + return llvm::Error::success(); + })); + return executeInWorkingDirectory(parentPath, "bash execute.sh"); } diff --git a/tools/hls-fuzzer/targets/BitwidthTypeSystem.cpp b/tools/hls-fuzzer/targets/BitwidthTypeSystem.cpp index f7cda3702b..1e99020983 100644 --- a/tools/hls-fuzzer/targets/BitwidthTypeSystem.cpp +++ b/tools/hls-fuzzer/targets/BitwidthTypeSystem.cpp @@ -89,6 +89,8 @@ auto dynamatic::gen::BitwidthTypeSystem::checkBinaryExpression( return std::nullopt; case ast::BinaryExpression::ShiftRight: + // TODO: Figure out constraints here. + return std::nullopt; case ast::BinaryExpression::Greater: case ast::BinaryExpression::GreaterEqual: case ast::BinaryExpression::Less: diff --git a/tools/hls-fuzzer/targets/RandomCTarget.cpp b/tools/hls-fuzzer/targets/RandomCTarget.cpp index ba91ee2a5b..8e05022ccd 100644 --- a/tools/hls-fuzzer/targets/RandomCTarget.cpp +++ b/tools/hls-fuzzer/targets/RandomCTarget.cpp @@ -16,7 +16,7 @@ class RandomCWorker : public AbstractWorker { : AbstractWorker(options, std::move(random)) {} void generate(llvm::raw_ostream &os, - llvm::StringRef functionName) const override; + llvm::StringRef functionName) override; VerificationResult verify(const std::filesystem::path &sourceFile) const override; @@ -30,7 +30,7 @@ RandomCTarget::createWorker(const Options &options, Randomly randomly) const { } void RandomCWorker::generate(llvm::raw_ostream &os, - llvm::StringRef functionName) const { + llvm::StringRef functionName) { gen::DynamaticTypeSystem dynamaticTypeSystem(random); gen::BasicCGenerator generator( random, dynamaticTypeSystem, @@ -50,5 +50,6 @@ void RandomCWorker::generate(llvm::raw_ostream &os, AbstractWorker::VerificationResult RandomCWorker::verify(const std::filesystem::path &sourceFile) const { - return performDifferentialTesting(sourceFile, options.dynamaticPath); + return performDifferentialTesting(sourceFile, + options.dynamaticExecutablePath); } diff --git a/tools/hls-fuzzer/targets/TargetUtils.cpp b/tools/hls-fuzzer/targets/TargetUtils.cpp index 3143d7aa1b..a68526d833 100644 --- a/tools/hls-fuzzer/targets/TargetUtils.cpp +++ b/tools/hls-fuzzer/targets/TargetUtils.cpp @@ -12,45 +12,60 @@ dynamatic::performDifferentialTesting(const std::filesystem::path &sourceFile, std::string executeFile = (parentPath / "execute.sh").string(); llvm::cantFail(llvm::writeToOutput( executeFile, [&](llvm::raw_ostream &os) -> llvm::Error { - os << dynamaticPath << " --exit-on-failure < llvm::Error { os << R"a(SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -cd $SCRIPT_DIR && bash )a" - << executeFile +cd $SCRIPT_DIR && )a" + << bashCommand // Canonicalize all error exists to exit code 1, even if dynamatic // crashed with e.g. SIGSEGV. We need this to differentiate between // bash exiting with a signal and dynamatic exiting with a signal. - << "|| exit 1\n"; + << " || exit 1\n"; return llvm::Error::success(); })); int exitCode = llvm::sys::ExecuteAndWait( "/usr/bin/bash", {"bash", executeCWDFile}, /*Env=*/std::nullopt, /*Redirects=*/{"", "", ""}); - switch (exitCode) { // Normal exit. case 0: diff --git a/tools/hls-fuzzer/targets/TargetUtils.h b/tools/hls-fuzzer/targets/TargetUtils.h index 4786bf89cd..50d4210f2b 100644 --- a/tools/hls-fuzzer/targets/TargetUtils.h +++ b/tools/hls-fuzzer/targets/TargetUtils.h @@ -17,6 +17,25 @@ AbstractWorker::VerificationResult performDifferentialTesting(const std::filesystem::path &sourceFile, llvm::StringRef dynamaticPath); +/// Outputs a bash commandline to 'os' that invokes dynamatic and executes the +/// given 'script'. +/// 'sourceFile' is the source file to be compiled while 'dynamaticPath' refers +/// to the path to the dynamatic executable. +/// 'script' can assume that the source file and dynamatic home are already +/// set. +void outputDynamaticInvocation(llvm::raw_ostream &os, + const std::filesystem::path &sourceFile, + llvm::StringRef dynamaticPath, + llvm::StringRef script); + +/// Executes a given 'bashCommand' in 'workingDirectory' by creating and +/// executing a script in 'workingDirectory'. +/// Normalizes the return code of the bash command to map failures to +/// 'VerificationResult::Bug'. +AbstractWorker::VerificationResult +executeInWorkingDirectory(const std::filesystem::path &workingDirectory, + llvm::StringRef bashCommand); + } // namespace dynamatic #endif From 9be56e57d19c8bb26205d843cf58708454c4a685 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20B=C3=B6ck?= Date: Mon, 23 Mar 2026 16:07:09 +0100 Subject: [PATCH 2/3] clang-format --- tools/hls-fuzzer/Opts.td | 8 ++++++-- tools/hls-fuzzer/targets/RandomCTarget.cpp | 3 +-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tools/hls-fuzzer/Opts.td b/tools/hls-fuzzer/Opts.td index b05884d749..9986861978 100644 --- a/tools/hls-fuzzer/Opts.td +++ b/tools/hls-fuzzer/Opts.td @@ -10,5 +10,9 @@ def help : Flag<["--"], "help">, HelpText<"displays this help text">; def grp_oracle : OptionGroup<"Oracle options">; -def functional : Flag<["--"], "functional">, HelpText<"Switch to functional testing">, Group; -def non_functional : Flag<["--"], "non-functional">, HelpText<"Switch to non-functional testing">, Group; +def functional : Flag<["--"], "functional">, + HelpText<"Switch to functional testing">, + Group; +def non_functional : Flag<["--"], "non-functional">, + HelpText<"Switch to non-functional testing">, + Group; diff --git a/tools/hls-fuzzer/targets/RandomCTarget.cpp b/tools/hls-fuzzer/targets/RandomCTarget.cpp index 8e05022ccd..87850eb1d1 100644 --- a/tools/hls-fuzzer/targets/RandomCTarget.cpp +++ b/tools/hls-fuzzer/targets/RandomCTarget.cpp @@ -15,8 +15,7 @@ class RandomCWorker : public AbstractWorker { explicit RandomCWorker(const Options &options, Randomly &&random) : AbstractWorker(options, std::move(random)) {} - void generate(llvm::raw_ostream &os, - llvm::StringRef functionName) override; + void generate(llvm::raw_ostream &os, llvm::StringRef functionName) override; VerificationResult verify(const std::filesystem::path &sourceFile) const override; From ca047a47644f5afdd2c06ee4f12182240e921e69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20B=C3=B6ck?= Date: Wed, 25 Mar 2026 10:13:00 +0100 Subject: [PATCH 3/3] address review comments --- .../targets/BitwidthOptimizationsTarget.cpp | 29 ++++++--------- tools/hls-fuzzer/targets/TargetUtils.cpp | 37 +++++++++++++++++-- tools/hls-fuzzer/targets/TargetUtils.h | 23 +++++++++++- 3 files changed, 66 insertions(+), 23 deletions(-) diff --git a/tools/hls-fuzzer/targets/BitwidthOptimizationsTarget.cpp b/tools/hls-fuzzer/targets/BitwidthOptimizationsTarget.cpp index 4024fb8602..ebc76aec38 100644 --- a/tools/hls-fuzzer/targets/BitwidthOptimizationsTarget.cpp +++ b/tools/hls-fuzzer/targets/BitwidthOptimizationsTarget.cpp @@ -60,6 +60,10 @@ void BitwidthOptimizationsGenerator::generate(llvm::raw_ostream &os, os << generator.generateTestBench(function); } +constexpr std::string_view ORACLE_EXECUTABLE = "hls-fuzzer-check-bitwidth"; +constexpr std::string_view COMPILATION_IR_OUTPUT = + "./out/comp/handshake_export.mlir"; + AbstractWorker::VerificationResult BitwidthOptimizationsGenerator::verify( const std::filesystem::path &sourceFile) const { switch (options.kind) { @@ -67,23 +71,12 @@ AbstractWorker::VerificationResult BitwidthOptimizationsGenerator::verify( return performDifferentialTesting(sourceFile, options.dynamaticExecutablePath); case OracleKind::NonFunctional: - break; + return performNonFunctionalTesting( + sourceFile, options.dynamaticExecutablePath, + (std::filesystem::path(options.executablePath).parent_path() / + ORACLE_EXECUTABLE) + .string(), + {COMPILATION_IR_OUTPUT, std::to_string(maxBitwidth)}); } - - std::filesystem::path parentPath = sourceFile.parent_path(); - std::string executeFile = (parentPath / "execute.sh").string(); - llvm::cantFail(llvm::writeToOutput( - executeFile, [&](llvm::raw_ostream &os) -> llvm::Error { - os << "set -e\n"; - outputDynamaticInvocation(os, sourceFile, - options.dynamaticExecutablePath, R"( -compile -)"); - os << (std::filesystem::path(options.executablePath).parent_path() / - "hls-fuzzer-check-bitwidth") - << " ./out/comp/handshake_export.mlir " - << static_cast(maxBitwidth) << '\n'; - return llvm::Error::success(); - })); - return executeInWorkingDirectory(parentPath, "bash execute.sh"); + llvm_unreachable("all enum cases handled"); } diff --git a/tools/hls-fuzzer/targets/TargetUtils.cpp b/tools/hls-fuzzer/targets/TargetUtils.cpp index a68526d833..024e88ee35 100644 --- a/tools/hls-fuzzer/targets/TargetUtils.cpp +++ b/tools/hls-fuzzer/targets/TargetUtils.cpp @@ -3,13 +3,16 @@ #include "llvm/Support/Error.h" #include "llvm/Support/Program.h" +constexpr std::string_view EXECUTE_SCRIPT = "execute.sh"; +constexpr std::string_view SHELL = "bash"; + dynamatic::AbstractWorker::VerificationResult dynamatic::performDifferentialTesting(const std::filesystem::path &sourceFile, llvm::StringRef dynamaticPath) { // Create an 'execute.sh' that can additionally be used as a nice reproducer // for e.g. 'cvise'. std::filesystem::path parentPath = sourceFile.parent_path(); - std::string executeFile = (parentPath / "execute.sh").string(); + std::string executeFile = (parentPath / EXECUTE_SCRIPT).string(); llvm::cantFail(llvm::writeToOutput( executeFile, [&](llvm::raw_ostream &os) -> llvm::Error { outputDynamaticInvocation(os, sourceFile, dynamaticPath, R"( @@ -19,7 +22,33 @@ simulate )"); return llvm::Error::success(); })); - return executeInWorkingDirectory(parentPath, "bash execute.sh"); + return executeInWorkingDirectory(parentPath, + llvm::Twine(SHELL) + " " + EXECUTE_SCRIPT); +} + +dynamatic::AbstractWorker::VerificationResult +dynamatic::performNonFunctionalTesting( + const std::filesystem::path &sourceFile, llvm::StringRef dynamaticPath, + llvm::StringRef oracleExecutable, + llvm::ArrayRef arguments) { + std::filesystem::path parentPath = sourceFile.parent_path(); + + std::string executeFile = (parentPath / EXECUTE_SCRIPT).string(); + llvm::cantFail(llvm::writeToOutput( + executeFile, [&](llvm::raw_ostream &os) -> llvm::Error { + os << "set -e\n"; + outputDynamaticInvocation(os, sourceFile, dynamaticPath, R"( +compile +)"); + os << oracleExecutable; + for (llvm::StringRef iter : arguments) + os << " " << iter; + + os << '\n'; + return llvm::Error::success(); + })); + return executeInWorkingDirectory(parentPath, + llvm::Twine(SHELL) + " " + EXECUTE_SCRIPT); } void dynamatic::outputDynamaticInvocation( @@ -44,7 +73,7 @@ void dynamatic::outputDynamaticInvocation( dynamatic::AbstractWorker::VerificationResult dynamatic::executeInWorkingDirectory( const std::filesystem::path &workingDirectory, - llvm::StringRef bashCommand) { + const llvm::Twine &bashCommand) { // LLVM's process creation does not support changing the current working // directory. We require this since dynamatic creates many of its artifacts @@ -64,7 +93,7 @@ cd $SCRIPT_DIR && )a" })); int exitCode = llvm::sys::ExecuteAndWait( - "/usr/bin/bash", {"bash", executeCWDFile}, /*Env=*/std::nullopt, + "/usr/bin/bash", {SHELL, executeCWDFile}, /*Env=*/std::nullopt, /*Redirects=*/{"", "", ""}); switch (exitCode) { // Normal exit. diff --git a/tools/hls-fuzzer/targets/TargetUtils.h b/tools/hls-fuzzer/targets/TargetUtils.h index 50d4210f2b..dda5b5f655 100644 --- a/tools/hls-fuzzer/targets/TargetUtils.h +++ b/tools/hls-fuzzer/targets/TargetUtils.h @@ -3,6 +3,8 @@ #include "hls-fuzzer/AbstractWorker.h" +#include "llvm/ADT/Twine.h" + namespace dynamatic { /// Performs differential testing of a C file called 'sourceFile'. @@ -17,6 +19,25 @@ AbstractWorker::VerificationResult performDifferentialTesting(const std::filesystem::path &sourceFile, llvm::StringRef dynamaticPath); +/// Performs non-functional testing on the compilation of a C file called +/// 'sourceFile'. +/// The source file is compiled to MLIR and the compilation output checked +/// using 'oracleExecutable'. +/// 'oracleExecutable' should return exit code 0 if the output is as expected, +/// any other value otherwise. +/// 'arguments' can be used to supply extra arguments to the 'oracleExecutable'. +/// The invocation of 'oracleExecutable' is in the same working directory as the +/// dynamatic invocation. +/// +/// 'dynamaticPath' should refer to where the dynamatic executable. +/// The directory of 'sourceFile' is assumed to be scratch space used for build +/// artifacts. +AbstractWorker::VerificationResult +performNonFunctionalTesting(const std::filesystem::path &sourceFile, + llvm::StringRef dynamaticPath, + llvm::StringRef oracleExecutable, + llvm::ArrayRef arguments); + /// Outputs a bash commandline to 'os' that invokes dynamatic and executes the /// given 'script'. /// 'sourceFile' is the source file to be compiled while 'dynamaticPath' refers @@ -34,7 +55,7 @@ void outputDynamaticInvocation(llvm::raw_ostream &os, /// 'VerificationResult::Bug'. AbstractWorker::VerificationResult executeInWorkingDirectory(const std::filesystem::path &workingDirectory, - llvm::StringRef bashCommand); + const llvm::Twine &bashCommand); } // namespace dynamatic