diff --git a/tools/hls-fuzzer/AbstractWorker.h b/tools/hls-fuzzer/AbstractWorker.h index 6e60297d91..121a00e8d8 100644 --- a/tools/hls-fuzzer/AbstractWorker.h +++ b/tools/hls-fuzzer/AbstractWorker.h @@ -21,7 +21,7 @@ class AbstractWorker { /// Creates a random C program that should be written to 'os'. /// 'os' writes to a file called 'functionName.c'. virtual void generate(llvm::raw_ostream &os, - llvm::StringRef functionName) const = 0; + llvm::StringRef functionName) = 0; /// Result of verification. enum VerificationResult { diff --git a/tools/hls-fuzzer/CMakeLists.txt b/tools/hls-fuzzer/CMakeLists.txt index 007403cb3e..06f3acdf94 100644 --- a/tools/hls-fuzzer/CMakeLists.txt +++ b/tools/hls-fuzzer/CMakeLists.txt @@ -37,3 +37,5 @@ target_link_libraries(hls-fuzzer PRIVATE MLIRSupport DynamaticHLSFuzzer) target_include_directories(hls-fuzzer PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) # Also keep dynamatic up-to-date such that we are always testing the up-to-date verison. add_dependencies(hls-fuzzer DynamaticHLSFuzzerOptsTableGen dynamatic) + +add_subdirectory(oracles) diff --git a/tools/hls-fuzzer/Options.h b/tools/hls-fuzzer/Options.h index 76b383d60d..c78b1d6b99 100644 --- a/tools/hls-fuzzer/Options.h +++ b/tools/hls-fuzzer/Options.h @@ -5,8 +5,16 @@ namespace dynamatic { +enum class OracleKind { + Functional, + NonFunctional, +}; + struct Options { - std::string dynamaticPath; + // Path of this executable. + std::string executablePath; + std::string dynamaticExecutablePath; + OracleKind kind = OracleKind::Functional; }; } // namespace dynamatic diff --git a/tools/hls-fuzzer/OptionsParser.cpp b/tools/hls-fuzzer/OptionsParser.cpp index c7d439f26f..15df07c0dc 100644 --- a/tools/hls-fuzzer/OptionsParser.cpp +++ b/tools/hls-fuzzer/OptionsParser.cpp @@ -69,7 +69,11 @@ dynamatic::OptionsParser::getPositionalArguments() const { dynamatic::Options dynamatic::OptionsParser::apply(Options defaults) { std::vector arguments = getPositionalArguments(); if (arguments.size() == 1) - defaults.dynamaticPath = getPositionalArguments()[0]; + defaults.dynamaticExecutablePath = getPositionalArguments()[0]; + defaults.kind = args.hasFlag(OPT_functional, OPT_non_functional, + defaults.kind == OracleKind::Functional) + ? OracleKind::Functional + : OracleKind::NonFunctional; return defaults; } diff --git a/tools/hls-fuzzer/Opts.td b/tools/hls-fuzzer/Opts.td index bda2f61c56..9986861978 100644 --- a/tools/hls-fuzzer/Opts.td +++ b/tools/hls-fuzzer/Opts.td @@ -7,3 +7,12 @@ def num_threads : JoinedOrSeparate<["-"], "j">, MetaVarName<"">, HelpText<"Number of threads to use">; def help : Flag<["--"], "help">, HelpText<"displays this help text">; + +def grp_oracle : OptionGroup<"Oracle options">; + +def functional : Flag<["--"], "functional">, + HelpText<"Switch to functional testing">, + Group; +def non_functional : Flag<["--"], "non-functional">, + HelpText<"Switch to non-functional testing">, + Group; diff --git a/tools/hls-fuzzer/hls-fuzzer.cpp b/tools/hls-fuzzer/hls-fuzzer.cpp index 291ffab1fd..9817db0140 100644 --- a/tools/hls-fuzzer/hls-fuzzer.cpp +++ b/tools/hls-fuzzer/hls-fuzzer.cpp @@ -13,6 +13,7 @@ #include "TargetRegistry.h" #include "llvm/DebugInfo/LogicalView/Core/LVOptions.h" +#include "llvm/Support/FileSystem.h" static std::atomic_bool quit = false; static std::mutex errorMutex; @@ -98,7 +99,15 @@ int main(int argc, char **argv) { return -1; } - auto options = optionsParser.apply(dynamatic::Options{}); + dynamatic::Options defaults{}; +#pragma clang diagnostic ignored "-Wmain" + defaults.executablePath = llvm::sys::fs::getMainExecutable( + argv[0], reinterpret_cast(&main)); + defaults.dynamaticExecutablePath = + std::filesystem::path(defaults.executablePath).parent_path() / + "dynamatic"; + + auto options = optionsParser.apply(defaults); std::optional numThreads = optionsParser.getNumThreads(); if (!numThreads) diff --git a/tools/hls-fuzzer/oracles/CMakeLists.txt b/tools/hls-fuzzer/oracles/CMakeLists.txt new file mode 100644 index 0000000000..33cfdb6f19 --- /dev/null +++ b/tools/hls-fuzzer/oracles/CMakeLists.txt @@ -0,0 +1,7 @@ +add_llvm_executable(hls-fuzzer-check-bitwidth + check-bitwidth.cpp +) +llvm_update_compile_flags(hls-fuzzer-check-bitwidth) +target_link_libraries(hls-fuzzer-check-bitwidth PRIVATE DynamaticHandshake MLIRParser) + +add_dependencies(hls-fuzzer hls-fuzzer-check-bitwidth) diff --git a/tools/hls-fuzzer/oracles/check-bitwidth.cpp b/tools/hls-fuzzer/oracles/check-bitwidth.cpp new file mode 100644 index 0000000000..09fb238ccf --- /dev/null +++ b/tools/hls-fuzzer/oracles/check-bitwidth.cpp @@ -0,0 +1,69 @@ + +#include "dynamatic/Dialect/Handshake/HandshakeDialect.h" +#include "mlir/Tools/ParseUtilities.h" + +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/raw_ostream.h" + +#include +#include + +using namespace mlir; +using namespace dynamatic; + +int main(int argc, char **argv) { + if (argc != 3) { + llvm::errs() << "expected exactly two arguments\n"; + return -1; + } + + StringRef mlirFile = argv[1]; + StringRef bitwidthArg = argv[2]; + std::uint32_t bitWidth; + if (bitwidthArg.getAsInteger(0, bitWidth)) { + llvm::errs() << "expected an integer as second argument\n"; + return -1; + } + + llvm::ErrorOr> buffer = + llvm::MemoryBuffer::getFileOrSTDIN(mlirFile, true); + if (!buffer) { + llvm::errs() << "failed to open" << mlirFile << "\n"; + return -1; + } + + auto sourceMgr = std::make_shared(); + sourceMgr->AddNewSourceBuffer(std::move(*buffer), SMLoc()); + DialectRegistry registry; + registry.insert(); + MLIRContext context(registry); + ParserConfig config(&context); + OwningOpRef module = + parseSourceFileForTool(sourceMgr, config, true); + + WalkResult result = module->walk([&](Operation *op) { + for (Value iter : op->getResults()) { + auto channelType = dyn_cast(iter.getType()); + if (!channelType || !isa(channelType.getDataType())) + continue; + + // Results that feed into 'end' ops are allowed to use a higher bitwidth + // as it is required for interface conformance. + if (llvm::any_of(iter.getUsers(), + [](Operation *op) { return isa(op); })) + continue; + + if (channelType.getDataBitWidth() > bitWidth) { + op->emitError("expected computation with at most a bitwidth of '") + << bitWidth << "' rather than '" << channelType.getDataBitWidth() + << "'"; + return WalkResult::interrupt(); + } + } + return WalkResult::advance(); + }); + if (result.wasInterrupted()) + return -1; + + return 0; +} diff --git a/tools/hls-fuzzer/targets/BitwidthOptimizationsTarget.cpp b/tools/hls-fuzzer/targets/BitwidthOptimizationsTarget.cpp index 7ed04f55d9..ebc76aec38 100644 --- a/tools/hls-fuzzer/targets/BitwidthOptimizationsTarget.cpp +++ b/tools/hls-fuzzer/targets/BitwidthOptimizationsTarget.cpp @@ -22,11 +22,13 @@ class BitwidthOptimizationsGenerator : public AbstractWorker { Randomly &&random) : AbstractWorker(options, std::move(random)) {} - void generate(llvm::raw_ostream &os, - llvm::StringRef functionName) const override; + void generate(llvm::raw_ostream &os, llvm::StringRef functionName) override; VerificationResult verify(const std::filesystem::path &sourceFile) const override; + +private: + std::uint8_t maxBitwidth; }; } // namespace @@ -38,10 +40,10 @@ BitwidthOptimizationsTarget::createWorker(const Options &options, std::move(randomly)); } -void BitwidthOptimizationsGenerator::generate( - llvm::raw_ostream &os, llvm::StringRef functionName) const { +void BitwidthOptimizationsGenerator::generate(llvm::raw_ostream &os, + llvm::StringRef functionName) { // Enforce a strict bitwidth requirement for the entire program. - auto maxBitwidth = random.getInteger(1, 32); + maxBitwidth = random.getInteger(1, 32); gen::BitwidthTypeSystem bitwidthTypeSystem(maxBitwidth, random); gen::BasicCGenerator generator(random, bitwidthTypeSystem, /*entryContext=*/ @@ -58,7 +60,23 @@ void BitwidthOptimizationsGenerator::generate( os << generator.generateTestBench(function); } +constexpr std::string_view ORACLE_EXECUTABLE = "hls-fuzzer-check-bitwidth"; +constexpr std::string_view COMPILATION_IR_OUTPUT = + "./out/comp/handshake_export.mlir"; + AbstractWorker::VerificationResult BitwidthOptimizationsGenerator::verify( const std::filesystem::path &sourceFile) const { - return performDifferentialTesting(sourceFile, options.dynamaticPath); + switch (options.kind) { + case OracleKind::Functional: + return performDifferentialTesting(sourceFile, + options.dynamaticExecutablePath); + case OracleKind::NonFunctional: + return performNonFunctionalTesting( + sourceFile, options.dynamaticExecutablePath, + (std::filesystem::path(options.executablePath).parent_path() / + ORACLE_EXECUTABLE) + .string(), + {COMPILATION_IR_OUTPUT, std::to_string(maxBitwidth)}); + } + llvm_unreachable("all enum cases handled"); } diff --git a/tools/hls-fuzzer/targets/BitwidthTypeSystem.cpp b/tools/hls-fuzzer/targets/BitwidthTypeSystem.cpp index f7cda3702b..1e99020983 100644 --- a/tools/hls-fuzzer/targets/BitwidthTypeSystem.cpp +++ b/tools/hls-fuzzer/targets/BitwidthTypeSystem.cpp @@ -89,6 +89,8 @@ auto dynamatic::gen::BitwidthTypeSystem::checkBinaryExpression( return std::nullopt; case ast::BinaryExpression::ShiftRight: + // TODO: Figure out constraints here. + return std::nullopt; case ast::BinaryExpression::Greater: case ast::BinaryExpression::GreaterEqual: case ast::BinaryExpression::Less: diff --git a/tools/hls-fuzzer/targets/RandomCTarget.cpp b/tools/hls-fuzzer/targets/RandomCTarget.cpp index ba91ee2a5b..87850eb1d1 100644 --- a/tools/hls-fuzzer/targets/RandomCTarget.cpp +++ b/tools/hls-fuzzer/targets/RandomCTarget.cpp @@ -15,8 +15,7 @@ class RandomCWorker : public AbstractWorker { explicit RandomCWorker(const Options &options, Randomly &&random) : AbstractWorker(options, std::move(random)) {} - void generate(llvm::raw_ostream &os, - llvm::StringRef functionName) const override; + void generate(llvm::raw_ostream &os, llvm::StringRef functionName) override; VerificationResult verify(const std::filesystem::path &sourceFile) const override; @@ -30,7 +29,7 @@ RandomCTarget::createWorker(const Options &options, Randomly randomly) const { } void RandomCWorker::generate(llvm::raw_ostream &os, - llvm::StringRef functionName) const { + llvm::StringRef functionName) { gen::DynamaticTypeSystem dynamaticTypeSystem(random); gen::BasicCGenerator generator( random, dynamaticTypeSystem, @@ -50,5 +49,6 @@ void RandomCWorker::generate(llvm::raw_ostream &os, AbstractWorker::VerificationResult RandomCWorker::verify(const std::filesystem::path &sourceFile) const { - return performDifferentialTesting(sourceFile, options.dynamaticPath); + return performDifferentialTesting(sourceFile, + options.dynamaticExecutablePath); } diff --git a/tools/hls-fuzzer/targets/TargetUtils.cpp b/tools/hls-fuzzer/targets/TargetUtils.cpp index 3143d7aa1b..024e88ee35 100644 --- a/tools/hls-fuzzer/targets/TargetUtils.cpp +++ b/tools/hls-fuzzer/targets/TargetUtils.cpp @@ -3,54 +3,98 @@ #include "llvm/Support/Error.h" #include "llvm/Support/Program.h" +constexpr std::string_view EXECUTE_SCRIPT = "execute.sh"; +constexpr std::string_view SHELL = "bash"; + dynamatic::AbstractWorker::VerificationResult dynamatic::performDifferentialTesting(const std::filesystem::path &sourceFile, llvm::StringRef dynamaticPath) { // Create an 'execute.sh' that can additionally be used as a nice reproducer // for e.g. 'cvise'. std::filesystem::path parentPath = sourceFile.parent_path(); - std::string executeFile = (parentPath / "execute.sh").string(); + std::string executeFile = (parentPath / EXECUTE_SCRIPT).string(); llvm::cantFail(llvm::writeToOutput( executeFile, [&](llvm::raw_ostream &os) -> llvm::Error { - os << dynamaticPath << " --exit-on-failure < arguments) { + std::filesystem::path parentPath = sourceFile.parent_path(); + + std::string executeFile = (parentPath / EXECUTE_SCRIPT).string(); + llvm::cantFail(llvm::writeToOutput( + executeFile, [&](llvm::raw_ostream &os) -> llvm::Error { + os << "set -e\n"; + outputDynamaticInvocation(os, sourceFile, dynamaticPath, R"( +compile +)"); + os << oracleExecutable; + for (llvm::StringRef iter : arguments) + os << " " << iter; + + os << '\n'; return llvm::Error::success(); })); + return executeInWorkingDirectory(parentPath, + llvm::Twine(SHELL) + " " + EXECUTE_SCRIPT); +} + +void dynamatic::outputDynamaticInvocation( + llvm::raw_ostream &os, const std::filesystem::path &sourceFile, + llvm::StringRef dynamaticPath, llvm::StringRef script) { + // Compute the dynamatic home path by assuming it's a parent directory of the + // dynamatic executable and contains the scripts directory used to implement + // the various commands. + std::filesystem::path dynamaticSourceRoot = dynamaticPath.str(); + while (!dynamaticSourceRoot.empty()) { + dynamaticSourceRoot = dynamaticSourceRoot.parent_path(); + if (exists(dynamaticSourceRoot / "tools" / "dynamatic" / "scripts")) + break; + } + + os << dynamaticPath << " --exit-on-failure < llvm::Error { os << R"a(SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -cd $SCRIPT_DIR && bash )a" - << executeFile +cd $SCRIPT_DIR && )a" + << bashCommand // Canonicalize all error exists to exit code 1, even if dynamatic // crashed with e.g. SIGSEGV. We need this to differentiate between // bash exiting with a signal and dynamatic exiting with a signal. - << "|| exit 1\n"; + << " || exit 1\n"; return llvm::Error::success(); })); int exitCode = llvm::sys::ExecuteAndWait( - "/usr/bin/bash", {"bash", executeCWDFile}, /*Env=*/std::nullopt, + "/usr/bin/bash", {SHELL, executeCWDFile}, /*Env=*/std::nullopt, /*Redirects=*/{"", "", ""}); - switch (exitCode) { // Normal exit. case 0: diff --git a/tools/hls-fuzzer/targets/TargetUtils.h b/tools/hls-fuzzer/targets/TargetUtils.h index 4786bf89cd..dda5b5f655 100644 --- a/tools/hls-fuzzer/targets/TargetUtils.h +++ b/tools/hls-fuzzer/targets/TargetUtils.h @@ -3,6 +3,8 @@ #include "hls-fuzzer/AbstractWorker.h" +#include "llvm/ADT/Twine.h" + namespace dynamatic { /// Performs differential testing of a C file called 'sourceFile'. @@ -17,6 +19,44 @@ AbstractWorker::VerificationResult performDifferentialTesting(const std::filesystem::path &sourceFile, llvm::StringRef dynamaticPath); +/// Performs non-functional testing on the compilation of a C file called +/// 'sourceFile'. +/// The source file is compiled to MLIR and the compilation output checked +/// using 'oracleExecutable'. +/// 'oracleExecutable' should return exit code 0 if the output is as expected, +/// any other value otherwise. +/// 'arguments' can be used to supply extra arguments to the 'oracleExecutable'. +/// The invocation of 'oracleExecutable' is in the same working directory as the +/// dynamatic invocation. +/// +/// 'dynamaticPath' should refer to where the dynamatic executable. +/// The directory of 'sourceFile' is assumed to be scratch space used for build +/// artifacts. +AbstractWorker::VerificationResult +performNonFunctionalTesting(const std::filesystem::path &sourceFile, + llvm::StringRef dynamaticPath, + llvm::StringRef oracleExecutable, + llvm::ArrayRef arguments); + +/// Outputs a bash commandline to 'os' that invokes dynamatic and executes the +/// given 'script'. +/// 'sourceFile' is the source file to be compiled while 'dynamaticPath' refers +/// to the path to the dynamatic executable. +/// 'script' can assume that the source file and dynamatic home are already +/// set. +void outputDynamaticInvocation(llvm::raw_ostream &os, + const std::filesystem::path &sourceFile, + llvm::StringRef dynamaticPath, + llvm::StringRef script); + +/// Executes a given 'bashCommand' in 'workingDirectory' by creating and +/// executing a script in 'workingDirectory'. +/// Normalizes the return code of the bash command to map failures to +/// 'VerificationResult::Bug'. +AbstractWorker::VerificationResult +executeInWorkingDirectory(const std::filesystem::path &workingDirectory, + const llvm::Twine &bashCommand); + } // namespace dynamatic #endif