[hls-fuzzer] Add non-functional fuzzer to bitwidth optimizations

tools/hls-fuzzer/AbstractWorker.h

@@ -21,7 +21,7 @@ class AbstractWorker {
   /// Creates a random C program that should be written to 'os'.
   /// 'os' writes to a file called 'functionName.c'.
   virtual void generate(llvm::raw_ostream &os,
-                        llvm::StringRef functionName) const = 0;
+                        llvm::StringRef functionName) = 0;
 
   /// Result of verification.
   enum VerificationResult {

tools/hls-fuzzer/CMakeLists.txt

@@ -37,3 +37,5 @@ target_link_libraries(hls-fuzzer PRIVATE MLIRSupport DynamaticHLSFuzzer)
 target_include_directories(hls-fuzzer PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
 # Also keep dynamatic up-to-date such that we are always testing the up-to-date verison.
 add_dependencies(hls-fuzzer DynamaticHLSFuzzerOptsTableGen dynamatic)
+
+add_subdirectory(oracles)

tools/hls-fuzzer/Options.h

@@ -5,8 +5,16 @@
 
 namespace dynamatic {
 
+enum class OracleKind {
+  Functional,
+  NonFunctional,
+};
+
 struct Options {
-  std::string dynamaticPath;
+  // Path of this executable.
+  std::string executablePath;
+  std::string dynamaticExecutablePath;
+  OracleKind kind = OracleKind::Functional;
 };
 
 } // namespace dynamatic

tools/hls-fuzzer/OptionsParser.cpp

@@ -69,7 +69,11 @@ dynamatic::OptionsParser::getPositionalArguments() const {
 dynamatic::Options dynamatic::OptionsParser::apply(Options defaults) {
   std::vector<std::string> arguments = getPositionalArguments();
   if (arguments.size() == 1)
-    defaults.dynamaticPath = getPositionalArguments()[0];
+    defaults.dynamaticExecutablePath = getPositionalArguments()[0];
 
+  defaults.kind = args.hasFlag(OPT_functional, OPT_non_functional,
+                               defaults.kind == OracleKind::Functional)
+                      ? OracleKind::Functional
+                      : OracleKind::NonFunctional;
   return defaults;
 }

tools/hls-fuzzer/Opts.td

@@ -7,3 +7,12 @@ def num_threads : JoinedOrSeparate<["-"], "j">,
                   MetaVarName<"<num-threads>">,
                   HelpText<"Number of threads to use">;
 def help : Flag<["--"], "help">, HelpText<"displays this help text">;
+
+def grp_oracle : OptionGroup<"Oracle options">;
+
+def functional : Flag<["--"], "functional">,
+                 HelpText<"Switch to functional testing">,
+                 Group<grp_oracle>;
+def non_functional : Flag<["--"], "non-functional">,
+                     HelpText<"Switch to non-functional testing">,
+                     Group<grp_oracle>;

tools/hls-fuzzer/hls-fuzzer.cpp

@@ -13,6 +13,7 @@
 #include "TargetRegistry.h"
 
 #include "llvm/DebugInfo/LogicalView/Core/LVOptions.h"
+#include "llvm/Support/FileSystem.h"
 
 static std::atomic_bool quit = false;
 static std::mutex errorMutex;
@@ -98,7 +99,15 @@ int main(int argc, char **argv) {
     return -1;
   }
 
-  auto options = optionsParser.apply(dynamatic::Options{});
+  dynamatic::Options defaults{};
+#pragma clang diagnostic ignored "-Wmain"
+  defaults.executablePath = llvm::sys::fs::getMainExecutable(
+      argv[0], reinterpret_cast<void *>(&main));
+  defaults.dynamaticExecutablePath =
+      std::filesystem::path(defaults.executablePath).parent_path() /
+      "dynamatic";
+
+  auto options = optionsParser.apply(defaults);
 
   std::optional<std::size_t> numThreads = optionsParser.getNumThreads();
   if (!numThreads)

tools/hls-fuzzer/oracles/CMakeLists.txt

@@ -0,0 +1,7 @@
+add_llvm_executable(hls-fuzzer-check-bitwidth
+  check-bitwidth.cpp
+)
+llvm_update_compile_flags(hls-fuzzer-check-bitwidth)
+target_link_libraries(hls-fuzzer-check-bitwidth PRIVATE DynamaticHandshake MLIRParser)
+
+add_dependencies(hls-fuzzer hls-fuzzer-check-bitwidth)

tools/hls-fuzzer/oracles/check-bitwidth.cpp

@@ -0,0 +1,69 @@
+
+#include "dynamatic/Dialect/Handshake/HandshakeDialect.h"
+#include "mlir/Tools/ParseUtilities.h"
+
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <dynamatic/Dialect/Handshake/HandshakeOps.h>
+#include <dynamatic/Dialect/Handshake/HandshakeTypes.h>
+
+using namespace mlir;
+using namespace dynamatic;
+
+int main(int argc, char **argv) {
+  if (argc != 3) {
+    llvm::errs() << "expected exactly two arguments\n";
+    return -1;
+  }
+
+  StringRef mlirFile = argv[1];
+  StringRef bitwidthArg = argv[2];
+  std::uint32_t bitWidth;
+  if (bitwidthArg.getAsInteger(0, bitWidth)) {
+    llvm::errs() << "expected an integer as second argument\n";
+    return -1;
+  }
+
+  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> buffer =
+      llvm::MemoryBuffer::getFileOrSTDIN(mlirFile, true);
+  if (!buffer) {
+    llvm::errs() << "failed to open" << mlirFile << "\n";
+    return -1;
+  }
+
+  auto sourceMgr = std::make_shared<llvm::SourceMgr>();
+  sourceMgr->AddNewSourceBuffer(std::move(*buffer), SMLoc());
+  DialectRegistry registry;
+  registry.insert<handshake::HandshakeDialect>();
+  MLIRContext context(registry);
+  ParserConfig config(&context);
+  OwningOpRef<Operation *> module =
+      parseSourceFileForTool(sourceMgr, config, true);
+
+  WalkResult result = module->walk([&](Operation *op) {
+    for (Value iter : op->getResults()) {
+      auto channelType = dyn_cast<handshake::ChannelType>(iter.getType());
+      if (!channelType || !isa<IntegerType>(channelType.getDataType()))
+        continue;
+
+      // Results that feed into 'end' ops are allowed to use a higher bitwidth
+      // as it is required for interface conformance.
+      if (llvm::any_of(iter.getUsers(),
+                       [](Operation *op) { return isa<handshake::EndOp>(op); }))
+        continue;
+
+      if (channelType.getDataBitWidth() > bitWidth) {
+        op->emitError("expected computation with at most a bitwidth of '")
+            << bitWidth << "' rather than '" << channelType.getDataBitWidth()
+            << "'";
+        return WalkResult::interrupt();
+      }
+    }
+    return WalkResult::advance();
+  });
+  if (result.wasInterrupted())
+    return -1;
+
+  return 0;
+}

tools/hls-fuzzer/targets/BitwidthOptimizationsTarget.cpp

@@ -22,11 +22,13 @@ class BitwidthOptimizationsGenerator : public AbstractWorker {
                                           Randomly &&random)
       : AbstractWorker(options, std::move(random)) {}
 
-  void generate(llvm::raw_ostream &os,
-                llvm::StringRef functionName) const override;
+  void generate(llvm::raw_ostream &os, llvm::StringRef functionName) override;
 
   VerificationResult
   verify(const std::filesystem::path &sourceFile) const override;
+
+private:
+  std::uint8_t maxBitwidth;
 };
 
 } // namespace
@@ -38,10 +40,10 @@ BitwidthOptimizationsTarget::createWorker(const Options &options,
                                                           std::move(randomly));
 }
 
-void BitwidthOptimizationsGenerator::generate(
-    llvm::raw_ostream &os, llvm::StringRef functionName) const {
+void BitwidthOptimizationsGenerator::generate(llvm::raw_ostream &os,
+                                              llvm::StringRef functionName) {
   // Enforce a strict bitwidth requirement for the entire program.
-  auto maxBitwidth = random.getInteger<std::uint8_t>(1, 32);
+  maxBitwidth = random.getInteger<std::uint8_t>(1, 32);
   gen::BitwidthTypeSystem bitwidthTypeSystem(maxBitwidth, random);
   gen::BasicCGenerator generator(random, bitwidthTypeSystem,
                                  /*entryContext=*/
@@ -58,7 +60,23 @@ void BitwidthOptimizationsGenerator::generate(
   os << generator.generateTestBench(function);
 }
 
+constexpr std::string_view ORACLE_EXECUTABLE = "hls-fuzzer-check-bitwidth";
+constexpr std::string_view COMPILATION_IR_OUTPUT =
+    "./out/comp/handshake_export.mlir";
+
 AbstractWorker::VerificationResult BitwidthOptimizationsGenerator::verify(
     const std::filesystem::path &sourceFile) const {
-  return performDifferentialTesting(sourceFile, options.dynamaticPath);
+  switch (options.kind) {
+  case OracleKind::Functional:
+    return performDifferentialTesting(sourceFile,
+                                      options.dynamaticExecutablePath);
+  case OracleKind::NonFunctional:
+    return performNonFunctionalTesting(
+        sourceFile, options.dynamaticExecutablePath,
+        (std::filesystem::path(options.executablePath).parent_path() /
+         ORACLE_EXECUTABLE)
+            .string(),
+        {COMPILATION_IR_OUTPUT, std::to_string(maxBitwidth)});
+  }
+  llvm_unreachable("all enum cases handled");
 }

tools/hls-fuzzer/targets/BitwidthTypeSystem.cpp

@@ -89,6 +89,8 @@ auto dynamatic::gen::BitwidthTypeSystem::checkBinaryExpression(
     return std::nullopt;
 
   case ast::BinaryExpression::ShiftRight:
+    // TODO: Figure out constraints here.
+    return std::nullopt;
   case ast::BinaryExpression::Greater:
   case ast::BinaryExpression::GreaterEqual:
   case ast::BinaryExpression::Less:

tools/hls-fuzzer/targets/RandomCTarget.cpp

@@ -15,8 +15,7 @@ class RandomCWorker : public AbstractWorker {
   explicit RandomCWorker(const Options &options, Randomly &&random)
       : AbstractWorker(options, std::move(random)) {}
 
-  void generate(llvm::raw_ostream &os,
-                llvm::StringRef functionName) const override;
+  void generate(llvm::raw_ostream &os, llvm::StringRef functionName) override;
 
   VerificationResult
   verify(const std::filesystem::path &sourceFile) const override;
@@ -30,7 +29,7 @@ RandomCTarget::createWorker(const Options &options, Randomly randomly) const {
 }
 
 void RandomCWorker::generate(llvm::raw_ostream &os,
-                             llvm::StringRef functionName) const {
+                             llvm::StringRef functionName) {
   gen::DynamaticTypeSystem dynamaticTypeSystem(random);
   gen::BasicCGenerator generator(
       random, dynamaticTypeSystem,
@@ -50,5 +49,6 @@ void RandomCWorker::generate(llvm::raw_ostream &os,
 
 AbstractWorker::VerificationResult
 RandomCWorker::verify(const std::filesystem::path &sourceFile) const {
-  return performDifferentialTesting(sourceFile, options.dynamaticPath);
+  return performDifferentialTesting(sourceFile,
+                                    options.dynamaticExecutablePath);
 }

tools/hls-fuzzer/targets/TargetUtils.cpp

@@ -3,54 +3,98 @@
 #include "llvm/Support/Error.h"
 #include "llvm/Support/Program.h"
 
+constexpr std::string_view EXECUTE_SCRIPT = "execute.sh";
+constexpr std::string_view SHELL = "bash";
+
 dynamatic::AbstractWorker::VerificationResult
 dynamatic::performDifferentialTesting(const std::filesystem::path &sourceFile,
                                       llvm::StringRef dynamaticPath) {
   // Create an 'execute.sh' that can additionally be used as a nice reproducer
   // for e.g. 'cvise'.
   std::filesystem::path parentPath = sourceFile.parent_path();
-  std::string executeFile = (parentPath / "execute.sh").string();
+  std::string executeFile = (parentPath / EXECUTE_SCRIPT).string();
   llvm::cantFail(llvm::writeToOutput(
       executeFile, [&](llvm::raw_ostream &os) -> llvm::Error {
-        os << dynamaticPath << " --exit-on-failure <<EOF\n";
-        os << "set-dynamatic-path "
-           << std::filesystem::path(dynamaticPath.str())
-                  .parent_path()
-                  .parent_path()
-                  .string()
-           << '\n';
-        os << "set-src " << sourceFile.filename().string();
-        os << R"(
+        outputDynamaticInvocation(os, sourceFile, dynamaticPath, R"(
 compile
 write-hdl
 simulate
-exit
-EOF
-)";
+)");
+        return llvm::Error::success();
+      }));
+  return executeInWorkingDirectory(parentPath,
+                                   llvm::Twine(SHELL) + " " + EXECUTE_SCRIPT);
+}
+
+dynamatic::AbstractWorker::VerificationResult
+dynamatic::performNonFunctionalTesting(
+    const std::filesystem::path &sourceFile, llvm::StringRef dynamaticPath,
+    llvm::StringRef oracleExecutable,
+    llvm::ArrayRef<llvm::StringRef> arguments) {
+  std::filesystem::path parentPath = sourceFile.parent_path();
+
+  std::string executeFile = (parentPath / EXECUTE_SCRIPT).string();
+  llvm::cantFail(llvm::writeToOutput(
+      executeFile, [&](llvm::raw_ostream &os) -> llvm::Error {
+        os << "set -e\n";
+        outputDynamaticInvocation(os, sourceFile, dynamaticPath, R"(
+compile
+)");
+        os << oracleExecutable;
+        for (llvm::StringRef iter : arguments)
+          os << " " << iter;
+
+        os << '\n';
         return llvm::Error::success();
       }));
+  return executeInWorkingDirectory(parentPath,
+                                   llvm::Twine(SHELL) + " " + EXECUTE_SCRIPT);
+}
+
+void dynamatic::outputDynamaticInvocation(
+    llvm::raw_ostream &os, const std::filesystem::path &sourceFile,
+    llvm::StringRef dynamaticPath, llvm::StringRef script) {
+  // Compute the dynamatic home path by assuming it's a parent directory of the
+  // dynamatic executable and contains the scripts directory used to implement
+  // the various commands.
+  std::filesystem::path dynamaticSourceRoot = dynamaticPath.str();
+  while (!dynamaticSourceRoot.empty()) {
+    dynamaticSourceRoot = dynamaticSourceRoot.parent_path();
+    if (exists(dynamaticSourceRoot / "tools" / "dynamatic" / "scripts"))
+      break;
+  }
+
+  os << dynamaticPath << " --exit-on-failure <<EOF\n";
+  os << "set-dynamatic-path " << dynamaticSourceRoot.string() << '\n';
+  os << "set-src " << sourceFile.filename().string();
+  os << "\n" << script.trim() << "\nexit\nEOF\n";
+}
+
+dynamatic::AbstractWorker::VerificationResult
+dynamatic::executeInWorkingDirectory(
+    const std::filesystem::path &workingDirectory,
+    const llvm::Twine &bashCommand) {
 
   // LLVM's process creation does not support changing the current working
   // directory. We require this since dynamatic creates many of its artifacts
   // in the working directory. Workaround this limitation using a wrapper
   // script that performs a 'cd' to the directory it is contained in.
-  std::string executeCWDFile = (parentPath / "execute_cwd.sh").string();
+  std::string executeCWDFile = (workingDirectory / "execute_cwd.sh").string();
   llvm::cantFail(llvm::writeToOutput(
       executeCWDFile, [&](llvm::raw_ostream &os) -> llvm::Error {
         os << R"a(SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-cd $SCRIPT_DIR && bash )a"
-           << executeFile
+cd $SCRIPT_DIR && )a"
+           << bashCommand
            // Canonicalize all error exists to exit code 1, even if dynamatic
            // crashed with e.g. SIGSEGV. We need this to differentiate between
            // bash exiting with a signal and dynamatic exiting with a signal.
-           << "|| exit 1\n";
+           << " || exit 1\n";
         return llvm::Error::success();
       }));
 
   int exitCode = llvm::sys::ExecuteAndWait(
-      "/usr/bin/bash", {"bash", executeCWDFile}, /*Env=*/std::nullopt,
+      "/usr/bin/bash", {SHELL, executeCWDFile}, /*Env=*/std::nullopt,
       /*Redirects=*/{"", "", ""});
-
   switch (exitCode) {
     // Normal exit.
   case 0: