From 4f2b15e37c4e6aa1b37e61772d013bbcc41f8894 Mon Sep 17 00:00:00 2001 From: coleramos425 Date: Mon, 19 May 2025 13:52:47 -0500 Subject: [PATCH 01/12] New header file for JSON output, include and implement JSON header into memory_analysis_handler, write output file on completion Signed-off-by: coleramos425 --- inc/json_output.h | 135 +++++++++++++++++++++++++++++++++ inc/memory_analysis_wrapper.h | 3 + src/memory_analysis_wrapper.cc | 30 +++++++- 3 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 inc/json_output.h diff --git a/inc/json_output.h b/inc/json_output.h new file mode 100644 index 0000000..8d0bd50 --- /dev/null +++ b/inc/json_output.h @@ -0,0 +1,135 @@ +#pragma once + +#include +#include +#include +#include +#include + +using json = nlohmann::json; + +class JsonOutputManager { +public: + static JsonOutputManager& getInstance() { + static JsonOutputManager instance; + return instance; + } + + void initializeKernelAnalysis(const std::string& kernel_name, uint64_t dispatch_id) { + if (!current_analysis_.contains("kernel_analyses")) { + current_analysis_["kernel_analyses"] = json::array(); + } + + json kernel_analysis; + kernel_analysis["kernel_info"]["name"] = kernel_name; + kernel_analysis["kernel_info"]["dispatch_id"] = dispatch_id; + kernel_analysis["cache_analysis"]["accesses"] = json::array(); + kernel_analysis["bank_conflicts"]["accesses"] = json::array(); + + current_analysis_["kernel_analyses"].push_back(kernel_analysis); + } + + void setMetadata(const std::string& gpu_arch, uint32_t cache_line_size, size_t kernels_found) { + current_analysis_["metadata"] = { + {"timestamp", std::chrono::system_clock::now().time_since_epoch().count()}, + {"version", "1.0"}, + {"gpu_info", { + {"architecture", gpu_arch}, + {"cache_line_size", cache_line_size} + }}, + {"kernels_found", kernels_found} + }; + } + + void updateKernelsFound(size_t kernels_found) { + if (current_analysis_.contains("metadata")) { + current_analysis_["metadata"]["kernels_found"] = kernels_found; + } + } + + void addCacheAnalysis(const std::string& file, uint32_t line, uint32_t column, + const std::string& code_context, const std::string& access_type, + uint16_t ir_bytes, uint16_t isa_bytes, const std::string& isa_instruction, + size_t execution_count, size_t cache_lines_needed, size_t cache_lines_used) { + if (current_analysis_["kernel_analyses"].empty()) return; + + json access; + access["source_location"] = { + {"file", file}, + {"line", line}, + {"column", column} + }; + access["code_context"] = code_context; + access["access_info"] = { + {"type", access_type}, + {"ir_bytes", ir_bytes}, + {"isa_bytes", isa_bytes}, + {"isa_instruction", isa_instruction}, + {"execution_count", execution_count}, + {"cache_lines", { + {"needed", cache_lines_needed}, + {"used", cache_lines_used} + }} + }; + + auto& current_kernel = current_analysis_["kernel_analyses"].back(); + current_kernel["cache_analysis"]["accesses"].push_back(access); + } + + void addBankConflict(const std::string& file, uint32_t line, uint32_t column, + const std::string& code_context, const std::string& access_type, + uint16_t ir_bytes, size_t execution_count, size_t total_conflicts) { + if (current_analysis_["kernel_analyses"].empty()) return; + + json access; + access["source_location"] = { + {"file", file}, + {"line", line}, + {"column", column} + }; + access["code_context"] = code_context; + access["access_info"] = { + {"type", access_type}, + {"ir_bytes", ir_bytes}, + {"execution_count", execution_count}, + {"total_conflicts", total_conflicts} + }; + + auto& current_kernel = current_analysis_["kernel_analyses"].back(); + current_kernel["bank_conflicts"]["accesses"].push_back(access); + } + + void setExecutionTimes(uint64_t start_ns, uint64_t end_ns, uint64_t complete_ns) { + if (current_analysis_["kernel_analyses"].empty()) return; + + auto& current_kernel = current_analysis_["kernel_analyses"].back(); + current_kernel["kernel_info"]["execution_time"] = { + {"start_ns", start_ns}, + {"end_ns", end_ns}, + {"complete_ns", complete_ns} + }; + } + + void setProcessingStats(size_t bytes_processed, double processing_time_seconds) { + if (current_analysis_["kernel_analyses"].empty()) return; + + auto& current_kernel = current_analysis_["kernel_analyses"].back(); + current_kernel["kernel_info"]["bytes_processed"] = bytes_processed; + current_kernel["kernel_info"]["processing_time_seconds"] = processing_time_seconds; + current_kernel["kernel_info"]["throughput_mib_per_sec"] = + (bytes_processed / processing_time_seconds) / 1.0e6; + } + + void writeToFile(const std::string& filename) { + std::ofstream out(filename); + out << current_analysis_.dump(2); + } + + void clear() { + current_analysis_.clear(); + } + +private: + JsonOutputManager() = default; + json current_analysis_; +}; \ No newline at end of file diff --git a/inc/memory_analysis_wrapper.h b/inc/memory_analysis_wrapper.h index 1d6b653..cab8335 100644 --- a/inc/memory_analysis_wrapper.h +++ b/inc/memory_analysis_wrapper.h @@ -22,6 +22,7 @@ THE SOFTWARE. #pragma once #include "message_handlers.h" #include "memory_analysis_handler.h" +#include "json_output.h" #include #include @@ -37,8 +38,10 @@ class memory_analysis_wrapper_t : public dh_comms::message_handler_base { virtual void report() override; virtual void clear() override; +private: const std::string& kernel_; uint64_t dispatch_id_; const std::string& location_; + bool verbose_; dh_comms::memory_analysis_handler_t wrapped_; }; diff --git a/src/memory_analysis_wrapper.cc b/src/memory_analysis_wrapper.cc index 4b39c9b..b86e8e5 100644 --- a/src/memory_analysis_wrapper.cc +++ b/src/memory_analysis_wrapper.cc @@ -8,6 +8,7 @@ #include #include #include +#include __attribute__((constructor)) void on_library_load() { std::cout << "Memory Analysis Wrapper loaded." << std::endl; } __attribute__((destructor)) void on_library_unload() { std::cout << "Memory Analysis Wrapper unloaded." << std::endl; } @@ -15,6 +16,15 @@ __attribute__((destructor)) void on_library_unload() { std::cout << "Memory Anal memory_analysis_wrapper_t::memory_analysis_wrapper_t(const std::string& kernel, uint64_t dispatch_id, const std::string& location, bool verbose) : kernel_(kernel), dispatch_id_(dispatch_id), location_(location), wrapped_(verbose) { + JsonOutputManager::getInstance().initializeKernelAnalysis(kernel, dispatch_id); + + // Get GPU architecture and cache line size + hipDeviceProp_t props; + hipGetDeviceProperties(&props, 0); + std::string gpu_arch = props.gcnArchName; + + // Initialize metadata with default kernels_found (will be updated in report) + JsonOutputManager::getInstance().setMetadata(gpu_arch, 128, 0); // Using default cache line size of 128 } bool memory_analysis_wrapper_t::handle(const dh_comms::message_t &message, const std::string& kernel, kernelDB::kernelDB& kdb) @@ -33,15 +43,33 @@ void memory_analysis_wrapper_t::report(const std::string& kernel_name, kernelDB: { std::vector lines; kdb.getKernelLines(kernel_name, lines); + + // Update kernels found count + std::vector kernels; + kdb.getKernels(kernels); + JsonOutputManager::getInstance().updateKernelsFound(kernels.size()); } report(); } void memory_analysis_wrapper_t::report() { - std::cout << "Memory analysis for " << kernel_ << " dispatch_id[" << std::dec << dispatch_id_ << "]" << std::endl; + if (verbose_) { + std::cout << "Memory analysis for " << kernel_ << " dispatch_id[" << std::dec << dispatch_id_ << "]" << std::endl; + } wrapped_.report(); + + // Create output directory if it doesn't exist + std::filesystem::path output_dir = location_; + if (!std::filesystem::exists(output_dir)) { + std::filesystem::create_directories(output_dir); + } + + // Generate output filename + std::string filename = (output_dir / ("memory_analysis_" + std::to_string(dispatch_id_) + ".json")).string(); + JsonOutputManager::getInstance().writeToFile(filename); } void memory_analysis_wrapper_t::clear() { wrapped_.clear(); + JsonOutputManager::getInstance().clear(); } From 00b06573865ed2c0e86e30a3d01a158cf7c08fc5 Mon Sep 17 00:00:00 2001 From: coleramos425 Date: Mon, 19 May 2025 13:53:38 -0500 Subject: [PATCH 02/12] Propagate verbose flag from omniprobe entrypoint to backend plugins Signed-off-by: coleramos425 --- omniprobe/omniprobe | 8 +++++++- plugins/memory_analysis_plugin.cc | 10 +++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/omniprobe/omniprobe b/omniprobe/omniprobe index 5f3147c..98cfed9 100755 --- a/omniprobe/omniprobe +++ b/omniprobe/omniprobe @@ -263,7 +263,13 @@ def setup_env(parms): env['LOGDUR_LOG_FORMAT'] = parms.log_format env_dump['LOGDUR_LOG_FORMAT'] = parms.log_format - + if parms.verbose: + env['LOGDUR_VERBOSE'] = "true" + env_dump['LOGDUR_VERBOSE'] = "true" + else: + env['LOGDUR_VERBOSE'] = "false" + env_dump['LOGDUR_VERBOSE'] = "false" + if len(parms.log_location): if parms.log_location != "console": if not os.path.isdir(parms.log_location): diff --git a/plugins/memory_analysis_plugin.cc b/plugins/memory_analysis_plugin.cc index 42a5251..94cc375 100644 --- a/plugins/memory_analysis_plugin.cc +++ b/plugins/memory_analysis_plugin.cc @@ -8,6 +8,14 @@ extern "C"{ const char* logDurLogLocation = std::getenv("LOGDUR_LOG_LOCATION"); if (logDurLogLocation != NULL) location = logDurLogLocation; - outHandlers.push_back(new memory_analysis_wrapper_t(kernel, dispatch_id, location, false)); + + // Read verbose setting from environment + bool verbose = false; + const char* logDurVerbose = std::getenv("LOGDUR_VERBOSE"); + if (logDurVerbose != NULL && (strcmp(logDurVerbose, "1") == 0 || strcmp(logDurVerbose, "true") == 0)) { + verbose = true; + } + + outHandlers.push_back(new memory_analysis_wrapper_t(kernel, dispatch_id, location, verbose)); } } From 95d41b168a8856511267a5ca0e6f988d9ebf39ba Mon Sep 17 00:00:00 2001 From: coleramos425 Date: Mon, 19 May 2025 13:58:28 -0500 Subject: [PATCH 03/12] Update dh_comms submodule Signed-off-by: coleramos425 --- external/dh_comms | 2 +- external/instrument-amdgpu-kernels | 2 +- external/kerneldb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/external/dh_comms b/external/dh_comms index b87a31c..18fe1df 160000 --- a/external/dh_comms +++ b/external/dh_comms @@ -1 +1 @@ -Subproject commit b87a31ce2c400d92263eb6979328283a0674edc8 +Subproject commit 18fe1dff07a233eeca1817d45f9d8807275debe6 diff --git a/external/instrument-amdgpu-kernels b/external/instrument-amdgpu-kernels index 1be4597..ad34401 160000 --- a/external/instrument-amdgpu-kernels +++ b/external/instrument-amdgpu-kernels @@ -1 +1 @@ -Subproject commit 1be4597cc2d018d38df9fdb4a115a979ea9bd49e +Subproject commit ad344010397071bba251edc0c75007b18351d421 diff --git a/external/kerneldb b/external/kerneldb index f439b0f..705e166 160000 --- a/external/kerneldb +++ b/external/kerneldb @@ -1 +1 @@ -Subproject commit f439b0ffb61e93b8dfcdea725eb659c985f0e22a +Subproject commit 705e1660c17782cd95649bf0e0e4af3f4850ff75 From 8b6da9794ec3f6e471f427f5a80452bc9a1acec1 Mon Sep 17 00:00:00 2001 From: coleramos425 Date: Mon, 19 May 2025 14:45:33 -0500 Subject: [PATCH 04/12] Moving json_output header into dh_comms, removing includes from logduration Signed-off-by: coleramos425 --- inc/json_output.h | 135 --------------------------------- inc/memory_analysis_wrapper.h | 1 - src/memory_analysis_wrapper.cc | 10 +-- 3 files changed, 5 insertions(+), 141 deletions(-) delete mode 100644 inc/json_output.h diff --git a/inc/json_output.h b/inc/json_output.h deleted file mode 100644 index 8d0bd50..0000000 --- a/inc/json_output.h +++ /dev/null @@ -1,135 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -using json = nlohmann::json; - -class JsonOutputManager { -public: - static JsonOutputManager& getInstance() { - static JsonOutputManager instance; - return instance; - } - - void initializeKernelAnalysis(const std::string& kernel_name, uint64_t dispatch_id) { - if (!current_analysis_.contains("kernel_analyses")) { - current_analysis_["kernel_analyses"] = json::array(); - } - - json kernel_analysis; - kernel_analysis["kernel_info"]["name"] = kernel_name; - kernel_analysis["kernel_info"]["dispatch_id"] = dispatch_id; - kernel_analysis["cache_analysis"]["accesses"] = json::array(); - kernel_analysis["bank_conflicts"]["accesses"] = json::array(); - - current_analysis_["kernel_analyses"].push_back(kernel_analysis); - } - - void setMetadata(const std::string& gpu_arch, uint32_t cache_line_size, size_t kernels_found) { - current_analysis_["metadata"] = { - {"timestamp", std::chrono::system_clock::now().time_since_epoch().count()}, - {"version", "1.0"}, - {"gpu_info", { - {"architecture", gpu_arch}, - {"cache_line_size", cache_line_size} - }}, - {"kernels_found", kernels_found} - }; - } - - void updateKernelsFound(size_t kernels_found) { - if (current_analysis_.contains("metadata")) { - current_analysis_["metadata"]["kernels_found"] = kernels_found; - } - } - - void addCacheAnalysis(const std::string& file, uint32_t line, uint32_t column, - const std::string& code_context, const std::string& access_type, - uint16_t ir_bytes, uint16_t isa_bytes, const std::string& isa_instruction, - size_t execution_count, size_t cache_lines_needed, size_t cache_lines_used) { - if (current_analysis_["kernel_analyses"].empty()) return; - - json access; - access["source_location"] = { - {"file", file}, - {"line", line}, - {"column", column} - }; - access["code_context"] = code_context; - access["access_info"] = { - {"type", access_type}, - {"ir_bytes", ir_bytes}, - {"isa_bytes", isa_bytes}, - {"isa_instruction", isa_instruction}, - {"execution_count", execution_count}, - {"cache_lines", { - {"needed", cache_lines_needed}, - {"used", cache_lines_used} - }} - }; - - auto& current_kernel = current_analysis_["kernel_analyses"].back(); - current_kernel["cache_analysis"]["accesses"].push_back(access); - } - - void addBankConflict(const std::string& file, uint32_t line, uint32_t column, - const std::string& code_context, const std::string& access_type, - uint16_t ir_bytes, size_t execution_count, size_t total_conflicts) { - if (current_analysis_["kernel_analyses"].empty()) return; - - json access; - access["source_location"] = { - {"file", file}, - {"line", line}, - {"column", column} - }; - access["code_context"] = code_context; - access["access_info"] = { - {"type", access_type}, - {"ir_bytes", ir_bytes}, - {"execution_count", execution_count}, - {"total_conflicts", total_conflicts} - }; - - auto& current_kernel = current_analysis_["kernel_analyses"].back(); - current_kernel["bank_conflicts"]["accesses"].push_back(access); - } - - void setExecutionTimes(uint64_t start_ns, uint64_t end_ns, uint64_t complete_ns) { - if (current_analysis_["kernel_analyses"].empty()) return; - - auto& current_kernel = current_analysis_["kernel_analyses"].back(); - current_kernel["kernel_info"]["execution_time"] = { - {"start_ns", start_ns}, - {"end_ns", end_ns}, - {"complete_ns", complete_ns} - }; - } - - void setProcessingStats(size_t bytes_processed, double processing_time_seconds) { - if (current_analysis_["kernel_analyses"].empty()) return; - - auto& current_kernel = current_analysis_["kernel_analyses"].back(); - current_kernel["kernel_info"]["bytes_processed"] = bytes_processed; - current_kernel["kernel_info"]["processing_time_seconds"] = processing_time_seconds; - current_kernel["kernel_info"]["throughput_mib_per_sec"] = - (bytes_processed / processing_time_seconds) / 1.0e6; - } - - void writeToFile(const std::string& filename) { - std::ofstream out(filename); - out << current_analysis_.dump(2); - } - - void clear() { - current_analysis_.clear(); - } - -private: - JsonOutputManager() = default; - json current_analysis_; -}; \ No newline at end of file diff --git a/inc/memory_analysis_wrapper.h b/inc/memory_analysis_wrapper.h index cab8335..80dcd9b 100644 --- a/inc/memory_analysis_wrapper.h +++ b/inc/memory_analysis_wrapper.h @@ -22,7 +22,6 @@ THE SOFTWARE. #pragma once #include "message_handlers.h" #include "memory_analysis_handler.h" -#include "json_output.h" #include #include diff --git a/src/memory_analysis_wrapper.cc b/src/memory_analysis_wrapper.cc index b86e8e5..25ff2e0 100644 --- a/src/memory_analysis_wrapper.cc +++ b/src/memory_analysis_wrapper.cc @@ -16,7 +16,7 @@ __attribute__((destructor)) void on_library_unload() { std::cout << "Memory Anal memory_analysis_wrapper_t::memory_analysis_wrapper_t(const std::string& kernel, uint64_t dispatch_id, const std::string& location, bool verbose) : kernel_(kernel), dispatch_id_(dispatch_id), location_(location), wrapped_(verbose) { - JsonOutputManager::getInstance().initializeKernelAnalysis(kernel, dispatch_id); + dh_comms::JsonOutputManager::getInstance().initializeKernelAnalysis(kernel, dispatch_id); // Get GPU architecture and cache line size hipDeviceProp_t props; @@ -24,7 +24,7 @@ memory_analysis_wrapper_t::memory_analysis_wrapper_t(const std::string& kernel, std::string gpu_arch = props.gcnArchName; // Initialize metadata with default kernels_found (will be updated in report) - JsonOutputManager::getInstance().setMetadata(gpu_arch, 128, 0); // Using default cache line size of 128 + dh_comms::JsonOutputManager::getInstance().setMetadata(gpu_arch, 128, 0); // Using default cache line size of 128 } bool memory_analysis_wrapper_t::handle(const dh_comms::message_t &message, const std::string& kernel, kernelDB::kernelDB& kdb) @@ -47,7 +47,7 @@ void memory_analysis_wrapper_t::report(const std::string& kernel_name, kernelDB: // Update kernels found count std::vector kernels; kdb.getKernels(kernels); - JsonOutputManager::getInstance().updateKernelsFound(kernels.size()); + dh_comms::JsonOutputManager::getInstance().updateKernelsFound(kernels.size()); } report(); } @@ -66,10 +66,10 @@ void memory_analysis_wrapper_t::report() { // Generate output filename std::string filename = (output_dir / ("memory_analysis_" + std::to_string(dispatch_id_) + ".json")).string(); - JsonOutputManager::getInstance().writeToFile(filename); + dh_comms::JsonOutputManager::getInstance().writeToFile(filename); } void memory_analysis_wrapper_t::clear() { wrapped_.clear(); - JsonOutputManager::getInstance().clear(); + dh_comms::JsonOutputManager::getInstance().clear(); } From 05b9dbe8bd8620e2c2306682091818d7aa4f42d6 Mon Sep 17 00:00:00 2001 From: coleramos425 Date: Mon, 19 May 2025 18:18:30 -0500 Subject: [PATCH 05/12] Add the C++ JSON library to the containers Signed-off-by: coleramos425 --- docker/logduration.Dockerfile | 2 +- docker/logduration.def | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/logduration.Dockerfile b/docker/logduration.Dockerfile index 4aa5775..2760a7c 100644 --- a/docker/logduration.Dockerfile +++ b/docker/logduration.Dockerfile @@ -12,7 +12,7 @@ COPY triton_install.sh /app/triton_install.sh RUN apt-get update && \ apt-get install -y software-properties-common && \ apt-get upgrade -y && \ - apt-get install -y git build-essential wget clang lld libzstd-dev libomp-dev ccache libdwarf-dev python3-dev && \ + apt-get install -y git build-essential wget clang lld libzstd-dev libomp-dev ccache libdwarf-dev python3-dev nlohmann-json3-dev && \ python3 -m pip install --upgrade pip && \ python3 -m pip install --upgrade setuptools diff --git a/docker/logduration.def b/docker/logduration.def index fdc392a..91d5ce3 100644 --- a/docker/logduration.def +++ b/docker/logduration.def @@ -27,7 +27,7 @@ From: rocm/rocm-build-ubuntu-22.04:6.3 apt-get update apt-get install -y software-properties-common apt-get upgrade -y - apt-get install -y git build-essential wget clang lld libzstd-dev libomp-dev ccache libdwarf-dev python3-dev + apt-get install -y git build-essential wget clang lld libzstd-dev libomp-dev ccache libdwarf-dev python3-dev nlohmann-json3-dev python3 -m pip install --upgrade pip python3 -m pip install --upgrade setuptools From 8985f80ce731d01acf3b5fae8e8caedb6cc17655 Mon Sep 17 00:00:00 2001 From: coleramos425 Date: Tue, 20 May 2025 20:32:49 -0500 Subject: [PATCH 06/12] Remove memory analysis wrapper deconstructor to avoid premature unloading The wrapped class memory_analysis_handler_t has a default deconstructor that should be called. Otherwise, the JsonOutputManger is destroyed prematurely when the library is unloaded, and then a new empty instance us being created inadvertently. Signed-off-by: coleramos425 --- src/memory_analysis_wrapper.cc | 86 +++++++++++++++++++++++++++------- 1 file changed, 70 insertions(+), 16 deletions(-) diff --git a/src/memory_analysis_wrapper.cc b/src/memory_analysis_wrapper.cc index 25ff2e0..02d004c 100644 --- a/src/memory_analysis_wrapper.cc +++ b/src/memory_analysis_wrapper.cc @@ -11,11 +11,18 @@ #include __attribute__((constructor)) void on_library_load() { std::cout << "Memory Analysis Wrapper loaded." << std::endl; } -__attribute__((destructor)) void on_library_unload() { std::cout << "Memory Analysis Wrapper unloaded." << std::endl; } + +// Manual cleanup when we're truly done +void on_library_unload() { + std::cout << "Memory Analysis Wrapper manual unload requested." << std::endl; + // Only cleanup JsonOutputManager after we're done with all analysis + dh_comms::JsonOutputManager::cleanup(); +} memory_analysis_wrapper_t::memory_analysis_wrapper_t(const std::string& kernel, uint64_t dispatch_id, const std::string& location, bool verbose) : kernel_(kernel), dispatch_id_(dispatch_id), location_(location), wrapped_(verbose) { + std::cout << "[MemoryAnalysisWrapper] Initializing wrapper for kernel: " << kernel << std::endl; dh_comms::JsonOutputManager::getInstance().initializeKernelAnalysis(kernel, dispatch_id); // Get GPU architecture and cache line size @@ -25,6 +32,7 @@ memory_analysis_wrapper_t::memory_analysis_wrapper_t(const std::string& kernel, // Initialize metadata with default kernels_found (will be updated in report) dh_comms::JsonOutputManager::getInstance().setMetadata(gpu_arch, 128, 0); // Using default cache line size of 128 + std::cout << "[MemoryAnalysisWrapper] Initialization complete" << std::endl; } bool memory_analysis_wrapper_t::handle(const dh_comms::message_t &message, const std::string& kernel, kernelDB::kernelDB& kdb) @@ -39,16 +47,17 @@ bool memory_analysis_wrapper_t::handle(const dh_comms::message_t &message) { void memory_analysis_wrapper_t::report(const std::string& kernel_name, kernelDB::kernelDB& kdb) { - if (kernel_name.length() == 0) - { + // Get all kernels and update the count + std::vector kernels; + kdb.getKernels(kernels); + dh_comms::JsonOutputManager::getInstance().updateKernelsFound(kernels.size()); + + // Only get lines if kernel_name is provided + if (!kernel_name.empty()) { std::vector lines; kdb.getKernelLines(kernel_name, lines); - - // Update kernels found count - std::vector kernels; - kdb.getKernels(kernels); - dh_comms::JsonOutputManager::getInstance().updateKernelsFound(kernels.size()); } + report(); } @@ -56,17 +65,62 @@ void memory_analysis_wrapper_t::report() { if (verbose_) { std::cout << "Memory analysis for " << kernel_ << " dispatch_id[" << std::dec << dispatch_id_ << "]" << std::endl; } - wrapped_.report(); - // Create output directory if it doesn't exist - std::filesystem::path output_dir = location_; - if (!std::filesystem::exists(output_dir)) { - std::filesystem::create_directories(output_dir); + std::cout << "[MemoryAnalysisWrapper] Starting wrapped_.report()..." << std::endl; + try { + wrapped_.report(); + std::cout << "[MemoryAnalysisWrapper] Completed wrapped_.report() successfully" << std::endl; + std::cout << "[MemoryAnalysisWrapper] Current analysis size after report: " + << dh_comms::JsonOutputManager::getInstance().getCurrentAnalysisSize() << std::endl; + } catch (const std::exception& e) { + std::cerr << "[MemoryAnalysisWrapper] ERROR: Exception in wrapped_.report(): " << e.what() << std::endl; + } catch (...) { + std::cerr << "[MemoryAnalysisWrapper] ERROR: Unknown exception in wrapped_.report()" << std::endl; } - // Generate output filename - std::string filename = (output_dir / ("memory_analysis_" + std::to_string(dispatch_id_) + ".json")).string(); - dh_comms::JsonOutputManager::getInstance().writeToFile(filename); + // Set up output directory path + std::filesystem::path output_dir = std::filesystem::current_path() / "memory_analysis_output"; + std::cout << "[MemoryAnalysisWrapper] Output directory: " << output_dir << std::endl; + + // Create output directory if it doesn't exist + std::cout << "[MemoryAnalysisWrapper] Creating output directory..." << std::endl; + try { + if (!std::filesystem::exists(output_dir)) { + std::filesystem::create_directories(output_dir); + std::cout << "[MemoryAnalysisWrapper] Created directory: " << output_dir << std::endl; + } + + // Generate output filename + std::string filename = (output_dir / ("memory_analysis_" + std::to_string(dispatch_id_) + ".json")).string(); + std::cout << "[MemoryAnalysisWrapper] Writing memory analysis to " << filename << std::endl; + + // Dump current state before writing + std::cout << "[MemoryAnalysisWrapper] Dumping current JsonOutputManager state..." << std::endl; + auto& manager = dh_comms::JsonOutputManager::getInstance(); + std::cout << "[MemoryAnalysisWrapper] Current analysis size before dump: " << manager.getCurrentAnalysisSize() << std::endl; + manager.dumpCurrentState(); + + // Attempt to write the file + std::cout << "[MemoryAnalysisWrapper] Writing to file..." << std::endl; + try { + manager.writeToFile(filename); + std::cout << "[MemoryAnalysisWrapper] Report completed successfully" << std::endl; + } catch (const std::exception& e) { + std::cerr << "[MemoryAnalysisWrapper] ERROR: Failed to write file: " << e.what() << std::endl; + // Try writing to current directory as fallback + filename = (std::filesystem::current_path() / ("memory_analysis_" + std::to_string(dispatch_id_) + ".json")).string(); + std::cout << "[MemoryAnalysisWrapper] Attempting to write to current directory: " << filename << std::endl; + manager.writeToFile(filename); + } + } catch (const std::filesystem::filesystem_error& e) { + std::cerr << "[MemoryAnalysisWrapper] ERROR: Filesystem error: " << e.what() << std::endl; + // Fall back to current directory + std::string filename = (std::filesystem::current_path() / ("memory_analysis_" + std::to_string(dispatch_id_) + ".json")).string(); + std::cout << "[MemoryAnalysisWrapper] Falling back to current directory: " << filename << std::endl; + dh_comms::JsonOutputManager::getInstance().writeToFile(filename); + } catch (const std::exception& e) { + std::cerr << "[MemoryAnalysisWrapper] ERROR: Unexpected error: " << e.what() << std::endl; + } } void memory_analysis_wrapper_t::clear() { From bee7b78f49c10e359da523a0f3575b3ba59e25ca Mon Sep 17 00:00:00 2001 From: coleramos425 Date: Wed, 21 May 2025 14:26:42 -0500 Subject: [PATCH 07/12] Update project submodules with latest commits Signed-off-by: coleramos425 --- external/dh_comms | 2 +- external/instrument-amdgpu-kernels | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/external/dh_comms b/external/dh_comms index 18fe1df..976c43f 160000 --- a/external/dh_comms +++ b/external/dh_comms @@ -1 +1 @@ -Subproject commit 18fe1dff07a233eeca1817d45f9d8807275debe6 +Subproject commit 976c43f05a9f633026a153c5e63afc12b761b8d6 diff --git a/external/instrument-amdgpu-kernels b/external/instrument-amdgpu-kernels index ad34401..70fcfa6 160000 --- a/external/instrument-amdgpu-kernels +++ b/external/instrument-amdgpu-kernels @@ -1 +1 @@ -Subproject commit ad344010397071bba251edc0c75007b18351d421 +Subproject commit 70fcfa6606d1fceb0269a263e00e01d74a9a23b9 From a3f2f7ac409acd97a8181d26a03ed12f21d19c3c Mon Sep 17 00:00:00 2001 From: coleramos425 Date: Tue, 27 May 2025 13:02:51 -0500 Subject: [PATCH 08/12] Remove nlohmann/json install from container. Adding header directly to dh_comms to avoid additional dependency Signed-off-by: coleramos425 --- docker/logduration.Dockerfile | 2 +- docker/logduration.def | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/logduration.Dockerfile b/docker/logduration.Dockerfile index 2760a7c..4aa5775 100644 --- a/docker/logduration.Dockerfile +++ b/docker/logduration.Dockerfile @@ -12,7 +12,7 @@ COPY triton_install.sh /app/triton_install.sh RUN apt-get update && \ apt-get install -y software-properties-common && \ apt-get upgrade -y && \ - apt-get install -y git build-essential wget clang lld libzstd-dev libomp-dev ccache libdwarf-dev python3-dev nlohmann-json3-dev && \ + apt-get install -y git build-essential wget clang lld libzstd-dev libomp-dev ccache libdwarf-dev python3-dev && \ python3 -m pip install --upgrade pip && \ python3 -m pip install --upgrade setuptools diff --git a/docker/logduration.def b/docker/logduration.def index 91d5ce3..e416838 100644 --- a/docker/logduration.def +++ b/docker/logduration.def @@ -27,7 +27,7 @@ From: rocm/rocm-build-ubuntu-22.04:6.3 apt-get update apt-get install -y software-properties-common apt-get upgrade -y - apt-get install -y git build-essential wget clang lld libzstd-dev libomp-dev ccache libdwarf-dev python3-dev nlohmann-json3-dev + apt-get install -y git build-essential wget clang lld libzstd-dev libomp-dev ccache libdwarf-dev python3-dev python3 -m pip install --upgrade pip python3 -m pip install --upgrade setuptools From a2ed5c3c1dc05b49d1c3b075cefdadaf80794e0f Mon Sep 17 00:00:00 2001 From: coleramos425 Date: Tue, 27 May 2025 13:05:22 -0500 Subject: [PATCH 09/12] Update dh_comms to latest feature branch commit Signed-off-by: coleramos425 --- external/dh_comms | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/dh_comms b/external/dh_comms index 976c43f..837140c 160000 --- a/external/dh_comms +++ b/external/dh_comms @@ -1 +1 @@ -Subproject commit 976c43f05a9f633026a153c5e63afc12b761b8d6 +Subproject commit 837140c858b458e6ff846f223a9cf02eb3542d1b From b1d664e35769187a63fb73e8a92bb8d95d8895fe Mon Sep 17 00:00:00 2001 From: coleramos425 Date: Tue, 27 May 2025 13:16:17 -0500 Subject: [PATCH 10/12] Update kerneldb with lates commits from Keith (5/26) Signed-off-by: coleramos425 --- external/kerneldb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/kerneldb b/external/kerneldb index 705e166..f439b0f 160000 --- a/external/kerneldb +++ b/external/kerneldb @@ -1 +1 @@ -Subproject commit 705e1660c17782cd95649bf0e0e4af3f4850ff75 +Subproject commit f439b0ffb61e93b8dfcdea725eb659c985f0e22a From 66db62c48dad72528193bbfbf6527d0130116e3d Mon Sep 17 00:00:00 2001 From: coleramos425 Date: Tue, 3 Jun 2025 14:54:48 -0500 Subject: [PATCH 11/12] Simplify JSON output by collating memory analysis for all dispatches in a single object Signed-off-by: coleramos425 --- src/memory_analysis_wrapper.cc | 52 +++++++++------------------------- 1 file changed, 13 insertions(+), 39 deletions(-) diff --git a/src/memory_analysis_wrapper.cc b/src/memory_analysis_wrapper.cc index 02d004c..3fc4d50 100644 --- a/src/memory_analysis_wrapper.cc +++ b/src/memory_analysis_wrapper.cc @@ -78,48 +78,22 @@ void memory_analysis_wrapper_t::report() { std::cerr << "[MemoryAnalysisWrapper] ERROR: Unknown exception in wrapped_.report()" << std::endl; } - // Set up output directory path - std::filesystem::path output_dir = std::filesystem::current_path() / "memory_analysis_output"; - std::cout << "[MemoryAnalysisWrapper] Output directory: " << output_dir << std::endl; + // Write to single output file + std::string filename = (std::filesystem::current_path() / "memory_analysis_output.json").string(); + std::cout << "[MemoryAnalysisWrapper] Writing memory analysis to " << filename << std::endl; - // Create output directory if it doesn't exist - std::cout << "[MemoryAnalysisWrapper] Creating output directory..." << std::endl; + // Dump current state before writing + std::cout << "[MemoryAnalysisWrapper] Dumping current JsonOutputManager state..." << std::endl; + auto& manager = dh_comms::JsonOutputManager::getInstance(); + std::cout << "[MemoryAnalysisWrapper] Current analysis size before dump: " << manager.getCurrentAnalysisSize() << std::endl; + manager.dumpCurrentState(); + + // Attempt to write the file try { - if (!std::filesystem::exists(output_dir)) { - std::filesystem::create_directories(output_dir); - std::cout << "[MemoryAnalysisWrapper] Created directory: " << output_dir << std::endl; - } - - // Generate output filename - std::string filename = (output_dir / ("memory_analysis_" + std::to_string(dispatch_id_) + ".json")).string(); - std::cout << "[MemoryAnalysisWrapper] Writing memory analysis to " << filename << std::endl; - - // Dump current state before writing - std::cout << "[MemoryAnalysisWrapper] Dumping current JsonOutputManager state..." << std::endl; - auto& manager = dh_comms::JsonOutputManager::getInstance(); - std::cout << "[MemoryAnalysisWrapper] Current analysis size before dump: " << manager.getCurrentAnalysisSize() << std::endl; - manager.dumpCurrentState(); - - // Attempt to write the file - std::cout << "[MemoryAnalysisWrapper] Writing to file..." << std::endl; - try { - manager.writeToFile(filename); - std::cout << "[MemoryAnalysisWrapper] Report completed successfully" << std::endl; - } catch (const std::exception& e) { - std::cerr << "[MemoryAnalysisWrapper] ERROR: Failed to write file: " << e.what() << std::endl; - // Try writing to current directory as fallback - filename = (std::filesystem::current_path() / ("memory_analysis_" + std::to_string(dispatch_id_) + ".json")).string(); - std::cout << "[MemoryAnalysisWrapper] Attempting to write to current directory: " << filename << std::endl; - manager.writeToFile(filename); - } - } catch (const std::filesystem::filesystem_error& e) { - std::cerr << "[MemoryAnalysisWrapper] ERROR: Filesystem error: " << e.what() << std::endl; - // Fall back to current directory - std::string filename = (std::filesystem::current_path() / ("memory_analysis_" + std::to_string(dispatch_id_) + ".json")).string(); - std::cout << "[MemoryAnalysisWrapper] Falling back to current directory: " << filename << std::endl; - dh_comms::JsonOutputManager::getInstance().writeToFile(filename); + manager.writeToFile(filename); + std::cout << "[MemoryAnalysisWrapper] Report completed successfully" << std::endl; } catch (const std::exception& e) { - std::cerr << "[MemoryAnalysisWrapper] ERROR: Unexpected error: " << e.what() << std::endl; + std::cerr << "[MemoryAnalysisWrapper] ERROR: Unexpected error writing file: " << e.what() << std::endl; } } From 25f96d8fa3dd1efcd6aa7a50c126e53368922dec Mon Sep 17 00:00:00 2001 From: coleramos425 Date: Tue, 3 Jun 2025 14:57:29 -0500 Subject: [PATCH 12/12] Update dh_comms submodule with latest commit (for combined JSON output) Signed-off-by: coleramos425 --- external/dh_comms | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/dh_comms b/external/dh_comms index 837140c..ec5d9d4 160000 --- a/external/dh_comms +++ b/external/dh_comms @@ -1 +1 @@ -Subproject commit 837140c858b458e6ff846f223a9cf02eb3542d1b +Subproject commit ec5d9d42a00f68fa1f37c9a118e5ffe7eb609f1f