diff --git a/.clang-tidy b/.clang-tidy index a142355fc4..23167e93d0 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -18,6 +18,9 @@ CheckOptions: - key: readability-qualified-auto.AllowedTypes value: 'MPI_Comm' + + - key: misc-include-cleaner.IgnoreHeaders + value: 'adios2/.*;bits/.*' --- Disabled checks and reasons: diff --git a/include/bout/adios_object.hxx b/include/bout/adios_object.hxx index 07482a0dcb..fcade4f55c 100755 --- a/include/bout/adios_object.hxx +++ b/include/bout/adios_object.hxx @@ -16,6 +16,8 @@ #if BOUT_HAS_ADIOS2 +#include "bout/boutexception.hxx" + #include #include #include @@ -36,14 +38,12 @@ IOPtr GetIOPtr(const std::string IOName); class ADIOSStream { public: adios2::IO io; - adios2::Engine engine; adios2::Variable vTime; adios2::Variable vStep; int adiosStep = 0; - bool isInStep = false; // true if BeginStep was called and EndStep was not yet called /** create or return the ADIOSStream based on the target file name */ - static ADIOSStream& ADIOSGetStream(const std::string& fname); + static ADIOSStream& ADIOSGetStream(const std::string& fname, adios2::Mode mode); ~ADIOSStream(); @@ -74,14 +74,63 @@ public: return v; } + auto engine() -> adios2::Engine& { + if (not engine_) { + engine_ = io.Open(fname, file_mode); + if (not engine_) { + throw BoutException("Could not open ADIOS file '{:s}' for writing", fname); + } + } + return engine_; + } + + void beginStep() { + if (not isInStep) { + engine().BeginStep(); + isInStep = true; + adiosStep = static_cast(engine().CurrentStep()); + } + } + + void endStep() { + if (isInStep) { + engine().EndStep(); + isInStep = false; + } + } + + void finish() { + if (engine_) { + engine().EndStep(); + engine().Close(); + } + } + private: - ADIOSStream(const std::string fname) : fname(fname){}; + ADIOSStream(const std::string& fname, adios2::Mode mode) + : fname(fname), file_mode(mode) { + + ADIOSPtr adiosp = GetADIOSPtr(); + std::string ioname = "write_" + fname; + try { + io = adiosp->AtIO(ioname); + } catch (const std::invalid_argument& e) { + io = adiosp->DeclareIO(ioname); + io.SetEngine("BP5"); + } + }; + std::string fname; + adios2::Mode file_mode; + adios2::Engine engine_; + + /// true if BeginStep was called and EndStep was not yet called + bool isInStep = false; }; /** Set user parameters for an IO group */ -void ADIOSSetParameters(const std::string& input, const char delimKeyValue, - const char delimItem, adios2::IO& io); +void ADIOSSetParameters(const std::string& input, char delimKeyValue, char delimItem, + adios2::IO& io); } // namespace bout diff --git a/src/sys/adios_object.cxx b/src/sys/adios_object.cxx index 53d8ccaa84..1b3e0a12e2 100644 --- a/src/sys/adios_object.cxx +++ b/src/sys/adios_object.cxx @@ -5,8 +5,8 @@ #include "bout/adios_object.hxx" #include "bout/boutexception.hxx" -#include -#include +#include + #include namespace bout { @@ -48,25 +48,25 @@ IOPtr GetIOPtr(const std::string IOName) { } ADIOSStream::~ADIOSStream() { - if (engine) { + if (engine_) { if (isInStep) { - engine.EndStep(); + engine_.EndStep(); isInStep = false; } - engine.Close(); + engine_.Close(); } } -ADIOSStream& ADIOSStream::ADIOSGetStream(const std::string& fname) { +ADIOSStream& ADIOSStream::ADIOSGetStream(const std::string& fname, adios2::Mode mode) { auto it = adiosStreams.find(fname); if (it == adiosStreams.end()) { - it = adiosStreams.emplace(fname, ADIOSStream(fname)).first; + it = adiosStreams.emplace(fname, ADIOSStream(fname, mode)).first; } return it->second; } -void ADIOSSetParameters(const std::string& input, const char delimKeyValue, - const char delimItem, adios2::IO& io) { +void ADIOSSetParameters(const std::string& input, char delimKeyValue, char delimItem, + adios2::IO& io) { auto lf_Trim = [](std::string& input) { input.erase(0, input.find_first_not_of(" \n\r\t")); // prefixing spaces input.erase(input.find_last_not_of(" \n\r\t") + 1); // suffixing spaces @@ -76,7 +76,7 @@ void ADIOSSetParameters(const std::string& input, const char delimKeyValue, std::string parameter; while (std::getline(inputSS, parameter, delimItem)) { const size_t position = parameter.find(delimKeyValue); - if (position == parameter.npos) { + if (position == std::string::npos) { throw BoutException("ADIOSSetParameters(): wrong format for IO parameter " + parameter + ", format must be key" + delimKeyValue + "value for each entry"); diff --git a/src/sys/options/options_adios.cxx b/src/sys/options/options_adios.cxx index 4a41a014eb..2e64423824 100644 --- a/src/sys/options/options_adios.cxx +++ b/src/sys/options/options_adios.cxx @@ -1,50 +1,172 @@ #include "bout/build_defines.hxx" -#include "bout/traits.hxx" #if BOUT_HAS_ADIOS2 #include "options_adios.hxx" -#include "bout/adios_object.hxx" -#include "bout/bout.hxx" +#include "bout/adios_object.hxx" +#include "bout/array.hxx" +#include "bout/assert.hxx" +#include "bout/bout_types.hxx" #include "bout/boutexception.hxx" +#include "bout/field2d.hxx" #include "bout/globals.hxx" #include "bout/mesh.hxx" +#include "bout/options.hxx" +#include "bout/options_io.hxx" +#include "bout/output.hxx" #include "bout/sys/timer.hxx" +#include "bout/sys/variant.hxx" +#include "bout/traits.hxx" +#include "bout/utils.hxx" -#include "adios2.h" +#include // IWYU pragma: keep +#include +#include +#include #include +#include #include -#include +#include +#include +#include #include #include -namespace bout { -/// Name of the attribute used to track individual variable's time indices -constexpr auto current_time_index_name = "current_time_index"; +namespace { +auto to_int_dims(const adios2::Dims& dims) { + std::vector int_dims; + int_dims.reserve(dims.size()); + std::transform(dims.begin(), dims.end(), std::back_inserter(int_dims), + [](auto dim) { return static_cast(dim); }); + return int_dims; +} -OptionsADIOS::OptionsADIOS(Options& options) : OptionsIO(options) { - if (options["file"].doc("File name. Defaults to /.pb").isSet()) { - filename = options["file"].as(); - } else { - // Both path and prefix must be set - filename = fmt::format("{}/{}.bp", options["path"].as(), - options["prefix"].as()); +// Helper class to construct Adios hyperslices +struct Selection { + // Offset of this processor's data into the global array + adios2::Dims start; + // The size of the mapped region + adios2::Dims count; + // Where the actual data starts in data pointer (to exclude ghost cells) + adios2::Dims mem_start; + // The actual size of data pointer in memory (including ghost cells) + adios2::Dims mem_count; + // Global shape, including boundaries but not guard cells + adios2::Dims shape; + // Shape of the local variable to read into + std::vector dims; + + // Distributed Field/Array/Matrix/Tensor + bool should_set_selection{false}; + + Selection(const std::vector& dim_names, const std::vector& dim_sizes, + const Mesh& mesh) { + const auto ndims = dim_names.size(); + const bool dim0_is_rank = ndims > 0 ? (dim_names[0] == "rank") : false; + const bool dim0_is_x = ndims > 0 ? (dim_names[0] == "x") : false; + const bool dim1_is_y = ndims > 1 ? (dim_names[1] == "y") : false; + const bool dim1_is_z = ndims > 1 ? (dim_names[1] == "z") : false; + const bool dim2_is_z = ndims > 2 ? (dim_names[2] == "z") : false; + + should_set_selection = dim0_is_rank or (ndims == 2 and dim0_is_x and dim1_is_y) + or (ndims == 2 and dim0_is_x and dim1_is_z) + or (ndims == 3 and dim0_is_x and dim1_is_y and dim2_is_z); + + if (dim0_is_rank) { + const auto ndim_sizes = dim_sizes.size(); + ASSERT3(ndim_sizes > 1); + + // This is a distributed array, so the local variable is going + // to be shape (dim_sizes[1]...) (that is, drop the rank) + dims.push_back(dim_sizes[1]); + // but we tell adios to read our rank's bit with the full ndims + start = {static_cast(BoutComm::rank()), 0}; + count = {std::size_t{1}, static_cast(dim_sizes[0])}; + + if (ndim_sizes > 2) { + dims.push_back(dim_sizes[2]); + start.push_back(0); + count.push_back(dim_sizes[1]); + } + if (ndim_sizes > 3) { + dims.push_back(dim_sizes[3]); + start.push_back(0); + count.push_back(dim_sizes[2]); + } + mem_count = count; + mem_start = start; + return; + } + + if (ndims > 0) { + dims.push_back(dim0_is_x ? mesh.LocalNx : dim_sizes[0]); + } + if (ndims > 1) { + if (dim1_is_y) { + dims.push_back(mesh.LocalNy); + } else if (dim1_is_z) { + dims.push_back(mesh.LocalNz); + } else { + dims.push_back(dim_sizes[1]); + } + } + if (ndims > 2) { + dims.push_back(dim2_is_z ? mesh.LocalNz : dim_sizes[2]); + } + + shape.push_back(static_cast(mesh.GlobalNx)); + start.push_back(static_cast(mesh.MapGlobalX)); + count.push_back(static_cast(mesh.MapCountX)); + mem_start.push_back(static_cast(mesh.MapLocalX)); + mem_count.push_back(static_cast(mesh.LocalNx)); + + if (dim1_is_y) { + shape.push_back(static_cast(mesh.GlobalNy)); + start.push_back(static_cast(mesh.MapGlobalY)); + count.push_back(static_cast(mesh.MapCountY)); + mem_start.push_back(static_cast(mesh.MapLocalY)); + mem_count.push_back(static_cast(mesh.LocalNy)); + } else if (dim1_is_z) { + shape.push_back(static_cast(mesh.GlobalNz)); + start.push_back(static_cast(mesh.MapGlobalZ)); + count.push_back(static_cast(mesh.MapCountZ)); + mem_start.push_back(static_cast(mesh.MapLocalZ)); + mem_count.push_back(static_cast(mesh.LocalNz)); + } + + if (dim2_is_z) { + shape.push_back(static_cast(mesh.GlobalNz)); + start.push_back(static_cast(mesh.MapGlobalZ)); + count.push_back(static_cast(mesh.MapCountZ)); + mem_start.push_back(static_cast(mesh.MapLocalZ)); + mem_count.push_back(static_cast(mesh.LocalNz)); + } } - file_mode = (options["append"].doc("Append to existing file?").withDefault(false)) - ? FileMode::append - : FileMode::replace; + auto selection() const { return adios2::Box{start, count}; } + auto memorySelection() const { return adios2::Box{mem_start, mem_count}; } +}; - singleWriteFile = options["singleWriteFile"].withDefault(false); +template