From e2a4d8071667d36bf648c81a66dc72d4f97c84e1 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 24 Sep 2021 18:38:19 -0700 Subject: [PATCH 0001/1425] Start building a Legate runtime and add a bootstrapping logic --- src/legate_c.h | 1 + src/mapping/core_mapper.cc | 12 +++- src/runtime/runtime.cc | 133 ++++++++++++++++++++++++++++++++----- src/runtime/runtime.h | 45 +++++++++++++ src/task/task.cc | 2 +- 5 files changed, 174 insertions(+), 19 deletions(-) diff --git a/src/legate_c.h b/src/legate_c.h index 3a260406e4..57c5d73ff5 100644 --- a/src/legate_c.h +++ b/src/legate_c.h @@ -18,6 +18,7 @@ #define __LEGATE_C_H__ typedef enum legate_core_task_id_t { + LEGATE_CORE_TOPLEVEL_TASK_ID, LEGATE_CORE_INITIALIZE_TASK_ID, LEGATE_CORE_FINALIZE_TASK_ID, LEGATE_CORE_EXTRACT_SCALAR_TASK_ID, diff --git a/src/mapping/core_mapper.cc b/src/mapping/core_mapper.cc index bcfc17efed..19b1f0c6b5 100644 --- a/src/mapping/core_mapper.cc +++ b/src/mapping/core_mapper.cc @@ -200,7 +200,8 @@ Mapper::MapperSyncModel CoreMapper::get_mapper_sync_model(void) const void CoreMapper::select_task_options(const MapperContext ctx, const Task& task, TaskOptions& output) { assert(context.valid_task_id(task.task_id)); - if (task.tag == LEGATE_CPU_VARIANT) { + if (task.tag == LEGATE_CPU_VARIANT || + context.get_local_task_id(task.task_id) == LEGATE_CORE_TOPLEVEL_TASK_ID) { assert(!local_cpus.empty()); output.initial_proc = local_cpus.front(); } else { @@ -277,7 +278,10 @@ void CoreMapper::map_task(const MapperContext ctx, assert(context.valid_task_id(task.task_id)); // Just put our target proc in the target processors for now output.target_procs.push_back(task.target_proc); - output.chosen_variant = task.tag; + if (context.get_local_task_id(task.task_id) == LEGATE_CORE_TOPLEVEL_TASK_ID) + output.chosen_variant = LEGATE_CPU_VARIANT; + else + output.chosen_variant = task.tag; } void CoreMapper::select_sharding_functor(const MapperContext ctx, @@ -368,7 +372,9 @@ void CoreMapper::select_tunable_value(const MapperContext ctx, LEGATE_ABORT } -void register_legate_core_mapper(Machine machine, Runtime* runtime, const LibraryContext& context) +void register_legate_core_mapper(Machine machine, + Legion::Runtime* runtime, + const LibraryContext& context) { // Replace all the default mappers with our custom mapper for the Legate // top-level task and init task diff --git a/src/runtime/runtime.cc b/src/runtime/runtime.cc index bb94f70552..adefa04a89 100644 --- a/src/runtime/runtime.cc +++ b/src/runtime/runtime.cc @@ -93,10 +93,23 @@ static CUDALibraries& get_cuda_libraries(Processor proc, bool check) } #endif +static void toplevel_task(const Task* task, + const std::vector& regions, + Context ctx, + Legion::Runtime* legion_runtime) +{ + auto runtime = Runtime::get_runtime(); + auto main = runtime->get_main_function(); + + auto args = Legion::Runtime::get_input_args(); + + main(args.argc, args.argv, runtime); +} + static void initialize_cpu_resource_task(const Task* task, const std::vector& regions, Context ctx, - Runtime* runtime) + Legion::Runtime* runtime) { // Nothing to do here yet... } @@ -104,7 +117,7 @@ static void initialize_cpu_resource_task(const Task* task, static void finalize_cpu_resource_task(const Task* task, const std::vector& regions, Context ctx, - Runtime* runtime) + Legion::Runtime* runtime) { // Nothing to do here yet... } @@ -112,7 +125,7 @@ static void finalize_cpu_resource_task(const Task* task, static ReturnValues extract_scalar_task(const Task* task, const std::vector& regions, Context legion_context, - Runtime* runtime) + Legion::Runtime* runtime) { TaskContext context(task, regions, legion_context, runtime); auto values = task->futures[0].get_result(); @@ -124,7 +137,7 @@ static ReturnValues extract_scalar_task(const Task* task, static void initialize_gpu_resource_task(const Task* task, const std::vector& regions, Context ctx, - Runtime* runtime) + Legion::Runtime* runtime) { const LegateResource resource = *((const LegateResource*)task->args); switch (resource) { @@ -144,7 +157,7 @@ static void initialize_gpu_resource_task(const Task* task, static void finalize_gpu_resource_task(const Task* task, const std::vector& regions, Context ctx, - Runtime* runtime) + Legion::Runtime* runtime) { CUDALibraries& libs = get_cuda_libraries(task->current_proc, true /*check*/); libs.finalize(); @@ -156,8 +169,15 @@ static void finalize_gpu_resource_task(const Task* task, // Nothing to do here yet... } -void register_legate_core_tasks(Machine machine, Runtime* runtime, const LibraryContext& context) +void register_legate_core_tasks(Machine machine, + Legion::Runtime* runtime, + const LibraryContext& context) { + const TaskID toplevel_task_id = context.get_task_id(LEGATE_CORE_TOPLEVEL_TASK_ID); + const char* toplevel_task_name = "Legate Core Toplevel Task"; + runtime->attach_name( + toplevel_task_id, toplevel_task_name, false /*mutable*/, true /*local only*/); + const TaskID initialize_task_id = context.get_task_id(LEGATE_CORE_INITIALIZE_TASK_ID); const char* initialize_task_name = "Legate Core Resource Initialization"; runtime->attach_name( @@ -182,6 +202,10 @@ void register_legate_core_tasks(Machine machine, Runtime* runtime, const Library }; // Register the task variant for both CPUs and GPUs + { + auto registrar = make_registrar(toplevel_task_id, toplevel_task_name, Processor::LOC_PROC); + runtime->register_task_variant(registrar, LEGATE_CPU_VARIANT); + } { auto registrar = make_registrar(initialize_task_id, initialize_task_name, Processor::LOC_PROC); runtime->register_task_variant(registrar, LEGATE_CPU_VARIANT); @@ -219,26 +243,105 @@ void register_legate_core_tasks(Machine machine, Runtime* runtime, const Library #endif } -/*static*/ void core_registration_callback(Machine machine, - Runtime* runtime, - const std::set& local_procs) +/*static*/ void core_library_registration_callback(Machine machine, + Legion::Runtime* legion_runtime, + const std::set& local_procs) { ResourceConfig config; config.max_tasks = LEGATE_CORE_NUM_TASK_IDS; config.max_projections = LEGATE_CORE_MAX_FUNCTOR_ID; // We register one sharding functor for each new projection functor config.max_shardings = LEGATE_CORE_MAX_FUNCTOR_ID; - LibraryContext context(runtime, core_library_name, config); - register_legate_core_tasks(machine, runtime, context); + auto runtime = Runtime::get_runtime(); + auto core_lib = runtime->create_library(core_library_name, config); + + register_legate_core_tasks(machine, legion_runtime, *core_lib); + + register_legate_core_mapper(machine, legion_runtime, *core_lib); + + register_legate_core_projection_functors(legion_runtime, *core_lib); + + register_legate_core_sharding_functors(legion_runtime, *core_lib); +} + +/*static*/ void core_library_bootstrapping_callback(Machine machine, + Legion::Runtime* legion_runtime, + const std::set& local_procs) +{ + core_library_registration_callback(machine, legion_runtime, local_procs); + + auto runtime = Runtime::get_runtime(); + auto core_lib = runtime->find_library(core_library_name); + legion_runtime->set_top_level_task_id(core_lib->get_task_id(LEGATE_CORE_TOPLEVEL_TASK_ID)); + legion_runtime->set_top_level_task_mapper_id(core_lib->get_mapper_id(0)); +} + +/*static*/ Runtime* Runtime::runtime_; + +Runtime::Runtime() {} + +Runtime::~Runtime() +{ + for (auto& pair : libraries_) delete pair.second; +} + +void Runtime::set_main_function(MainFnPtr main_fn) { main_fn_ = main_fn; } + +Runtime::MainFnPtr Runtime::get_main_function() const { return main_fn_; } + +LibraryContext* Runtime::find_library(const std::string& library_name, + bool can_fail /*=false*/) const +{ + auto finder = libraries_.find(library_name); + if (libraries_.end() == finder) { + if (!can_fail) { + log_legate.error("Library %s does not exist", library_name.c_str()); + LEGATE_ABORT + } else + return nullptr; + } + return finder->second; +} + +LibraryContext* Runtime::create_library(const std::string& library_name, + const ResourceConfig& config) +{ + if (libraries_.find(library_name) != libraries_.end()) { + log_legate.error("Library %s already exists", library_name.c_str()); + LEGATE_ABORT + } + + log_legate.debug("Library %s is created", library_name.c_str()); + auto context = new LibraryContext(Legion::Runtime::get_runtime(), library_name, config); + libraries_[library_name] = context; + return context; +} - register_legate_core_mapper(machine, runtime, context); +/*static*/ void Runtime::initialize(int32_t argc, char** argv) +{ + Legion::Runtime::initialize(&argc, &argv, true /*filter legion and realm args*/); + Legion::Runtime::perform_registration_callback(legate::core_library_bootstrapping_callback, + true /*global*/); + runtime_ = new Runtime(); +} - register_legate_core_projection_functors(runtime, context); +/*static*/ int32_t Runtime::start(int32_t argc, char** argv) +{ + return Legion::Runtime::start(argc, argv); +} + +/*static*/ Runtime* Runtime::get_runtime() { return Runtime::runtime_; } + +void initialize(int32_t argc, char** argv) { Runtime::initialize(argc, argv); } - register_legate_core_sharding_functors(runtime, context); +void set_main_function(Runtime::MainFnPtr main_fn) +{ + Runtime::get_runtime()->set_main_function(main_fn); } +int32_t start(int32_t argc, char** argv) { return Runtime::start(argc, argv); } + } // namespace legate extern "C" { @@ -247,7 +350,7 @@ void legate_core_perform_registration() { // Tell the runtime about our registration callback so we can register ourselves // Make sure it is global so this shared object always gets loaded on all nodes - Legion::Runtime::perform_registration_callback(legate::core_registration_callback, + Legion::Runtime::perform_registration_callback(legate::core_library_registration_callback, true /*global*/); } } diff --git a/src/runtime/runtime.h b/src/runtime/runtime.h index 82d2539729..f9e5e739cb 100644 --- a/src/runtime/runtime.h +++ b/src/runtime/runtime.h @@ -40,4 +40,49 @@ class Core { #endif }; +class ResourceConfig; +class LibraryContext; + +class Runtime { + public: + using MainFnPtr = void (*)(int32_t, char**, Runtime*); + + public: + Runtime(); + ~Runtime(); + + public: + friend void initialize(int32_t argc, char** argv); + friend void set_main_function(MainFnPtr p_main); + friend int32_t start(int32_t argc, char** argv); + + public: + void set_main_function(MainFnPtr main_fn); + MainFnPtr get_main_function() const; + + public: + LibraryContext* find_library(const std::string& library_name, bool can_fail = false) const; + LibraryContext* create_library(const std::string& library_name, const ResourceConfig& config); + + public: + static void initialize(int32_t argc, char** argv); + static int32_t start(int32_t argc, char** argv); + static Runtime* get_runtime(); + + private: + static Runtime* runtime_; + + private: + std::map libraries_; + + private: + MainFnPtr main_fn_{nullptr}; +}; + +void initialize(int32_t argc, char** argv); + +void set_main_function(Runtime::MainFnPtr p_main); + +int32_t start(int32_t argc, char** argv); + } // namespace legate diff --git a/src/task/task.cc b/src/task/task.cc index 2044e5f77d..f417ca944b 100644 --- a/src/task/task.cc +++ b/src/task/task.cc @@ -54,7 +54,7 @@ void LegateTaskRegistrar::record_variant(TaskID tid, registrar.set_idempotent(idempotent); } -void LegateTaskRegistrar::register_all_tasks(Runtime* runtime, LibraryContext& context) +void LegateTaskRegistrar::register_all_tasks(Legion::Runtime* runtime, LibraryContext& context) { // Do all our registrations for (auto& task : pending_task_variants_) { From 3c2a20ee827348b0a60f8a74a365bbe40d81bfa2 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Mon, 27 Sep 2021 17:02:43 -0700 Subject: [PATCH 0002/1425] Start building region and field managers and add logical store abstractions --- src/core.mk | 2 + src/data/logical_store.cc | 64 ++++++++++++++++ src/data/logical_store.h | 96 +++++++++++++++++++++++ src/runtime/runtime.cc | 157 +++++++++++++++++++++++++++++++++++++- src/runtime/runtime.h | 69 +++++++++++++++++ 5 files changed, 385 insertions(+), 3 deletions(-) create mode 100644 src/data/logical_store.cc create mode 100644 src/data/logical_store.h diff --git a/src/core.mk b/src/core.mk index ed1dec64b9..4f7bcebb24 100644 --- a/src/core.mk +++ b/src/core.mk @@ -16,6 +16,7 @@ # General source files GEN_CPU_SRC = legate_c.cc \ + data/logical_store.cc \ data/scalar.cc \ data/store.cc \ data/transform.cc \ @@ -50,6 +51,7 @@ INSTALL_HEADERS = legate.h \ legate_defines.h \ legate_preamble.h \ data/buffer.h \ + data/logical_store.h \ data/scalar.h \ data/scalar.inl \ data/store.h \ diff --git a/src/data/logical_store.cc b/src/data/logical_store.cc new file mode 100644 index 0000000000..188edef009 --- /dev/null +++ b/src/data/logical_store.cc @@ -0,0 +1,64 @@ +/* Copyright 2021 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "data/logical_store.h" + +#include "runtime/runtime.h" + +using namespace Legion; + +namespace legate { + +LogicalRegionField::LogicalRegionField(Runtime* runtime, const LogicalRegion& lr, FieldID fid) + : runtime_(runtime), lr_(lr), fid_(fid) +{ +} + +int32_t LogicalRegionField::dim() const { return lr_.get_dim(); } + +Domain LogicalRegionField::domain() const +{ + return runtime_->get_index_space_domain(lr_.get_index_space()); +} + +LogicalStore::LogicalStore(Runtime* runtime, + LegateTypeCode code, + std::vector extents, + std::shared_ptr transform /*= nullptr*/) + : runtime_(runtime), code_(code), extents_(std::move(extents)), transform_(std::move(transform)) +{ +} + +int32_t LogicalStore::dim() const { return static_cast(extents_.size()); } + +Domain LogicalStore::domain() const +{ + assert(nullptr != region_field_); + return region_field_->domain(); +} + +std::shared_ptr LogicalStore::get_storage() +{ + if (!has_storage()) create_storage(); + return region_field_; +} + +void LogicalStore::create_storage() +{ + region_field_ = runtime_->create_region_field(extents_, code_); +} + +} // namespace legate diff --git a/src/data/logical_store.h b/src/data/logical_store.h new file mode 100644 index 0000000000..1595f8103a --- /dev/null +++ b/src/data/logical_store.h @@ -0,0 +1,96 @@ +/* Copyright 2021 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include + +#include "legion.h" + +#include "data/transform.h" +#include "utilities/typedefs.h" + +namespace legate { + +class Runtime; + +class LogicalRegionField { + public: + LogicalRegionField() {} + LogicalRegionField(Runtime* runtime, const Legion::LogicalRegion& lr, Legion::FieldID fid); + + public: + LogicalRegionField(const LogicalRegionField& other) = default; + LogicalRegionField& operator=(const LogicalRegionField& other) = default; + + public: + int32_t dim() const; + Legion::LogicalRegion region() const { return lr_; } + Legion::FieldID field_id() const { return fid_; } + + // public: + // RegionField map(); + + public: + // template + // Legion::Rect shape() const; + Legion::Domain domain() const; + + private: + Runtime* runtime_{nullptr}; + Legion::LogicalRegion lr_{}; + Legion::FieldID fid_{-1U}; +}; + +class LogicalStore { + public: + LogicalStore() {} + LogicalStore(Runtime* runtime, + LegateTypeCode code, + std::vector extents, + std::shared_ptr transform = nullptr); + + public: + LogicalStore(const LogicalStore& other) = default; + LogicalStore& operator=(const LogicalStore& other) = default; + + public: + LogicalStore(LogicalStore&& other) = default; + LogicalStore& operator=(LogicalStore&& other) = default; + + public: + int32_t dim() const; + LegateTypeCode code() const { return code_; } + + public: + Legion::Domain domain() const; + + public: + bool has_storage() const { return nullptr != region_field_; } + std::shared_ptr get_storage(); + + private: + void create_storage(); + + private: + Runtime* runtime_{nullptr}; + LegateTypeCode code_{MAX_TYPE_NUMBER}; + std::vector extents_; + std::shared_ptr region_field_{nullptr}; + std::shared_ptr transform_{nullptr}; +}; + +} // namespace legate diff --git a/src/runtime/runtime.cc b/src/runtime/runtime.cc index adefa04a89..1c32fe12fe 100644 --- a/src/runtime/runtime.cc +++ b/src/runtime/runtime.cc @@ -14,6 +14,7 @@ * */ +#include "data/logical_store.h" #include "legate.h" #include "mapping/core_mapper.h" #include "runtime/context.h" @@ -99,10 +100,10 @@ static void toplevel_task(const Task* task, Legion::Runtime* legion_runtime) { auto runtime = Runtime::get_runtime(); - auto main = runtime->get_main_function(); + runtime->set_legion_context(ctx); + auto main = runtime->get_main_function(); auto args = Legion::Runtime::get_input_args(); - main(args.argc, args.argv, runtime); } @@ -271,12 +272,74 @@ void register_legate_core_tasks(Machine machine, { core_library_registration_callback(machine, legion_runtime, local_procs); - auto runtime = Runtime::get_runtime(); + auto runtime = Runtime::get_runtime(); + runtime->set_legion_runtime(legion_runtime); + auto core_lib = runtime->find_library(core_library_name); legion_runtime->set_top_level_task_id(core_lib->get_task_id(LEGATE_CORE_TOPLEVEL_TASK_ID)); legion_runtime->set_top_level_task_mapper_id(core_lib->get_mapper_id(0)); } +//////////////////////////////////////////////////// +// legate::RegionManager +//////////////////////////////////////////////////// + +RegionManager::RegionManager(Runtime* runtime, const Domain& shape) + : runtime_(runtime), shape_(shape) +{ +} + +LogicalRegion RegionManager::active_region() const { return regions_.back(); } + +bool RegionManager::has_space() const { return regions_.size() > 0; } + +void RegionManager::create_region() +{ + auto is = runtime_->find_or_create_index_space(shape_); + auto fs = runtime_->create_field_space(); + regions_.push_back(runtime_->create_region(is, fs)); +} + +std::pair RegionManager::allocate_field(size_t field_size) +{ + if (!has_space()) create_region(); + auto lr = active_region(); + auto fid = runtime_->allocate_field(lr.get_field_space(), field_size); + return std::make_pair(lr, fid); +} + +//////////////////////////////////////////////////// +// legate::FieldManager +//////////////////////////////////////////////////// + +struct field_size_fn { + template + size_t operator()() + { + return sizeof(legate_type_of); + } +}; + +static size_t get_field_size(LegateTypeCode code) { return type_dispatch(code, field_size_fn{}); } + +FieldManager::FieldManager(Runtime* runtime, const Legion::Domain& shape, LegateTypeCode code) + : runtime_(runtime), shape_(shape), code_(code), field_size_(get_field_size(code)) +{ +} + +std::shared_ptr FieldManager::allocate_field() +{ + auto rgn_mgr = runtime_->find_or_create_region_manager(shape_); + LogicalRegion lr; + FieldID fid; + std::tie(lr, fid) = rgn_mgr->allocate_field(field_size_); + return std::make_shared(runtime_, lr, fid); +} + +//////////////////////////////////////////////////// +// legate::Runtime +//////////////////////////////////////////////////// + /*static*/ Runtime* Runtime::runtime_; Runtime::Runtime() {} @@ -318,6 +381,94 @@ LibraryContext* Runtime::create_library(const std::string& library_name, return context; } +void Runtime::set_legion_runtime(Legion::Runtime* legion_runtime) +{ + legion_runtime_ = legion_runtime; +} + +void Runtime::set_legion_context(Legion::Context legion_context) +{ + legion_context_ = legion_context; +} + +std::shared_ptr Runtime::create_store(std::vector extents, + LegateTypeCode code) +{ + return std::make_shared(this, code, extents); +} + +std::shared_ptr Runtime::create_region_field( + const std::vector& extents, LegateTypeCode code) +{ + DomainPoint lo, hi; + hi.dim = lo.dim = static_cast(extents.size()); + assert(lo.dim <= LEGION_MAX_DIM); + for (int32_t dim = 0; dim < lo.dim; ++dim) lo[dim] = 0; + for (int32_t dim = 0; dim < lo.dim; ++dim) hi[dim] = extents[dim] - 1; + + Domain shape(lo, hi); + auto fld_mgr = runtime_->find_or_create_field_manager(shape, code); + return fld_mgr->allocate_field(); +} + +RegionManager* Runtime::find_or_create_region_manager(const Domain& shape) +{ + auto finder = region_managers_.find(shape); + if (finder != region_managers_.end()) + return finder->second; + else { + auto rgn_mgr = new RegionManager(this, shape); + region_managers_[shape] = rgn_mgr; + return rgn_mgr; + } +} + +FieldManager* Runtime::find_or_create_field_manager(const Domain& shape, LegateTypeCode code) +{ + auto key = std::make_pair(shape, code); + auto finder = field_managers_.find(key); + if (finder != field_managers_.end()) + return finder->second; + else { + auto fld_mgr = new FieldManager(this, shape, code); + field_managers_[key] = fld_mgr; + return fld_mgr; + } +} + +IndexSpace Runtime::find_or_create_index_space(const Domain& shape) +{ + auto finder = index_spaces_.find(shape); + if (finder != index_spaces_.end()) + return finder->second; + else { + auto is = legion_runtime_->create_index_space(legion_context_, shape); + index_spaces_[shape] = is; + return is; + } +} + +FieldSpace Runtime::create_field_space() +{ + return legion_runtime_->create_field_space(legion_context_); +} + +LogicalRegion Runtime::create_region(const IndexSpace& index_space, const FieldSpace& field_space) +{ + return legion_runtime_->create_logical_region(legion_context_, index_space, field_space); +} + +FieldID Runtime::allocate_field(const FieldSpace& field_space, size_t field_size) +{ + auto allocator = legion_runtime_->create_field_allocator(legion_context_, field_space); + return allocator.allocate_field(field_size); +} + +Domain Runtime::get_index_space_domain(const IndexSpace& index_space) const +{ + return legion_runtime_->get_index_space_domain(legion_context_, index_space); +} + /*static*/ void Runtime::initialize(int32_t argc, char** argv) { Legion::Runtime::initialize(&argc, &argv, true /*filter legion and realm args*/); diff --git a/src/runtime/runtime.h b/src/runtime/runtime.h index f9e5e739cb..85dba1cc50 100644 --- a/src/runtime/runtime.h +++ b/src/runtime/runtime.h @@ -16,6 +16,8 @@ #pragma once +#include + #include "legion.h" #include "utilities/typedefs.h" @@ -41,7 +43,42 @@ class Core { }; class ResourceConfig; +class Runtime; class LibraryContext; +class LogicalRegionField; +class LogicalStore; + +class RegionManager { + public: + RegionManager(Runtime* runtime, const Legion::Domain& shape); + + private: + Legion::LogicalRegion active_region() const; + bool has_space() const; + void create_region(); + + public: + std::pair allocate_field(size_t field_size); + + private: + Runtime* runtime_; + Legion::Domain shape_; + std::vector regions_{}; +}; + +class FieldManager { + public: + FieldManager(Runtime* runtime, const Legion::Domain& shape, LegateTypeCode code); + + public: + std::shared_ptr allocate_field(); + + private: + Runtime* runtime_; + Legion::Domain shape_; + LegateTypeCode code_; + size_t field_size_; +}; class Runtime { public: @@ -64,6 +101,27 @@ class Runtime { LibraryContext* find_library(const std::string& library_name, bool can_fail = false) const; LibraryContext* create_library(const std::string& library_name, const ResourceConfig& config); + public: + void set_legion_runtime(Legion::Runtime* legion_runtime); + void set_legion_context(Legion::Context legion_context); + + public: + std::shared_ptr create_store(std::vector extents, LegateTypeCode code); + std::shared_ptr create_region_field(const std::vector& extents, + LegateTypeCode code); + + public: + RegionManager* find_or_create_region_manager(const Legion::Domain& shape); + FieldManager* find_or_create_field_manager(const Legion::Domain& shape, LegateTypeCode code); + + public: + Legion::IndexSpace find_or_create_index_space(const Legion::Domain& shape); + Legion::FieldSpace create_field_space(); + Legion::LogicalRegion create_region(const Legion::IndexSpace& index_space, + const Legion::FieldSpace& field_space); + Legion::FieldID allocate_field(const Legion::FieldSpace& field_space, size_t field_size); + Legion::Domain get_index_space_domain(const Legion::IndexSpace& index_space) const; + public: static void initialize(int32_t argc, char** argv); static int32_t start(int32_t argc, char** argv); @@ -72,6 +130,17 @@ class Runtime { private: static Runtime* runtime_; + private: + Legion::Runtime* legion_runtime_{nullptr}; + Legion::Context legion_context_{nullptr}; + + private: + std::map, FieldManager*> field_managers_; + std::map region_managers_; + + private: + std::map index_spaces_; + private: std::map libraries_; From e5f099592c51d848f955c031404dddd41a22f97a Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 29 Sep 2021 00:53:03 -0700 Subject: [PATCH 0003/1425] Initial code path to create and launch tasks --- src/core.mk | 3 + src/data/scalar.h | 1 + src/legate.h | 1 + src/mapping/core_mapper.cc | 28 ++-- src/runtime/launcher.cc | 331 +++++++++++++++++++++++++++++++++++++ src/runtime/launcher.h | 146 ++++++++++++++++ src/runtime/operation.cc | 60 +++++++ src/runtime/operation.h | 72 ++++++++ src/runtime/runtime.cc | 42 ++++- src/runtime/runtime.h | 11 ++ 10 files changed, 676 insertions(+), 19 deletions(-) create mode 100644 src/runtime/launcher.cc create mode 100644 src/runtime/launcher.h create mode 100644 src/runtime/operation.cc create mode 100644 src/runtime/operation.h diff --git a/src/core.mk b/src/core.mk index 4f7bcebb24..de02575543 100644 --- a/src/core.mk +++ b/src/core.mk @@ -26,6 +26,8 @@ GEN_CPU_SRC = legate_c.cc \ mapping/mapping.cc \ mapping/task.cc \ runtime/context.cc \ + runtime/launcher.cc \ + runtime/operation.cc \ runtime/projection.cc \ runtime/runtime.cc \ runtime/shard.cc \ @@ -62,6 +64,7 @@ INSTALL_HEADERS = legate.h \ mapping/task.h \ mapping/task.inl \ runtime/context.h \ + runtime/operation.h \ runtime/runtime.h \ task/return.h \ task/task.h \ diff --git a/src/data/scalar.h b/src/data/scalar.h index 34a9e7b155..46173c9f77 100644 --- a/src/data/scalar.h +++ b/src/data/scalar.h @@ -37,6 +37,7 @@ class Scalar { public: bool is_tuple() const { return tuple_; } + LegateTypeCode code() const { return code_; } size_t size() const; public: diff --git a/src/legate.h b/src/legate.h index 5c886cd8ca..6950fe080a 100644 --- a/src/legate.h +++ b/src/legate.h @@ -22,6 +22,7 @@ #include "data/store.h" #include "legate_c.h" #include "legate_defines.h" +#include "runtime/operation.h" #include "runtime/runtime.h" #include "task/task.h" #include "utilities/deserializer.h" diff --git a/src/mapping/core_mapper.cc b/src/mapping/core_mapper.cc index 19b1f0c6b5..508162c30d 100644 --- a/src/mapping/core_mapper.cc +++ b/src/mapping/core_mapper.cc @@ -57,17 +57,19 @@ class CoreMapper : public Legion::Mapping::NullMapper { virtual bool request_valid_instances(void) const { return false; } public: // Task mapping calls - virtual void select_task_options(const MapperContext ctx, const Task& task, TaskOptions& output); + virtual void select_task_options(const MapperContext ctx, + const Legion::Task& task, + TaskOptions& output); virtual void slice_task(const MapperContext ctx, - const Task& task, + const Legion::Task& task, const SliceTaskInput& input, SliceTaskOutput& output); virtual void map_task(const MapperContext ctx, - const Task& task, + const Legion::Task& task, const MapTaskInput& input, MapTaskOutput& output); virtual void select_sharding_functor(const MapperContext ctx, - const Task& task, + const Legion::Task& task, const SelectShardingFunctorInput& input, SelectShardingFunctorOutput& output); virtual void select_steal_targets(const MapperContext ctx, @@ -79,13 +81,13 @@ class CoreMapper : public Legion::Mapping::NullMapper { public: virtual void configure_context(const MapperContext ctx, - const Task& task, + const Legion::Task& task, ContextConfigOutput& output); void map_future_map_reduction(const MapperContext ctx, const FutureMapReductionInput& input, FutureMapReductionOutput& output); virtual void select_tunable_value(const MapperContext ctx, - const Task& task, + const Legion::Task& task, const SelectTunableInput& input, SelectTunableOutput& output); void pack_tunable(const int value, Mapper::SelectTunableOutput& output); @@ -197,7 +199,9 @@ Mapper::MapperSyncModel CoreMapper::get_mapper_sync_model(void) const return SERIALIZED_REENTRANT_MAPPER_MODEL; } -void CoreMapper::select_task_options(const MapperContext ctx, const Task& task, TaskOptions& output) +void CoreMapper::select_task_options(const MapperContext ctx, + const Legion::Task& task, + TaskOptions& output) { assert(context.valid_task_id(task.task_id)); if (task.tag == LEGATE_CPU_VARIANT || @@ -212,7 +216,7 @@ void CoreMapper::select_task_options(const MapperContext ctx, const Task& task, } void CoreMapper::slice_task(const MapperContext ctx, - const Task& task, + const Legion::Task& task, const SliceTaskInput& input, SliceTaskOutput& output) { @@ -271,7 +275,7 @@ void CoreMapper::slice_task(const MapperContext ctx, } void CoreMapper::map_task(const MapperContext ctx, - const Task& task, + const Legion::Task& task, const MapTaskInput& input, MapTaskOutput& output) { @@ -285,7 +289,7 @@ void CoreMapper::map_task(const MapperContext ctx, } void CoreMapper::select_sharding_functor(const MapperContext ctx, - const Task& task, + const Legion::Task& task, const SelectShardingFunctorInput& input, SelectShardingFunctorOutput& output) { @@ -311,7 +315,7 @@ void CoreMapper::select_tasks_to_map(const MapperContext ctx, } void CoreMapper::configure_context(const MapperContext ctx, - const Task& task, + const Legion::Task& task, ContextConfigOutput& output) { // Use the defaults currently @@ -332,7 +336,7 @@ void CoreMapper::map_future_map_reduction(const MapperContext ctx, } void CoreMapper::select_tunable_value(const MapperContext ctx, - const Task& task, + const Legion::Task& task, const SelectTunableInput& input, SelectTunableOutput& output) { diff --git a/src/runtime/launcher.cc b/src/runtime/launcher.cc new file mode 100644 index 0000000000..eb5ecd5b02 --- /dev/null +++ b/src/runtime/launcher.cc @@ -0,0 +1,331 @@ +/* Copyright 2021 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "runtime/launcher.h" +#include "data/logical_store.h" +#include "data/scalar.h" +#include "runtime/context.h" +#include "runtime/runtime.h" + +namespace legate { + +class BufferBuilder { + public: + BufferBuilder(); + + public: + template + void pack(const T& value); + void pack_buffer(const void* buffer, size_t size); + + public: + Legion::UntypedBuffer to_legion_buffer() const; + + private: + std::vector buffer_; +}; + +class RequirementAnalyzer { + private: + using SingleTask = Legion::TaskLauncher*; + + public: + ~RequirementAnalyzer(); + + public: + void insert(RegionReq* req, Legion::FieldID field_id); + uint32_t get_requirement_index(RegionReq* req, Legion::FieldID field_id) const; + + public: + void analyze_requirements(); + void populate_launcher(SingleTask task) const; + + private: + std::map req_indices_; + std::vector>> requirements_; +}; + +struct ArgWrapper { + virtual void pack(BufferBuilder& buffer) const = 0; +}; + +template +struct ScalarArg : public ArgWrapper { + public: + ScalarArg(const T& value) : value_(value) {} + + public: + virtual void pack(BufferBuilder& buffer) const override; + + private: + T value_; +}; + +struct UntypedScalarArg : public ArgWrapper { + public: + UntypedScalarArg(const Scalar& scalar) : scalar_(scalar) {} + + public: + virtual void pack(BufferBuilder& buffer) const override; + + private: + Scalar scalar_; +}; + +struct RegionFieldArg : public ArgWrapper { + private: + using LogicalStoreP = std::shared_ptr; + + public: + RegionFieldArg(RequirementAnalyzer* analyzer, + LogicalStoreP store, + int32_t dim, + RegionReq* req, + Legion::FieldID field_id, + Legion::ReductionOpID redop); + + public: + virtual void pack(BufferBuilder& buffer) const override; + + private: + RequirementAnalyzer* analyzer_; + LogicalStoreP store_; + int32_t dim_; + RegionReq* req_; + Legion::FieldID field_id_; + Legion::ReductionOpID redop_; +}; + +BufferBuilder::BufferBuilder() +{ + // Reserve 4KB to minimize resizing while packing the arguments. + buffer_.reserve(4096); +} + +template +void BufferBuilder::pack(const T& value) +{ + pack_buffer(reinterpret_cast(&value), sizeof(T)); +} + +void BufferBuilder::pack_buffer(const void* src, size_t size) +{ + auto tgt = buffer_.data() + buffer_.size(); + buffer_.resize(buffer_.size() + size); + memcpy(tgt, src, size); +} + +Legion::UntypedBuffer BufferBuilder::to_legion_buffer() const +{ + return Legion::UntypedBuffer(buffer_.data(), buffer_.size()); +} + +RequirementAnalyzer::~RequirementAnalyzer() +{ + for (auto& pair : requirements_) delete pair.first; +} + +void RequirementAnalyzer::insert(RegionReq* req, Legion::FieldID field_id) +{ + uint32_t req_idx = static_cast(requirements_.size()); + requirements_.push_back(std::make_pair(req, std::vector({field_id}))); + req_indices_[req] = req_idx; +} + +uint32_t RequirementAnalyzer::get_requirement_index(RegionReq* req, Legion::FieldID field_id) const +{ + auto finder = req_indices_.find(req); + assert(finder != req_indices_.end()); + return finder->second; +} + +void RequirementAnalyzer::analyze_requirements() {} + +void RequirementAnalyzer::populate_launcher(SingleTask task) const +{ + for (auto& pair : requirements_) { + auto& req = pair.first; + auto& fields = pair.second; + req->proj->add(task, *req, fields); + } +} + +template +void ScalarArg::pack(BufferBuilder& buffer) const +{ + buffer.pack(value_); +} + +void UntypedScalarArg::pack(BufferBuilder& buffer) const +{ + buffer.pack(scalar_.is_tuple()); + buffer.pack(scalar_.code()); + buffer.pack_buffer(scalar_.ptr(), scalar_.size()); +} + +RegionFieldArg::RegionFieldArg(RequirementAnalyzer* analyzer, + LogicalStoreP store, + int32_t dim, + RegionReq* req, + Legion::FieldID field_id, + Legion::ReductionOpID redop) + : analyzer_(analyzer), store_(store), dim_(dim), req_(req), field_id_(field_id), redop_(redop) +{ +} + +void RegionFieldArg::pack(BufferBuilder& buffer) const +{ + buffer.pack(false); + buffer.pack(store_->dim()); + buffer.pack(store_->code()); + buffer.pack(-1); + + buffer.pack(redop_); + buffer.pack(dim_); + buffer.pack(analyzer_->get_requirement_index(req_, field_id_)); + buffer.pack(field_id_); +} + +Projection::Projection(Legion::ReductionOpID r) : redop(std::make_unique(r)) +{ +} + +Broadcast::Broadcast() : Projection() {} + +Broadcast::Broadcast(Legion::ReductionOpID redop) : Projection(redop) {} +void Broadcast::add(SingleTask task, + const RegionReq& req, + const std::vector& fields) const +{ + if (req.priv == REDUCE) { + Legion::RegionRequirement legion_req(req.region, *redop, EXCLUSIVE, req.region, req.tag); + legion_req.add_fields(fields); + task->add_region_requirement(legion_req); + } else { + Legion::RegionRequirement legion_req(req.region, req.priv, EXCLUSIVE, req.region, req.tag); + legion_req.add_fields(fields); + task->add_region_requirement(legion_req); + } +} + +RegionReq::RegionReq(Legion::LogicalRegion _region, + Legion::PrivilegeMode _priv, + ProjectionP _proj, + int64_t _tag) + : region(_region), priv(_priv), proj(std::move(_proj)), tag(_tag) +{ +} + +TaskLauncher::TaskLauncher(Runtime* runtime, + LibraryContext* library, + int64_t task_id, + int64_t mapper_id /*= 0*/, + int64_t tag /*= 0*/) + : runtime_(runtime), library_(library), task_id_(task_id), mapper_id_(mapper_id), tag_(tag) +{ + req_analyzer_ = new RequirementAnalyzer(); + buffer_ = new BufferBuilder(); +} + +TaskLauncher::~TaskLauncher() +{ + for (auto& arg : inputs_) delete arg; + for (auto& arg : outputs_) delete arg; + for (auto& arg : reductions_) delete arg; + for (auto& arg : scalars_) delete arg; + + delete req_analyzer_; + delete buffer_; +} + +int64_t TaskLauncher::legion_task_id() const { return library_->get_task_id(task_id_); } + +int64_t TaskLauncher::legion_mapper_id() const { return library_->get_mapper_id(mapper_id_); } + +void TaskLauncher::add_scalar(const Scalar& scalar) +{ + scalars_.push_back(new UntypedScalarArg(scalar)); +} + +void TaskLauncher::add_input(LogicalStoreP store, ProjectionP proj, uint64_t tag /*= 0*/) +{ + add_store(inputs_, std::move(store), std::move(proj), READ_ONLY, tag); +} + +void TaskLauncher::add_output(LogicalStoreP store, ProjectionP proj, uint64_t tag /*= 0*/) +{ + add_store(outputs_, std::move(store), std::move(proj), WRITE_ONLY, tag); +} + +void TaskLauncher::add_reduction(LogicalStoreP store, + ProjectionP proj, + uint64_t tag /*= 0*/, + bool read_write /*= false*/) +{ + assert(!read_write); + add_store(reductions_, std::move(store), std::move(proj), REDUCE, tag); +} + +void TaskLauncher::execute_single() +{ + auto legion_launcher = build_single_task(); + runtime_->dispatch(legion_launcher); + delete legion_launcher; +} + +void TaskLauncher::add_store(std::vector& args, + LogicalStoreP store, + ProjectionP proj, + Legion::PrivilegeMode privilege, + uint64_t tag) +{ + auto storage = store->get_storage(); + auto region = storage->region(); + auto field_id = storage->field_id(); + + auto redop = nullptr != proj->redop ? *proj->redop : -1; + auto req = new RegionReq(region, privilege, std::move(proj), tag); + + req_analyzer_->insert(req, field_id); + args.push_back( + new RegionFieldArg(req_analyzer_, std::move(store), region.get_dim(), req, field_id, redop)); +} + +void TaskLauncher::pack_args(const std::vector& args) +{ + buffer_->pack(args.size()); + for (auto& arg : args) arg->pack(*buffer_); +} + +TaskLauncher::SingleTask TaskLauncher::build_single_task() +{ + pack_args(inputs_); + pack_args(outputs_); + pack_args(reductions_); + pack_args(scalars_); + + auto single_task = new Legion::TaskLauncher(legion_task_id(), + buffer_->to_legion_buffer(), + Legion::Predicate::TRUE_PRED, + legion_mapper_id(), + tag_); + + req_analyzer_->populate_launcher(single_task); + + return single_task; +} + +} // namespace legate diff --git a/src/runtime/launcher.h b/src/runtime/launcher.h new file mode 100644 index 0000000000..6dc3eb7f12 --- /dev/null +++ b/src/runtime/launcher.h @@ -0,0 +1,146 @@ +/* Copyright 2021 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include + +#include "legion.h" + +namespace legate { + +class BufferBuilder; +class RegionReq; +class RequirementAnalyzer; +class ArgWrapper; + +class LogicalStore; +class Scalar; + +class Runtime; +class LibraryContext; + +class Projection { + protected: + using SingleTask = Legion::TaskLauncher*; + + protected: + Projection() {} + Projection(Legion::ReductionOpID redop); + + public: + virtual ~Projection() {} + + public: + virtual void add(SingleTask task, + const RegionReq& req, + const std::vector& fields) const = 0; + + public: + std::unique_ptr redop{nullptr}; +}; + +class Broadcast : public Projection { + public: + Broadcast(); + Broadcast(Legion::ReductionOpID redop); + + public: + virtual ~Broadcast() {} + + public: + virtual void add(SingleTask task, + const RegionReq& req, + const std::vector& fields) const override; +}; + +class RegionReq { + private: + using ProjectionP = std::unique_ptr; + + public: + RegionReq(Legion::LogicalRegion region, + Legion::PrivilegeMode priv, + ProjectionP proj, + int64_t tag); + + public: + Legion::LogicalRegion region; + Legion::PrivilegeMode priv; + ProjectionP proj; + int64_t tag; +}; + +class TaskLauncher { + private: + using LogicalStoreP = std::shared_ptr; + using ProjectionP = std::unique_ptr; + using SingleTask = Legion::TaskLauncher*; + + public: + TaskLauncher(Runtime* runtime, + LibraryContext* library, + int64_t task_id, + int64_t mapper_id = 0, + int64_t tag = 0); + ~TaskLauncher(); + + public: + int64_t legion_task_id() const; + int64_t legion_mapper_id() const; + + public: + void add_scalar(const Scalar& scalar); + void add_input(LogicalStoreP store, ProjectionP proj, uint64_t tag = 0); + void add_output(LogicalStoreP store, ProjectionP proj, uint64_t tag = 0); + void add_reduction(LogicalStoreP store, + ProjectionP proj, + uint64_t tag = 0, + bool read_write = false); + + private: + void add_store(std::vector& args, + LogicalStoreP store, + ProjectionP proj, + Legion::PrivilegeMode privilege, + uint64_t tag); + + public: + void execute_single(); + + private: + void pack_args(const std::vector& args); + SingleTask build_single_task(); + + private: + Runtime* runtime_; + LibraryContext* library_; + int64_t task_id_; + int64_t mapper_id_; + int64_t tag_; + + private: + std::vector inputs_; + std::vector outputs_; + std::vector reductions_; + std::vector scalars_; + + private: + RequirementAnalyzer* req_analyzer_; + BufferBuilder* buffer_; +}; + +} // namespace legate diff --git a/src/runtime/operation.cc b/src/runtime/operation.cc new file mode 100644 index 0000000000..62ae52791b --- /dev/null +++ b/src/runtime/operation.cc @@ -0,0 +1,60 @@ +/* Copyright 2021 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "runtime/operation.h" +#include "data/logical_store.h" +#include "data/scalar.h" +#include "runtime/context.h" +#include "runtime/launcher.h" +#include "runtime/runtime.h" + +namespace legate { + +Operation::Operation(Runtime* runtime, LibraryContext* library, int64_t mapper_id) + : runtime_(runtime), library_(library), mapper_id_(mapper_id) +{ +} + +void Operation::add_input(LogicalStoreP store) { inputs_.push_back(store); } + +void Operation::add_output(LogicalStoreP store) { outputs_.push_back(store); } + +void Operation::add_reduction(LogicalStoreP store, Legion::ReductionOpID redop) +{ + reductions_.push_back(Reduction(store, redop)); +} + +Task::Task(Runtime* runtime, LibraryContext* library, int64_t task_id, int64_t mapper_id /*=0*/) + : Operation(runtime, library, mapper_id), task_id_(task_id) +{ +} + +void Task::add_scalar_arg(const Scalar& scalar) { scalars_.push_back(scalar); } + +void Task::launch() const +{ + TaskLauncher launcher(runtime_, library_, task_id_, mapper_id_); + + for (auto& input : inputs_) launcher.add_input(input, std::make_unique()); + for (auto& output : outputs_) launcher.add_output(output, std::make_unique()); + for (auto& pair : reductions_) + launcher.add_reduction(pair.first, std::make_unique(pair.second)); + for (auto& scalar : scalars_) launcher.add_scalar(scalar); + + launcher.execute_single(); +} + +} // namespace legate diff --git a/src/runtime/operation.h b/src/runtime/operation.h new file mode 100644 index 0000000000..a7b66efb09 --- /dev/null +++ b/src/runtime/operation.h @@ -0,0 +1,72 @@ +/* Copyright 2021 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include + +#include "legion.h" + +namespace legate { + +class LogicalStore; +class Scalar; +class Runtime; +class LibraryContext; + +class Operation { + private: + using LogicalStoreP = std::shared_ptr; + using Reduction = std::pair; + + public: + Operation(Runtime* runtime, LibraryContext* library, int64_t mapper_id); + + public: + void add_input(LogicalStoreP store); + void add_output(LogicalStoreP store); + void add_reduction(LogicalStoreP store, Legion::ReductionOpID redop); + + public: + virtual void launch() const = 0; + + protected: + Runtime* runtime_; + LibraryContext* library_; + int64_t mapper_id_; + + protected: + std::vector inputs_; + std::vector outputs_; + std::vector reductions_; +}; + +class Task : public Operation { + public: + Task(Runtime* runtime, LibraryContext* library, int64_t task_id, int64_t mapper_id = 0); + + public: + void add_scalar_arg(const Scalar& scalar); + + public: + virtual void launch() const override; + + private: + int64_t task_id_; + std::vector scalars_{}; +}; + +} // namespace legate diff --git a/src/runtime/runtime.cc b/src/runtime/runtime.cc index 1c32fe12fe..f4bb5146e7 100644 --- a/src/runtime/runtime.cc +++ b/src/runtime/runtime.cc @@ -18,6 +18,7 @@ #include "legate.h" #include "mapping/core_mapper.h" #include "runtime/context.h" +#include "runtime/operation.h" #include "runtime/projection.h" #include "runtime/shard.h" #include "utilities/deserializer.h" @@ -94,7 +95,7 @@ static CUDALibraries& get_cuda_libraries(Processor proc, bool check) } #endif -static void toplevel_task(const Task* task, +static void toplevel_task(const Legion::Task* task, const std::vector& regions, Context ctx, Legion::Runtime* legion_runtime) @@ -107,7 +108,7 @@ static void toplevel_task(const Task* task, main(args.argc, args.argv, runtime); } -static void initialize_cpu_resource_task(const Task* task, +static void initialize_cpu_resource_task(const Legion::Task* task, const std::vector& regions, Context ctx, Legion::Runtime* runtime) @@ -115,7 +116,7 @@ static void initialize_cpu_resource_task(const Task* task, // Nothing to do here yet... } -static void finalize_cpu_resource_task(const Task* task, +static void finalize_cpu_resource_task(const Legion::Task* task, const std::vector& regions, Context ctx, Legion::Runtime* runtime) @@ -123,7 +124,7 @@ static void finalize_cpu_resource_task(const Task* task, // Nothing to do here yet... } -static ReturnValues extract_scalar_task(const Task* task, +static ReturnValues extract_scalar_task(const Legion::Task* task, const std::vector& regions, Context legion_context, Legion::Runtime* runtime) @@ -135,7 +136,7 @@ static ReturnValues extract_scalar_task(const Task* task, } #ifdef LEGATE_USE_CUDA -static void initialize_gpu_resource_task(const Task* task, +static void initialize_gpu_resource_task(const Legion::Task* task, const std::vector& regions, Context ctx, Legion::Runtime* runtime) @@ -155,7 +156,7 @@ static void initialize_gpu_resource_task(const Task* task, } } -static void finalize_gpu_resource_task(const Task* task, +static void finalize_gpu_resource_task(const Legion::Task* task, const std::vector& regions, Context ctx, Legion::Runtime* runtime) @@ -204,7 +205,12 @@ void register_legate_core_tasks(Machine machine, // Register the task variant for both CPUs and GPUs { - auto registrar = make_registrar(toplevel_task_id, toplevel_task_name, Processor::LOC_PROC); + TaskVariantRegistrar registrar(toplevel_task_id, toplevel_task_name); + registrar.add_constraint(ProcessorConstraint(Processor::LOC_PROC)); + registrar.set_leaf(false); + registrar.set_inner(true); + registrar.set_replicable(true); + registrar.global_registration = false; runtime->register_task_variant(registrar, LEGATE_CPU_VARIANT); } { @@ -278,6 +284,8 @@ void register_legate_core_tasks(Machine machine, auto core_lib = runtime->find_library(core_library_name); legion_runtime->set_top_level_task_id(core_lib->get_task_id(LEGATE_CORE_TOPLEVEL_TASK_ID)); legion_runtime->set_top_level_task_mapper_id(core_lib->get_mapper_id(0)); + + Core::parse_config(); } //////////////////////////////////////////////////// @@ -391,6 +399,20 @@ void Runtime::set_legion_context(Legion::Context legion_context) legion_context_ = legion_context; } +// This function should be moved to the library context +std::unique_ptr Runtime::create_task(LibraryContext* library, + int64_t task_id, + int64_t mapper_id /*=0*/) +{ + return std::make_unique(this, library, task_id, mapper_id); +} + +void Runtime::submit(std::unique_ptr op) +{ + // TODO: We need to build a lazy evaluation pipeline here + op->launch(); +} + std::shared_ptr Runtime::create_store(std::vector extents, LegateTypeCode code) { @@ -469,6 +491,12 @@ Domain Runtime::get_index_space_domain(const IndexSpace& index_space) const return legion_runtime_->get_index_space_domain(legion_context_, index_space); } +std::shared_ptr Runtime::dispatch(TaskLauncher* launcher) +{ + legion_runtime_->execute_task(legion_context_, *launcher); + return nullptr; +} + /*static*/ void Runtime::initialize(int32_t argc, char** argv) { Legion::Runtime::initialize(&argc, &argv, true /*filter legion and realm args*/); diff --git a/src/runtime/runtime.h b/src/runtime/runtime.h index 85dba1cc50..36ca21b098 100644 --- a/src/runtime/runtime.h +++ b/src/runtime/runtime.h @@ -44,6 +44,8 @@ class Core { class ResourceConfig; class Runtime; +class Operation; +class Task; class LibraryContext; class LogicalRegionField; class LogicalStore; @@ -105,6 +107,12 @@ class Runtime { void set_legion_runtime(Legion::Runtime* legion_runtime); void set_legion_context(Legion::Context legion_context); + public: + std::unique_ptr create_task(LibraryContext* library, + int64_t task_id, + int64_t mapper_id = 0); + void submit(std::unique_ptr op); + public: std::shared_ptr create_store(std::vector extents, LegateTypeCode code); std::shared_ptr create_region_field(const std::vector& extents, @@ -122,6 +130,9 @@ class Runtime { Legion::FieldID allocate_field(const Legion::FieldSpace& field_space, size_t field_size); Legion::Domain get_index_space_domain(const Legion::IndexSpace& index_space) const; + public: + std::shared_ptr dispatch(Legion::TaskLauncher* launcher); + public: static void initialize(int32_t argc, char** argv); static int32_t start(int32_t argc, char** argv); From b156b38e61ca664b8ef1b2d6d5b10f3c5a9252e0 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 29 Sep 2021 14:19:18 -0700 Subject: [PATCH 0004/1425] Post merge clean-up --- src/core.mk | 10 +++++----- src/{ => core}/data/logical_store.cc | 4 ++-- src/{ => core}/data/logical_store.h | 4 ++-- src/{ => core}/runtime/launcher.cc | 10 +++++----- src/{ => core}/runtime/launcher.h | 0 src/{ => core}/runtime/operation.cc | 12 ++++++------ src/{ => core}/runtime/operation.h | 0 src/core/runtime/runtime.cc | 2 +- src/legate.h | 2 +- 9 files changed, 22 insertions(+), 22 deletions(-) rename src/{ => core}/data/logical_store.cc (96%) rename src/{ => core}/data/logical_store.h (97%) rename src/{ => core}/runtime/launcher.cc (98%) rename src/{ => core}/runtime/launcher.h (100%) rename src/{ => core}/runtime/operation.cc (89%) rename src/{ => core}/runtime/operation.h (100%) diff --git a/src/core.mk b/src/core.mk index da451dbafe..d95e47e49c 100644 --- a/src/core.mk +++ b/src/core.mk @@ -16,18 +16,18 @@ # General source files GEN_CPU_SRC = core/legate_c.cc \ + core/data/logical_store.cc \ core/data/scalar.cc \ core/data/store.cc \ core/data/transform.cc \ - data/logical_store.cc \ core/mapping/base_mapper.cc \ core/mapping/core_mapper.cc \ core/mapping/instance_manager.cc \ core/mapping/mapping.cc \ core/mapping/task.cc \ core/runtime/context.cc \ - runtime/launcher.cc \ - runtime/operation.cc \ + core/runtime/launcher.cc \ + core/runtime/operation.cc \ core/runtime/projection.cc \ core/runtime/runtime.cc \ core/runtime/shard.cc \ @@ -53,7 +53,7 @@ INSTALL_HEADERS = legate.h \ legate_preamble.h \ core/legate_c.h \ core/data/buffer.h \ - data/logical_store.h \ + core/data/logical_store.h \ core/data/scalar.h \ core/data/scalar.inl \ core/data/store.h \ @@ -64,7 +64,7 @@ INSTALL_HEADERS = legate.h \ core/mapping/task.h \ core/mapping/task.inl \ core/runtime/context.h \ - runtime/operation.h \ + core/runtime/operation.h \ core/runtime/runtime.h \ core/task/return.h \ core/task/task.h \ diff --git a/src/data/logical_store.cc b/src/core/data/logical_store.cc similarity index 96% rename from src/data/logical_store.cc rename to src/core/data/logical_store.cc index 188edef009..1227cfe013 100644 --- a/src/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -14,9 +14,9 @@ * */ -#include "data/logical_store.h" +#include "core/data/logical_store.h" -#include "runtime/runtime.h" +#include "core/runtime/runtime.h" using namespace Legion; diff --git a/src/data/logical_store.h b/src/core/data/logical_store.h similarity index 97% rename from src/data/logical_store.h rename to src/core/data/logical_store.h index 1595f8103a..d6b9404c8a 100644 --- a/src/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -20,8 +20,8 @@ #include "legion.h" -#include "data/transform.h" -#include "utilities/typedefs.h" +#include "core/data/transform.h" +#include "core/utilities/typedefs.h" namespace legate { diff --git a/src/runtime/launcher.cc b/src/core/runtime/launcher.cc similarity index 98% rename from src/runtime/launcher.cc rename to src/core/runtime/launcher.cc index eb5ecd5b02..55bf9487d0 100644 --- a/src/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -14,11 +14,11 @@ * */ -#include "runtime/launcher.h" -#include "data/logical_store.h" -#include "data/scalar.h" -#include "runtime/context.h" -#include "runtime/runtime.h" +#include "core/runtime/launcher.h" +#include "core/data/logical_store.h" +#include "core/data/scalar.h" +#include "core/runtime/context.h" +#include "core/runtime/runtime.h" namespace legate { diff --git a/src/runtime/launcher.h b/src/core/runtime/launcher.h similarity index 100% rename from src/runtime/launcher.h rename to src/core/runtime/launcher.h diff --git a/src/runtime/operation.cc b/src/core/runtime/operation.cc similarity index 89% rename from src/runtime/operation.cc rename to src/core/runtime/operation.cc index 62ae52791b..49d8fbdd42 100644 --- a/src/runtime/operation.cc +++ b/src/core/runtime/operation.cc @@ -14,12 +14,12 @@ * */ -#include "runtime/operation.h" -#include "data/logical_store.h" -#include "data/scalar.h" -#include "runtime/context.h" -#include "runtime/launcher.h" -#include "runtime/runtime.h" +#include "core/runtime/operation.h" +#include "core/data/logical_store.h" +#include "core/data/scalar.h" +#include "core/runtime/context.h" +#include "core/runtime/launcher.h" +#include "core/runtime/runtime.h" namespace legate { diff --git a/src/runtime/operation.h b/src/core/runtime/operation.h similarity index 100% rename from src/runtime/operation.h rename to src/core/runtime/operation.h diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 9543f48727..1f1760aeac 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -14,12 +14,12 @@ * */ +#include "core/data/logical_store.h" #include "core/mapping/core_mapper.h" #include "core/runtime/context.h" #include "core/runtime/projection.h" #include "core/runtime/shard.h" #include "core/utilities/deserializer.h" -#include "data/logical_store.h" #include "legate.h" #ifdef LEGATE_USE_CUDA #include "core/gpu/cudalibs.h" diff --git a/src/legate.h b/src/legate.h index 363130456c..0e1d44d20d 100644 --- a/src/legate.h +++ b/src/legate.h @@ -21,6 +21,7 @@ #include "core/data/scalar.h" #include "core/data/store.h" #include "core/legate_c.h" +#include "core/runtime/operation.h" #include "core/runtime/runtime.h" #include "core/task/task.h" #include "core/utilities/deserializer.h" @@ -28,4 +29,3 @@ #include "core/utilities/type_traits.h" #include "core/utilities/typedefs.h" #include "legate_defines.h" -#include "runtime/operation.h" From e6027d0e5fcb8ccd07a89e6162980a9180a7cf55 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 29 Sep 2021 15:33:46 -0700 Subject: [PATCH 0005/1425] Make sure that the runtime is created for both C++ and Python interfaces --- src/core/runtime/runtime.cc | 25 +++++++++++++++---------- src/core/runtime/runtime.h | 6 +++--- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 1f1760aeac..4a04778483 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -253,6 +253,8 @@ void register_legate_core_tasks(Machine machine, Legion::Runtime* legion_runtime, const std::set& local_procs) { + Runtime::create_runtime(legion_runtime); + ResourceConfig config; config.max_tasks = LEGATE_CORE_NUM_TASK_IDS; config.max_projections = LEGATE_CORE_MAX_FUNCTOR_ID; @@ -278,7 +280,6 @@ void register_legate_core_tasks(Machine machine, core_library_registration_callback(machine, legion_runtime, local_procs); auto runtime = Runtime::get_runtime(); - runtime->set_legion_runtime(legion_runtime); auto core_lib = runtime->find_library(core_library_name); legion_runtime->set_top_level_task_id(core_lib->get_task_id(LEGATE_CORE_TOPLEVEL_TASK_ID)); @@ -349,7 +350,7 @@ std::shared_ptr FieldManager::allocate_field() /*static*/ Runtime* Runtime::runtime_; -Runtime::Runtime() {} +Runtime::Runtime(Legion::Runtime* legion_runtime) : legion_runtime_(legion_runtime) {} Runtime::~Runtime() { @@ -388,11 +389,6 @@ LibraryContext* Runtime::create_library(const std::string& library_name, return context; } -void Runtime::set_legion_runtime(Legion::Runtime* legion_runtime) -{ - legion_runtime_ = legion_runtime; -} - void Runtime::set_legion_context(Legion::Context legion_context) { legion_context_ = legion_context; @@ -459,6 +455,7 @@ FieldManager* Runtime::find_or_create_field_manager(const Domain& shape, LegateT IndexSpace Runtime::find_or_create_index_space(const Domain& shape) { + assert(nullptr != legion_context_); auto finder = index_spaces_.find(shape); if (finder != index_spaces_.end()) return finder->second; @@ -471,27 +468,32 @@ IndexSpace Runtime::find_or_create_index_space(const Domain& shape) FieldSpace Runtime::create_field_space() { + assert(nullptr != legion_context_); return legion_runtime_->create_field_space(legion_context_); } LogicalRegion Runtime::create_region(const IndexSpace& index_space, const FieldSpace& field_space) { + assert(nullptr != legion_context_); return legion_runtime_->create_logical_region(legion_context_, index_space, field_space); } FieldID Runtime::allocate_field(const FieldSpace& field_space, size_t field_size) { + assert(nullptr != legion_context_); auto allocator = legion_runtime_->create_field_allocator(legion_context_, field_space); return allocator.allocate_field(field_size); } Domain Runtime::get_index_space_domain(const IndexSpace& index_space) const { + assert(nullptr != legion_context_); return legion_runtime_->get_index_space_domain(legion_context_, index_space); } std::shared_ptr Runtime::dispatch(TaskLauncher* launcher) { + assert(nullptr != legion_context_); legion_runtime_->execute_task(legion_context_, *launcher); return nullptr; } @@ -499,9 +501,7 @@ std::shared_ptr Runtime::dispatch(TaskLauncher* launcher) /*static*/ void Runtime::initialize(int32_t argc, char** argv) { Legion::Runtime::initialize(&argc, &argv, true /*filter legion and realm args*/); - Legion::Runtime::perform_registration_callback(legate::core_library_bootstrapping_callback, - true /*global*/); - runtime_ = new Runtime(); + Legion::Runtime::add_registration_callback(legate::core_library_bootstrapping_callback); } /*static*/ int32_t Runtime::start(int32_t argc, char** argv) @@ -511,6 +511,11 @@ std::shared_ptr Runtime::dispatch(TaskLauncher* launcher) /*static*/ Runtime* Runtime::get_runtime() { return Runtime::runtime_; } +/*static*/ void Runtime::create_runtime(Legion::Runtime* legion_runtime) +{ + runtime_ = new Runtime(legion_runtime); +} + void initialize(int32_t argc, char** argv) { Runtime::initialize(argc, argv); } void set_main_function(Runtime::MainFnPtr main_fn) diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 3770faa227..47af56530c 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -87,7 +87,7 @@ class Runtime { using MainFnPtr = void (*)(int32_t, char**, Runtime*); public: - Runtime(); + Runtime(Legion::Runtime* legion_runtime); ~Runtime(); public: @@ -104,7 +104,6 @@ class Runtime { LibraryContext* create_library(const std::string& library_name, const ResourceConfig& config); public: - void set_legion_runtime(Legion::Runtime* legion_runtime); void set_legion_context(Legion::Context legion_context); public: @@ -137,12 +136,13 @@ class Runtime { static void initialize(int32_t argc, char** argv); static int32_t start(int32_t argc, char** argv); static Runtime* get_runtime(); + static void create_runtime(Legion::Runtime* legion_runtime); private: static Runtime* runtime_; private: - Legion::Runtime* legion_runtime_{nullptr}; + Legion::Runtime* legion_runtime_; Legion::Context legion_context_{nullptr}; private: From 3eee26de13f07c5163a10597b8c45d88736e0608 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 29 Sep 2021 15:44:55 -0700 Subject: [PATCH 0006/1425] Move main function from the runtime to the corelib to be able to set it before the runtime initializes --- src/core/runtime/runtime.cc | 20 ++++++++++---------- src/core/runtime/runtime.h | 13 ++----------- src/core/utilities/typedefs.h | 1 + 3 files changed, 13 insertions(+), 21 deletions(-) diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 4a04778483..88705fd7d0 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -37,6 +37,8 @@ static const char* const core_library_name = "legate.core"; /*static*/ bool Core::show_progress = false; +/*static*/ LegateMainFnPtr Core::main_fn = nullptr; + /*static*/ void Core::parse_config(void) { #ifndef LEGATE_USE_CUDA @@ -102,9 +104,14 @@ static void toplevel_task(const Legion::Task* task, auto runtime = Runtime::get_runtime(); runtime->set_legion_context(ctx); - auto main = runtime->get_main_function(); + if (nullptr == Core::main_fn) { + log_legate.error( + "No main function was provided. Please register one with 'legate::set_main_function'."); + LEGATE_ABORT + } + auto args = Legion::Runtime::get_input_args(); - main(args.argc, args.argv, runtime); + Core::main_fn(args.argc, args.argv); } static void initialize_cpu_resource_task(const Legion::Task* task, @@ -357,10 +364,6 @@ Runtime::~Runtime() for (auto& pair : libraries_) delete pair.second; } -void Runtime::set_main_function(MainFnPtr main_fn) { main_fn_ = main_fn; } - -Runtime::MainFnPtr Runtime::get_main_function() const { return main_fn_; } - LibraryContext* Runtime::find_library(const std::string& library_name, bool can_fail /*=false*/) const { @@ -518,10 +521,7 @@ std::shared_ptr Runtime::dispatch(TaskLauncher* launcher) void initialize(int32_t argc, char** argv) { Runtime::initialize(argc, argv); } -void set_main_function(Runtime::MainFnPtr main_fn) -{ - Runtime::get_runtime()->set_main_function(main_fn); -} +void set_main_function(LegateMainFnPtr main_fn) { Core::main_fn = main_fn; } int32_t start(int32_t argc, char** argv) { return Runtime::start(argc, argv); } diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 47af56530c..dded2cddc8 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -36,6 +36,7 @@ class Core { public: // Configuration settings static bool show_progress; + static LegateMainFnPtr main_fn; #ifdef LEGATE_USE_CUDA public: static cublasContext* get_cublas(void); @@ -84,21 +85,14 @@ class FieldManager { class Runtime { public: - using MainFnPtr = void (*)(int32_t, char**, Runtime*); - public: Runtime(Legion::Runtime* legion_runtime); ~Runtime(); public: friend void initialize(int32_t argc, char** argv); - friend void set_main_function(MainFnPtr p_main); friend int32_t start(int32_t argc, char** argv); - public: - void set_main_function(MainFnPtr main_fn); - MainFnPtr get_main_function() const; - public: LibraryContext* find_library(const std::string& library_name, bool can_fail = false) const; LibraryContext* create_library(const std::string& library_name, const ResourceConfig& config); @@ -154,14 +148,11 @@ class Runtime { private: std::map libraries_; - - private: - MainFnPtr main_fn_{nullptr}; }; void initialize(int32_t argc, char** argv); -void set_main_function(Runtime::MainFnPtr p_main); +void set_main_function(LegateMainFnPtr p_main); int32_t start(int32_t argc, char** argv); diff --git a/src/core/utilities/typedefs.h b/src/core/utilities/typedefs.h index e7b5811aa3..31696f4349 100644 --- a/src/core/utilities/typedefs.h +++ b/src/core/utilities/typedefs.h @@ -51,5 +51,6 @@ using LegateVariantCode = legate_core_variant_t; using LegateTypeCode = legate_core_type_code_t; using LegateResource = legate_core_resource_t; using LegateMappingTag = legate_core_mapping_tag_t; +using LegateMainFnPtr = void (*)(int32_t, char**); } // namespace legate From 99a2a29dedfae5dbf6f4e0b0bc56e732226ccb5f Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 29 Sep 2021 21:10:13 -0700 Subject: [PATCH 0007/1425] Toplevel task cannot be inner, as the client may want to inline map stores --- src/core/runtime/runtime.cc | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 88705fd7d0..56f71a1256 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -214,7 +214,7 @@ void register_legate_core_tasks(Machine machine, TaskVariantRegistrar registrar(toplevel_task_id, toplevel_task_name); registrar.add_constraint(ProcessorConstraint(Processor::LOC_PROC)); registrar.set_leaf(false); - registrar.set_inner(true); + registrar.set_inner(false); registrar.set_replicable(true); registrar.global_registration = false; runtime->register_task_variant(registrar, LEGATE_CPU_VARIANT); @@ -431,6 +431,22 @@ std::shared_ptr Runtime::create_region_field( return fld_mgr->allocate_field(); } +RegionField Runtime::map_region_field(LibraryContext* context, + std::shared_ptr rf) +{ + auto region = rf->region(); + auto field_id = rf->field_id(); + + RegionRequirement req(region, READ_WRITE, EXCLUSIVE, region); + req.add_field(field_id); + + auto mapper_id = context->get_mapper_id(0); + // TODO: We need to pass the metadata about logical store + InlineLauncher launcher(req, mapper_id); + auto pr = legion_runtime_->map_region(legion_context_, launcher); + return RegionField(rf->dim(), pr, field_id); +} + RegionManager* Runtime::find_or_create_region_manager(const Domain& shape) { auto finder = region_managers_.find(shape); From fa7339f5fcb7185c6482ae44de5d78b62f4617d6 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 29 Sep 2021 21:10:55 -0700 Subject: [PATCH 0008/1425] Interface to inline map logical stores --- src/core/data/logical_store.cc | 10 +++++++++- src/core/data/logical_store.h | 8 ++++++-- src/core/runtime/runtime.h | 3 +++ src/legate.h | 1 + 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index 1227cfe013..d1610869b2 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -15,7 +15,7 @@ */ #include "core/data/logical_store.h" - +#include "core/data/store.h" #include "core/runtime/runtime.h" using namespace Legion; @@ -61,4 +61,12 @@ void LogicalStore::create_storage() region_field_ = runtime_->create_region_field(extents_, code_); } +std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) +{ + if (nullptr != mapped_) return mapped_; + auto rf = runtime_->map_region_field(context, region_field_); + mapped_ = std::make_shared(dim(), code_, -1, std::move(rf), transform_); + return mapped_; +} + } // namespace legate diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index d6b9404c8a..68fbe77766 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -26,6 +26,8 @@ namespace legate { class Runtime; +class Store; +class LibraryContext; class LogicalRegionField { public: @@ -74,8 +76,6 @@ class LogicalStore { public: int32_t dim() const; LegateTypeCode code() const { return code_; } - - public: Legion::Domain domain() const; public: @@ -85,12 +85,16 @@ class LogicalStore { private: void create_storage(); + public: + std::shared_ptr get_physical_store(LibraryContext* context); + private: Runtime* runtime_{nullptr}; LegateTypeCode code_{MAX_TYPE_NUMBER}; std::vector extents_; std::shared_ptr region_field_{nullptr}; std::shared_ptr transform_{nullptr}; + std::shared_ptr mapped_{nullptr}; }; } // namespace legate diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index dded2cddc8..04d3cf4a01 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -20,6 +20,7 @@ #include "legion.h" +#include "core/data/store.h" #include "core/utilities/typedefs.h" namespace legate { @@ -110,6 +111,8 @@ class Runtime { std::shared_ptr create_store(std::vector extents, LegateTypeCode code); std::shared_ptr create_region_field(const std::vector& extents, LegateTypeCode code); + RegionField map_region_field(LibraryContext* context, + std::shared_ptr region_field); public: RegionManager* find_or_create_region_manager(const Legion::Domain& shape); diff --git a/src/legate.h b/src/legate.h index 0e1d44d20d..d1a2b5e62a 100644 --- a/src/legate.h +++ b/src/legate.h @@ -18,6 +18,7 @@ #include "legion.h" // legion.h has to go before these +#include "core/data/logical_store.h" #include "core/data/scalar.h" #include "core/data/store.h" #include "core/legate_c.h" From 4cd73cf2e0078f3480b9903428514a63c265ea82 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 1 Oct 2021 14:14:44 -0700 Subject: [PATCH 0009/1425] Make the code path to run auto-partitioner on operations and launch them with the computed strategy. Everything still runs sequentially now. --- src/core.mk | 2 + src/core/data/logical_store.cc | 14 +++ src/core/data/logical_store.h | 6 + src/core/partitioning/partition.cc | 169 +++++++++++++++++++++++++++ src/core/partitioning/partition.h | 55 +++++++++ src/core/partitioning/partitioner.cc | 78 +++++++++++++ src/core/partitioning/partitioner.h | 66 +++++++++++ src/core/runtime/launcher.cc | 112 ++++++++++++++++-- src/core/runtime/launcher.h | 52 +++++++-- src/core/runtime/operation.cc | 52 +++++++-- src/core/runtime/operation.h | 8 +- src/core/runtime/runtime.cc | 44 ++++++- src/core/runtime/runtime.h | 15 +++ 13 files changed, 641 insertions(+), 32 deletions(-) create mode 100644 src/core/partitioning/partition.cc create mode 100644 src/core/partitioning/partition.h create mode 100644 src/core/partitioning/partitioner.cc create mode 100644 src/core/partitioning/partitioner.h diff --git a/src/core.mk b/src/core.mk index d95e47e49c..9d79c0f91c 100644 --- a/src/core.mk +++ b/src/core.mk @@ -25,6 +25,8 @@ GEN_CPU_SRC = core/legate_c.cc \ core/mapping/instance_manager.cc \ core/mapping/mapping.cc \ core/mapping/task.cc \ + core/partitioning/partition.cc \ + core/partitioning/partitioner.cc \ core/runtime/context.cc \ core/runtime/launcher.cc \ core/runtime/operation.cc \ diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index d1610869b2..3ffd1deeb7 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -16,6 +16,8 @@ #include "core/data/logical_store.h" #include "core/data/store.h" +#include "core/partitioning/partition.h" +#include "core/runtime/launcher.h" #include "core/runtime/runtime.h" using namespace Legion; @@ -56,6 +58,11 @@ std::shared_ptr LogicalStore::get_storage() return region_field_; } +std::shared_ptr LogicalStore::get_storage_unsafe() const +{ + return region_field_; +} + void LogicalStore::create_storage() { region_field_ = runtime_->create_region_field(extents_, code_); @@ -69,4 +76,11 @@ std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) return mapped_; } +std::unique_ptr LogicalStore::find_or_create_partition(const Partition* partition) +{ + auto lp = + partition->construct(this, partition->is_disjoint_for(this), partition->is_complete_for(this)); + return std::make_unique(lp, 0); +} + } // namespace legate diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index 68fbe77766..13adf9e86d 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -28,6 +28,8 @@ namespace legate { class Runtime; class Store; class LibraryContext; +class Partition; +class Projection; class LogicalRegionField { public: @@ -81,6 +83,7 @@ class LogicalStore { public: bool has_storage() const { return nullptr != region_field_; } std::shared_ptr get_storage(); + std::shared_ptr get_storage_unsafe() const; private: void create_storage(); @@ -88,6 +91,9 @@ class LogicalStore { public: std::shared_ptr get_physical_store(LibraryContext* context); + public: + std::unique_ptr find_or_create_partition(const Partition* partition); + private: Runtime* runtime_{nullptr}; LegateTypeCode code_{MAX_TYPE_NUMBER}; diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc new file mode 100644 index 0000000000..ad8fab7927 --- /dev/null +++ b/src/core/partitioning/partition.cc @@ -0,0 +1,169 @@ +/* Copyright 2021 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/partitioning/partition.h" +#include "core/data/logical_store.h" +#include "core/runtime/launcher.h" +#include "core/runtime/runtime.h" + +namespace legate { + +using Shape = std::vector; + +class NoPartition : public Partition { + public: + NoPartition(Runtime* runtime); + + public: + virtual bool is_complete_for(const LogicalStore* store) const override; + virtual bool is_disjoint_for(const LogicalStore* store) const override; + + public: + virtual Legion::LogicalPartition construct(const LogicalStore* store, + bool disjoint, + bool complete) const override; + virtual std::unique_ptr get_projection(LogicalStore* store) const override; + + private: + Runtime* runtime_; +}; + +class Tiling : public Partition { + public: + Tiling(Runtime* runtime, Shape&& tile_shape, Shape&& color_shape, Shape&& offsets); + + public: + virtual bool is_complete_for(const LogicalStore* store) const override; + virtual bool is_disjoint_for(const LogicalStore* store) const override; + + public: + virtual Legion::LogicalPartition construct(const LogicalStore* store, + bool disjoint, + bool complete) const override; + virtual std::unique_ptr get_projection(LogicalStore* store) const override; + + private: + Runtime* runtime_; + Shape tile_shape_; + Shape color_shape_; + Shape offsets_; +}; + +struct PartitionByRestriction : public PartitioningFunctor { + public: + PartitionByRestriction(Legion::DomainTransform transform, Legion::Domain extent); + + public: + virtual Legion::IndexPartition construct(Legion::Runtime* legion_runtime, + Legion::Context legion_context, + const Legion::IndexSpace& parent, + const Legion::IndexSpace& color_space, + Legion::PartitionKind kind) const override; + + private: + Legion::DomainTransform transform_; + Legion::Domain extent_; +}; + +Tiling::Tiling(Runtime* runtime, Shape&& tile_shape, Shape&& color_shape, Shape&& offsets) + : runtime_(runtime), + tile_shape_(std::forward(tile_shape)), + color_shape_(std::forward(color_shape)), + offsets_(std::forward(offsets)) +{ +} + +NoPartition::NoPartition(Runtime* runtime) : runtime_(runtime) {} + +bool NoPartition::is_complete_for(const LogicalStore* store) const { return false; } + +bool NoPartition::is_disjoint_for(const LogicalStore* store) const { return false; } + +Legion::LogicalPartition NoPartition::construct(const LogicalStore* store, + bool disjoint, + bool complete) const +{ + return Legion::LogicalPartition::NO_PART; +} + +std::unique_ptr NoPartition::get_projection(LogicalStore* store) const +{ + return std::make_unique(); +} + +bool Tiling::is_complete_for(const LogicalStore* store) const {} + +bool Tiling::is_disjoint_for(const LogicalStore* store) const { return true; } + +Legion::LogicalPartition Tiling::construct(const LogicalStore* store, + bool disjoint, + bool complete) const +{ + Legion::DomainTransform transform; + Legion::Domain extent; + + Legion::Domain color_domain; + + auto region = store->get_storage_unsafe()->region(); + auto color_space = runtime_->find_or_create_index_space(color_domain); + auto index_space = region.get_index_space(); + + auto kind = complete ? (disjoint ? LEGION_DISJOINT_COMPLETE_KIND : LEGION_ALIASED_COMPLETE_KIND) + : (disjoint ? LEGION_DISJOINT_KIND : LEGION_ALIASED_KIND); + + PartitionByRestriction functor(transform, extent); + auto index_partition = runtime_->create_index_partition(index_space, color_space, kind, &functor); + return runtime_->create_logical_partition(region, index_partition); +} + +std::unique_ptr Tiling::get_projection(LogicalStore* store) const +{ + return store->find_or_create_partition(this); +} + +PartitionByRestriction::PartitionByRestriction(Legion::DomainTransform transform, + Legion::Domain extent) + : transform_(transform), extent_(extent) +{ +} + +Legion::IndexPartition PartitionByRestriction::construct(Legion::Runtime* legion_runtime, + Legion::Context legion_context, + const Legion::IndexSpace& parent, + const Legion::IndexSpace& color_space, + Legion::PartitionKind kind) const +{ + return legion_runtime->create_partition_by_restriction( + legion_context, parent, color_space, transform_, extent_, kind); +} + +std::unique_ptr create_tiling(Runtime* runtime, + Shape&& tile_shape, + Shape&& color_shape, + Shape&& offsets /*= {}*/) +{ + return std::make_unique(runtime, + std::forward(tile_shape), + std::forward(color_shape), + std::forward(offsets)); +} + +std::unique_ptr create_no_partition(Runtime* runtime) +{ + return std::make_unique(runtime); +} + +} // namespace legate diff --git a/src/core/partitioning/partition.h b/src/core/partitioning/partition.h new file mode 100644 index 0000000000..fce30bd296 --- /dev/null +++ b/src/core/partitioning/partition.h @@ -0,0 +1,55 @@ +/* Copyright 2021 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include + +#include "legion.h" + +namespace legate { + +class LogicalStore; +class Projection; +class Runtime; + +struct Partition { + public: + virtual bool is_complete_for(const LogicalStore* store) const = 0; + virtual bool is_disjoint_for(const LogicalStore* store) const = 0; + virtual Legion::LogicalPartition construct(const LogicalStore* store, + bool disjoint = false, + bool complete = false) const = 0; + virtual std::unique_ptr get_projection(LogicalStore* store) const = 0; +}; + +struct PartitioningFunctor { + public: + virtual Legion::IndexPartition construct(Legion::Runtime* legion_runtime, + Legion::Context legion_context, + const Legion::IndexSpace& parent, + const Legion::IndexSpace& color_space, + Legion::PartitionKind kind) const = 0; +}; + +std::unique_ptr create_no_partition(Runtime* runtime); + +std::unique_ptr create_tiling(Runtime* runtime, + std::vector&& tile_shape, + std::vector&& color_shape, + std::vector&& offsets = {}); + +} // namespace legate diff --git a/src/core/partitioning/partitioner.cc b/src/core/partitioning/partitioner.cc new file mode 100644 index 0000000000..2e079106e3 --- /dev/null +++ b/src/core/partitioning/partitioner.cc @@ -0,0 +1,78 @@ +/* Copyright 2021 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/partitioning/partitioner.h" +#include "core/data/scalar.h" +#include "core/partitioning/partition.h" +#include "core/runtime/launcher.h" +#include "core/runtime/operation.h" +#include "core/runtime/runtime.h" + +namespace legate { + +Strategy::Strategy() : launch_domain_(nullptr) {} + +Strategy::Strategy(const Legion::Domain& launch_domain) + : launch_domain_(std::make_unique(launch_domain)) +{ +} + +bool Strategy::parallel() const { return nullptr != launch_domain_; } + +Legion::Domain Strategy::launch_domain() const { return *launch_domain_; } + +void Strategy::set_launch_domain(const Legion::Domain& launch_domain) +{ + launch_domain_ = std::make_unique(launch_domain); +} + +void Strategy::insert(const LogicalStore* store, std::shared_ptr partition) +{ + assert(assignments_.find(store) == assignments_.end()); + assignments_[store] = std::move(partition); +} + +std::shared_ptr Strategy::find(const LogicalStore* store) const +{ + auto finder = assignments_.find(store); + assert(finder != assignments_.end()); + return finder->second; +} + +std::unique_ptr Strategy::get_projection(LogicalStore* store) const +{ + auto partition = find(store); + return partition->get_projection(store); +} + +Partitioner::Partitioner(Runtime* runtime, std::vector&& operations) + : runtime_(runtime), operations_(std::forward>(operations)) +{ +} + +std::unique_ptr Partitioner::partition_stores() +{ + auto strategy = std::make_unique(); + + for (auto op : operations_) { + auto all_stores = op->all_stores(); + for (auto store : all_stores) strategy->insert(store, create_no_partition(runtime_)); + } + + return std::move(strategy); +} + +} // namespace legate diff --git a/src/core/partitioning/partitioner.h b/src/core/partitioning/partitioner.h new file mode 100644 index 0000000000..e9829d5562 --- /dev/null +++ b/src/core/partitioning/partitioner.h @@ -0,0 +1,66 @@ +/* Copyright 2021 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include +#include + +#include "legion.h" + +namespace legate { + +class Operation; +class Runtime; +class LogicalStore; +class Partition; +class Projection; + +class Strategy { + public: + Strategy(); + Strategy(const Legion::Domain& launch_domain); + + public: + bool parallel() const; + Legion::Domain launch_domain() const; + void set_launch_domain(const Legion::Domain& launch_domain); + + public: + void insert(const LogicalStore* store, std::shared_ptr partition); + std::shared_ptr find(const LogicalStore* store) const; + + public: + std::unique_ptr get_projection(LogicalStore* store) const; + + private: + std::unique_ptr launch_domain_; + std::unordered_map> assignments_; +}; + +class Partitioner { + public: + Partitioner(Runtime* runtime, std::vector&& operations); + + public: + std::unique_ptr partition_stores(); + + private: + Runtime* runtime_; + std::vector operations_; +}; + +} // namespace legate diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/launcher.cc index 55bf9487d0..f5fa874aef 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -39,9 +39,6 @@ class BufferBuilder { }; class RequirementAnalyzer { - private: - using SingleTask = Legion::TaskLauncher*; - public: ~RequirementAnalyzer(); @@ -51,7 +48,8 @@ class RequirementAnalyzer { public: void analyze_requirements(); - void populate_launcher(SingleTask task) const; + template + void populate_launcher(Task* task) const; private: std::map req_indices_; @@ -154,12 +152,13 @@ uint32_t RequirementAnalyzer::get_requirement_index(RegionReq* req, Legion::Fiel void RequirementAnalyzer::analyze_requirements() {} -void RequirementAnalyzer::populate_launcher(SingleTask task) const +template +void RequirementAnalyzer::populate_launcher(Task* task) const { for (auto& pair : requirements_) { auto& req = pair.first; auto& fields = pair.second; - req->proj->add(task, *req, fields); + req->proj->populate_launcher(task, *req, fields); } } @@ -203,12 +202,18 @@ Projection::Projection(Legion::ReductionOpID r) : redop(std::make_unique(r); +} + Broadcast::Broadcast() : Projection() {} Broadcast::Broadcast(Legion::ReductionOpID redop) : Projection(redop) {} -void Broadcast::add(SingleTask task, - const RegionReq& req, - const std::vector& fields) const + +void Broadcast::populate_launcher(Legion::TaskLauncher* task, + const RegionReq& req, + const std::vector& fields) const { if (req.priv == REDUCE) { Legion::RegionRequirement legion_req(req.region, *redop, EXCLUSIVE, req.region, req.tag); @@ -221,6 +226,65 @@ void Broadcast::add(SingleTask task, } } +void Broadcast::populate_launcher(Legion::IndexTaskLauncher* task, + const RegionReq& req, + const std::vector& fields) const +{ + if (req.priv == REDUCE) { + Legion::RegionRequirement legion_req(req.region, *redop, EXCLUSIVE, req.region, req.tag); + legion_req.add_fields(fields); + task->add_region_requirement(legion_req); + } else { + Legion::RegionRequirement legion_req(req.region, req.priv, EXCLUSIVE, req.region, req.tag); + legion_req.add_fields(fields); + task->add_region_requirement(legion_req); + } +} + +MapPartition::MapPartition(Legion::LogicalPartition partition, Legion::ProjectionID proj_id) + : Projection(), partition_(partition), proj_id_(proj_id) +{ +} + +MapPartition::MapPartition(Legion::LogicalPartition partition, + Legion::ProjectionID proj_id, + Legion::ReductionOpID redop) + : Projection(redop), partition_(partition), proj_id_(proj_id) +{ +} + +void MapPartition::populate_launcher(Legion::TaskLauncher* task, + const RegionReq& req, + const std::vector& fields) const +{ + if (req.priv == REDUCE) { + Legion::RegionRequirement legion_req(req.region, *redop, EXCLUSIVE, req.region, req.tag); + legion_req.add_fields(fields); + task->add_region_requirement(legion_req); + } else { + Legion::RegionRequirement legion_req(req.region, req.priv, EXCLUSIVE, req.region, req.tag); + legion_req.add_fields(fields); + task->add_region_requirement(legion_req); + } +} + +void MapPartition::populate_launcher(Legion::IndexTaskLauncher* task, + const RegionReq& req, + const std::vector& fields) const +{ + if (req.priv == REDUCE) { + Legion::RegionRequirement legion_req( + partition_, proj_id_, *redop, EXCLUSIVE, req.region, req.tag); + legion_req.add_fields(fields); + task->add_region_requirement(legion_req); + } else { + Legion::RegionRequirement legion_req( + partition_, proj_id_, req.priv, EXCLUSIVE, req.region, req.tag); + legion_req.add_fields(fields); + task->add_region_requirement(legion_req); + } +} + RegionReq::RegionReq(Legion::LogicalRegion _region, Legion::PrivilegeMode _priv, ProjectionP _proj, @@ -279,6 +343,13 @@ void TaskLauncher::add_reduction(LogicalStoreP store, add_store(reductions_, std::move(store), std::move(proj), REDUCE, tag); } +void TaskLauncher::execute(const Legion::Domain& launch_domain) +{ + auto legion_launcher = build_index_task(launch_domain); + runtime_->dispatch(legion_launcher); + delete legion_launcher; +} + void TaskLauncher::execute_single() { auto legion_launcher = build_single_task(); @@ -310,7 +381,7 @@ void TaskLauncher::pack_args(const std::vector& args) for (auto& arg : args) arg->pack(*buffer_); } -TaskLauncher::SingleTask TaskLauncher::build_single_task() +Legion::TaskLauncher* TaskLauncher::build_single_task() { pack_args(inputs_); pack_args(outputs_); @@ -328,4 +399,25 @@ TaskLauncher::SingleTask TaskLauncher::build_single_task() return single_task; } +Legion::IndexTaskLauncher* TaskLauncher::build_index_task(const Legion::Domain& launch_domain) +{ + pack_args(inputs_); + pack_args(outputs_); + pack_args(reductions_); + pack_args(scalars_); + + auto index_task = new Legion::IndexTaskLauncher(legion_task_id(), + launch_domain, + buffer_->to_legion_buffer(), + Legion::ArgumentMap(), + Legion::Predicate::TRUE_PRED, + false /*must*/, + legion_mapper_id(), + tag_); + + req_analyzer_->populate_launcher(index_task); + + return index_task; +} + } // namespace legate diff --git a/src/core/runtime/launcher.h b/src/core/runtime/launcher.h index 6dc3eb7f12..b8cbd55841 100644 --- a/src/core/runtime/launcher.h +++ b/src/core/runtime/launcher.h @@ -34,9 +34,6 @@ class Runtime; class LibraryContext; class Projection { - protected: - using SingleTask = Legion::TaskLauncher*; - protected: Projection() {} Projection(Legion::ReductionOpID redop); @@ -45,9 +42,15 @@ class Projection { virtual ~Projection() {} public: - virtual void add(SingleTask task, - const RegionReq& req, - const std::vector& fields) const = 0; + virtual void populate_launcher(Legion::TaskLauncher* task, + const RegionReq& req, + const std::vector& fields) const = 0; + virtual void populate_launcher(Legion::IndexTaskLauncher* task, + const RegionReq& req, + const std::vector& fields) const = 0; + + public: + void set_reduction_op(Legion::ReductionOpID redop); public: std::unique_ptr redop{nullptr}; @@ -62,9 +65,35 @@ class Broadcast : public Projection { virtual ~Broadcast() {} public: - virtual void add(SingleTask task, - const RegionReq& req, - const std::vector& fields) const override; + virtual void populate_launcher(Legion::TaskLauncher* task, + const RegionReq& req, + const std::vector& fields) const override; + virtual void populate_launcher(Legion::IndexTaskLauncher* task, + const RegionReq& req, + const std::vector& fields) const override; +}; + +class MapPartition : public Projection { + public: + MapPartition(Legion::LogicalPartition partition, Legion::ProjectionID proj_id); + MapPartition(Legion::LogicalPartition partition, + Legion::ProjectionID proj_id, + Legion::ReductionOpID redop); + + public: + virtual ~MapPartition() {} + + public: + virtual void populate_launcher(Legion::TaskLauncher* task, + const RegionReq& req, + const std::vector& fields) const override; + virtual void populate_launcher(Legion::IndexTaskLauncher* task, + const RegionReq& req, + const std::vector& fields) const override; + + private: + Legion::LogicalPartition partition_; + Legion::ProjectionID proj_id_; }; class RegionReq { @@ -88,7 +117,6 @@ class TaskLauncher { private: using LogicalStoreP = std::shared_ptr; using ProjectionP = std::unique_ptr; - using SingleTask = Legion::TaskLauncher*; public: TaskLauncher(Runtime* runtime, @@ -119,11 +147,13 @@ class TaskLauncher { uint64_t tag); public: + void execute(const Legion::Domain& launch_domain); void execute_single(); private: void pack_args(const std::vector& args); - SingleTask build_single_task(); + Legion::IndexTaskLauncher* build_index_task(const Legion::Domain& launch_domain); + Legion::TaskLauncher* build_single_task(); private: Runtime* runtime_; diff --git a/src/core/runtime/operation.cc b/src/core/runtime/operation.cc index 49d8fbdd42..388db1806d 100644 --- a/src/core/runtime/operation.cc +++ b/src/core/runtime/operation.cc @@ -14,11 +14,14 @@ * */ -#include "core/runtime/operation.h" +#include + #include "core/data/logical_store.h" #include "core/data/scalar.h" +#include "core/partitioning/partitioner.h" #include "core/runtime/context.h" #include "core/runtime/launcher.h" +#include "core/runtime/operation.h" #include "core/runtime/runtime.h" namespace legate { @@ -37,6 +40,35 @@ void Operation::add_reduction(LogicalStoreP store, Legion::ReductionOpID redop) reductions_.push_back(Reduction(store, redop)); } +std::vector Operation::all_stores() const +{ + std::vector result; + std::unordered_set added; + + auto add_all = [&](auto& stores) { + for (auto& store : stores) { + auto p_store = store.get(); + if (added.find(p_store) == added.end()) { + result.push_back(p_store); + added.insert(p_store); + } + } + }; + + add_all(inputs_); + add_all(outputs_); + for (auto& reduction : reductions_) { + auto& store = reduction.first; + auto p_store = store.get(); + if (added.find(p_store) == added.end()) { + result.push_back(p_store); + added.insert(p_store); + } + } + + return std::move(result); +} + Task::Task(Runtime* runtime, LibraryContext* library, int64_t task_id, int64_t mapper_id /*=0*/) : Operation(runtime, library, mapper_id), task_id_(task_id) { @@ -44,17 +76,23 @@ Task::Task(Runtime* runtime, LibraryContext* library, int64_t task_id, int64_t m void Task::add_scalar_arg(const Scalar& scalar) { scalars_.push_back(scalar); } -void Task::launch() const +void Task::launch(Strategy* strategy) const { TaskLauncher launcher(runtime_, library_, task_id_, mapper_id_); - for (auto& input : inputs_) launcher.add_input(input, std::make_unique()); - for (auto& output : outputs_) launcher.add_output(output, std::make_unique()); - for (auto& pair : reductions_) - launcher.add_reduction(pair.first, std::make_unique(pair.second)); + for (auto& input : inputs_) launcher.add_input(input, strategy->get_projection(input.get())); + for (auto& output : outputs_) launcher.add_output(output, strategy->get_projection(output.get())); + for (auto& pair : reductions_) { + auto projection = strategy->get_projection(pair.first.get()); + projection->set_reduction_op(pair.second); + launcher.add_reduction(pair.first, std::move(projection)); + } for (auto& scalar : scalars_) launcher.add_scalar(scalar); - launcher.execute_single(); + if (strategy->parallel()) + launcher.execute(strategy->launch_domain()); + else + launcher.execute_single(); } } // namespace legate diff --git a/src/core/runtime/operation.h b/src/core/runtime/operation.h index a7b66efb09..fdfbff2f29 100644 --- a/src/core/runtime/operation.h +++ b/src/core/runtime/operation.h @@ -26,6 +26,7 @@ class LogicalStore; class Scalar; class Runtime; class LibraryContext; +class Strategy; class Operation { private: @@ -41,7 +42,10 @@ class Operation { void add_reduction(LogicalStoreP store, Legion::ReductionOpID redop); public: - virtual void launch() const = 0; + std::vector all_stores() const; + + public: + virtual void launch(Strategy* strategy) const = 0; protected: Runtime* runtime_; @@ -62,7 +66,7 @@ class Task : public Operation { void add_scalar_arg(const Scalar& scalar); public: - virtual void launch() const override; + virtual void launch(Strategy* strategy) const override; private: int64_t task_id_; diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 56f71a1256..7de2a9591a 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -16,6 +16,8 @@ #include "core/data/logical_store.h" #include "core/mapping/core_mapper.h" +#include "core/partitioning/partition.h" +#include "core/partitioning/partitioner.h" #include "core/runtime/context.h" #include "core/runtime/projection.h" #include "core/runtime/shard.h" @@ -407,8 +409,24 @@ std::unique_ptr Runtime::create_task(LibraryContext* library, void Runtime::submit(std::unique_ptr op) { - // TODO: We need to build a lazy evaluation pipeline here - op->launch(); + operations_.push_back(std::move(op)); + if (operations_.size() >= window_size_) { + std::vector> to_schedule; + to_schedule.swap(operations_); + schedule(std::move(to_schedule)); + } +} + +void Runtime::schedule(std::vector> operations) +{ + std::vector op_pointers{}; + op_pointers.reserve(operations.size()); + for (auto& op : operations) op_pointers.push_back(op.get()); + + Partitioner partitioner(this, std::move(op_pointers)); + auto strategy = partitioner.partition_stores(); + + for (auto& op : operations) op->launch(strategy.get()); } std::shared_ptr Runtime::create_store(std::vector extents, @@ -485,6 +503,14 @@ IndexSpace Runtime::find_or_create_index_space(const Domain& shape) } } +Legion::IndexPartition Runtime::create_index_partition(const Legion::IndexSpace& index_space, + const Legion::IndexSpace& color_space, + Legion::PartitionKind kind, + const PartitioningFunctor* functor) +{ + return functor->construct(legion_runtime_, legion_context_, index_space, color_space, kind); +} + FieldSpace Runtime::create_field_space() { assert(nullptr != legion_context_); @@ -497,6 +523,13 @@ LogicalRegion Runtime::create_region(const IndexSpace& index_space, const FieldS return legion_runtime_->create_logical_region(legion_context_, index_space, field_space); } +Legion::LogicalPartition Runtime::create_logical_partition( + const Legion::LogicalRegion& logical_region, const Legion::IndexPartition& index_partition) +{ + assert(nullptr != legion_context_); + return legion_runtime_->get_logical_partition(legion_context_, logical_region, index_partition); +} + FieldID Runtime::allocate_field(const FieldSpace& field_space, size_t field_size) { assert(nullptr != legion_context_); @@ -517,6 +550,13 @@ std::shared_ptr Runtime::dispatch(TaskLauncher* launcher) return nullptr; } +std::shared_ptr Runtime::dispatch(IndexTaskLauncher* launcher) +{ + assert(nullptr != legion_context_); + legion_runtime_->execute_index_space(legion_context_, *launcher); + return nullptr; +} + /*static*/ void Runtime::initialize(int32_t argc, char** argv) { Legion::Runtime::initialize(&argc, &argv, true /*filter legion and realm args*/); diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 04d3cf4a01..fdbd856fb0 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -51,6 +51,7 @@ class Task; class LibraryContext; class LogicalRegionField; class LogicalStore; +class PartitioningFunctor; class RegionManager { public: @@ -120,14 +121,24 @@ class Runtime { public: Legion::IndexSpace find_or_create_index_space(const Legion::Domain& shape); + Legion::IndexPartition create_index_partition(const Legion::IndexSpace& index_space, + const Legion::IndexSpace& color_space, + Legion::PartitionKind kind, + const PartitioningFunctor* functor); Legion::FieldSpace create_field_space(); Legion::LogicalRegion create_region(const Legion::IndexSpace& index_space, const Legion::FieldSpace& field_space); + Legion::LogicalPartition create_logical_partition(const Legion::LogicalRegion& logical_region, + const Legion::IndexPartition& index_partition); Legion::FieldID allocate_field(const Legion::FieldSpace& field_space, size_t field_size); Legion::Domain get_index_space_domain(const Legion::IndexSpace& index_space) const; public: std::shared_ptr dispatch(Legion::TaskLauncher* launcher); + std::shared_ptr dispatch(Legion::IndexTaskLauncher* launcher); + + private: + void schedule(std::vector> operations); public: static void initialize(int32_t argc, char** argv); @@ -149,6 +160,10 @@ class Runtime { private: std::map index_spaces_; + private: + std::vector> operations_; + size_t window_size_{1}; + private: std::map libraries_; }; From c4e6ad605e0f94206c7d5779febbdf0e308948a0 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 1 Oct 2021 16:45:56 -0700 Subject: [PATCH 0010/1425] Hide region and field managers from the API --- src/core/runtime/runtime.cc | 32 ++++++++++++++++++++++++++++++++ src/core/runtime/runtime.h | 34 ++-------------------------------- 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 7de2a9591a..cdb5928a32 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -301,6 +301,24 @@ void register_legate_core_tasks(Machine machine, // legate::RegionManager //////////////////////////////////////////////////// +class RegionManager { + public: + RegionManager(Runtime* runtime, const Legion::Domain& shape); + + private: + Legion::LogicalRegion active_region() const; + bool has_space() const; + void create_region(); + + public: + std::pair allocate_field(size_t field_size); + + private: + Runtime* runtime_; + Legion::Domain shape_; + std::vector regions_{}; +}; + RegionManager::RegionManager(Runtime* runtime, const Domain& shape) : runtime_(runtime), shape_(shape) { @@ -329,6 +347,20 @@ std::pair RegionManager::allocate_field( // legate::FieldManager //////////////////////////////////////////////////// +class FieldManager { + public: + FieldManager(Runtime* runtime, const Legion::Domain& shape, LegateTypeCode code); + + public: + std::shared_ptr allocate_field(); + + private: + Runtime* runtime_; + Legion::Domain shape_; + LegateTypeCode code_; + size_t field_size_; +}; + struct field_size_fn { template size_t operator()() diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index fdbd856fb0..ed0c556082 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -52,38 +52,8 @@ class LibraryContext; class LogicalRegionField; class LogicalStore; class PartitioningFunctor; - -class RegionManager { - public: - RegionManager(Runtime* runtime, const Legion::Domain& shape); - - private: - Legion::LogicalRegion active_region() const; - bool has_space() const; - void create_region(); - - public: - std::pair allocate_field(size_t field_size); - - private: - Runtime* runtime_; - Legion::Domain shape_; - std::vector regions_{}; -}; - -class FieldManager { - public: - FieldManager(Runtime* runtime, const Legion::Domain& shape, LegateTypeCode code); - - public: - std::shared_ptr allocate_field(); - - private: - Runtime* runtime_; - Legion::Domain shape_; - LegateTypeCode code_; - size_t field_size_; -}; +class RegionManager; +class FieldManager; class Runtime { public: From ee61ba2d17be79f1eb4ffa4a5c4ce16285593bd8 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 1 Oct 2021 23:05:10 -0700 Subject: [PATCH 0011/1425] Add a partition manager and a code path for parallel task launch --- src/core.mk | 1 + src/core/data/logical_store.cc | 24 ++++++++++- src/core/data/logical_store.h | 7 ++- src/core/partitioning/partition.cc | 53 ++++++++++++++++++++++- src/core/partitioning/partition.h | 10 ++++- src/core/partitioning/partitioner.cc | 38 +++++++++++------ src/core/partitioning/partitioner.h | 13 +++--- src/core/runtime/operation.cc | 10 ++--- src/core/runtime/operation.h | 2 +- src/core/runtime/runtime.cc | 64 +++++++++++++++++++++++++--- src/core/runtime/runtime.h | 32 ++++++++++++-- src/core/runtime/runtime.inl | 27 ++++++++++++ 12 files changed, 239 insertions(+), 42 deletions(-) create mode 100644 src/core/runtime/runtime.inl diff --git a/src/core.mk b/src/core.mk index 9d79c0f91c..932b480015 100644 --- a/src/core.mk +++ b/src/core.mk @@ -68,6 +68,7 @@ INSTALL_HEADERS = legate.h \ core/runtime/context.h \ core/runtime/operation.h \ core/runtime/runtime.h \ + core/runtime/runtime.inl \ core/task/return.h \ core/task/task.h \ core/utilities/deserializer.h \ diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index 3ffd1deeb7..b6ac195736 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -38,7 +38,7 @@ Domain LogicalRegionField::domain() const LogicalStore::LogicalStore(Runtime* runtime, LegateTypeCode code, - std::vector extents, + std::vector extents, std::shared_ptr transform /*= nullptr*/) : runtime_(runtime), code_(code), extents_(std::move(extents)), transform_(std::move(transform)) { @@ -52,6 +52,13 @@ Domain LogicalStore::domain() const return region_field_->domain(); } +size_t LogicalStore::volume() const +{ + size_t vol = 1; + for (auto extent : extents_) vol *= extent; + return vol; +} + std::shared_ptr LogicalStore::get_storage() { if (!has_storage()) create_storage(); @@ -78,9 +85,24 @@ std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) std::unique_ptr LogicalStore::find_or_create_partition(const Partition* partition) { + // We're about to create a legion partition for this store, so the store should have its region + // created. + if (!has_storage()) create_storage(); auto lp = partition->construct(this, partition->is_disjoint_for(this), partition->is_complete_for(this)); return std::make_unique(lp, 0); } +std::unique_ptr LogicalStore::find_or_create_key_partition() +{ + auto part_mgr = runtime_->get_partition_manager(); + auto launch_shape = part_mgr->compute_launch_shape(this); + if (launch_shape.empty()) + return create_no_partition(runtime_); + else { + auto tile_shape = part_mgr->compute_tile_shape(extents_, launch_shape); + return create_tiling(runtime_, std::move(tile_shape), std::move(launch_shape)); + } +} + } // namespace legate diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index 13adf9e86d..4be95d3b58 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -64,7 +64,7 @@ class LogicalStore { LogicalStore() {} LogicalStore(Runtime* runtime, LegateTypeCode code, - std::vector extents, + std::vector extents, std::shared_ptr transform = nullptr); public: @@ -79,6 +79,8 @@ class LogicalStore { int32_t dim() const; LegateTypeCode code() const { return code_; } Legion::Domain domain() const; + const std::vector& extents() const { return extents_; } + size_t volume() const; public: bool has_storage() const { return nullptr != region_field_; } @@ -93,11 +95,12 @@ class LogicalStore { public: std::unique_ptr find_or_create_partition(const Partition* partition); + std::unique_ptr find_or_create_key_partition(); private: Runtime* runtime_{nullptr}; LegateTypeCode code_{MAX_TYPE_NUMBER}; - std::vector extents_; + std::vector extents_; std::shared_ptr region_field_{nullptr}; std::shared_ptr transform_{nullptr}; std::shared_ptr mapped_{nullptr}; diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index ad8fab7927..b3271e421c 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -37,6 +37,10 @@ class NoPartition : public Partition { bool complete) const override; virtual std::unique_ptr get_projection(LogicalStore* store) const override; + public: + virtual bool has_launch_domain() const override; + virtual Legion::Domain launch_domain() const override; + private: Runtime* runtime_; }; @@ -55,6 +59,10 @@ class Tiling : public Partition { bool complete) const override; virtual std::unique_ptr get_projection(LogicalStore* store) const override; + public: + virtual bool has_launch_domain() const override; + virtual Legion::Domain launch_domain() const override; + private: Runtime* runtime_; Shape tile_shape_; @@ -84,6 +92,10 @@ Tiling::Tiling(Runtime* runtime, Shape&& tile_shape, Shape&& color_shape, Shape& color_shape_(std::forward(color_shape)), offsets_(std::forward(offsets)) { + if (offsets_.empty()) + for (auto _ : tile_shape_) offsets_.push_back(0); + assert(tile_shape_.size() == color_shape_.size()); + assert(tile_shape_.size() == offsets_.size()); } NoPartition::NoPartition(Runtime* runtime) : runtime_(runtime) {} @@ -104,7 +116,15 @@ std::unique_ptr NoPartition::get_projection(LogicalStore* store) con return std::make_unique(); } -bool Tiling::is_complete_for(const LogicalStore* store) const {} +bool NoPartition::has_launch_domain() const { return false; } + +Legion::Domain NoPartition::launch_domain() const +{ + assert(false); + return Legion::Domain(); +} + +bool Tiling::is_complete_for(const LogicalStore* store) const { return false; } bool Tiling::is_disjoint_for(const LogicalStore* store) const { return true; } @@ -112,10 +132,27 @@ Legion::LogicalPartition Tiling::construct(const LogicalStore* store, bool disjoint, bool complete) const { + auto ndim = static_cast(tile_shape_.size()); + Legion::DomainTransform transform; + transform.m = ndim; + transform.n = ndim; + for (int32_t idx = 0; idx < ndim * ndim; ++idx) transform.matrix[idx] = 0; + for (int32_t idx = 0; idx < ndim; ++idx) transform.matrix[ndim * idx + idx] = tile_shape_[idx]; + Legion::Domain extent; + extent.dim = ndim; + for (int32_t idx = 0; idx < ndim; ++idx) { + extent.rect_data[idx] = offsets_[idx]; + extent.rect_data[idx + ndim] = tile_shape_[idx] - 1 + offsets_[idx]; + } Legion::Domain color_domain; + color_domain.dim = ndim; + for (int32_t idx = 0; idx < ndim; ++idx) { + color_domain.rect_data[idx] = 0; + color_domain.rect_data[idx + ndim] = color_shape_[idx] - 1; + } auto region = store->get_storage_unsafe()->region(); auto color_space = runtime_->find_or_create_index_space(color_domain); @@ -134,6 +171,20 @@ std::unique_ptr Tiling::get_projection(LogicalStore* store) const return store->find_or_create_partition(this); } +bool Tiling::has_launch_domain() const { return true; } + +Legion::Domain Tiling::launch_domain() const +{ + Legion::Domain launch_domain; + int32_t ndim = static_cast(color_shape_.size()); + launch_domain.dim = ndim; + for (int32_t idx = 0; idx < ndim; ++idx) { + launch_domain.rect_data[idx] = 0; + launch_domain.rect_data[idx + ndim] = color_shape_[idx] - 1; + } + return launch_domain; +} + PartitionByRestriction::PartitionByRestriction(Legion::DomainTransform transform, Legion::Domain extent) : transform_(transform), extent_(extent) diff --git a/src/core/partitioning/partition.h b/src/core/partitioning/partition.h index fce30bd296..2b797186d8 100644 --- a/src/core/partitioning/partition.h +++ b/src/core/partitioning/partition.h @@ -28,12 +28,18 @@ class Runtime; struct Partition { public: - virtual bool is_complete_for(const LogicalStore* store) const = 0; - virtual bool is_disjoint_for(const LogicalStore* store) const = 0; + virtual bool is_complete_for(const LogicalStore* store) const = 0; + virtual bool is_disjoint_for(const LogicalStore* store) const = 0; + + public: virtual Legion::LogicalPartition construct(const LogicalStore* store, bool disjoint = false, bool complete = false) const = 0; virtual std::unique_ptr get_projection(LogicalStore* store) const = 0; + + public: + virtual bool has_launch_domain() const = 0; + virtual Legion::Domain launch_domain() const = 0; }; struct PartitioningFunctor { diff --git a/src/core/partitioning/partitioner.cc b/src/core/partitioning/partitioner.cc index 2e079106e3..72ec287224 100644 --- a/src/core/partitioning/partitioner.cc +++ b/src/core/partitioning/partitioner.cc @@ -15,6 +15,7 @@ */ #include "core/partitioning/partitioner.h" +#include "core/data/logical_store.h" #include "core/data/scalar.h" #include "core/partitioning/partition.h" #include "core/runtime/launcher.h" @@ -23,20 +24,24 @@ namespace legate { -Strategy::Strategy() : launch_domain_(nullptr) {} +Strategy::Strategy() {} -Strategy::Strategy(const Legion::Domain& launch_domain) - : launch_domain_(std::make_unique(launch_domain)) +bool Strategy::parallel(const Operation* op) const { + auto finder = launch_domains_.find(op); + return finder != launch_domains_.end(); } -bool Strategy::parallel() const { return nullptr != launch_domain_; } - -Legion::Domain Strategy::launch_domain() const { return *launch_domain_; } +Legion::Domain Strategy::launch_domain(const Operation* op) const +{ + auto finder = launch_domains_.find(op); + assert(finder != launch_domains_.end()); + return finder->second; +} -void Strategy::set_launch_domain(const Legion::Domain& launch_domain) +void Strategy::set_launch_domain(const Operation* op, const Legion::Domain& launch_domain) { - launch_domain_ = std::make_unique(launch_domain); + launch_domains_[op] = launch_domain; } void Strategy::insert(const LogicalStore* store, std::shared_ptr partition) @@ -58,8 +63,8 @@ std::unique_ptr Strategy::get_projection(LogicalStore* store) const return partition->get_projection(store); } -Partitioner::Partitioner(Runtime* runtime, std::vector&& operations) - : runtime_(runtime), operations_(std::forward>(operations)) +Partitioner::Partitioner(Runtime* runtime, std::vector&& operations) + : runtime_(runtime), operations_(std::forward>(operations)) { } @@ -68,8 +73,17 @@ std::unique_ptr Partitioner::partition_stores() auto strategy = std::make_unique(); for (auto op : operations_) { - auto all_stores = op->all_stores(); - for (auto store : all_stores) strategy->insert(store, create_no_partition(runtime_)); + bool determined_launch_domain = false; + auto all_stores = op->all_stores(); + for (auto store : all_stores) { + auto key_partition = store->find_or_create_key_partition(); + if (!determined_launch_domain) { + determined_launch_domain = true; + if (key_partition->has_launch_domain()) + strategy->set_launch_domain(op, key_partition->launch_domain()); + } + strategy->insert(store, std::move(key_partition)); + } } return std::move(strategy); diff --git a/src/core/partitioning/partitioner.h b/src/core/partitioning/partitioner.h index e9829d5562..70e36ef187 100644 --- a/src/core/partitioning/partitioner.h +++ b/src/core/partitioning/partitioner.h @@ -32,12 +32,11 @@ class Projection; class Strategy { public: Strategy(); - Strategy(const Legion::Domain& launch_domain); public: - bool parallel() const; - Legion::Domain launch_domain() const; - void set_launch_domain(const Legion::Domain& launch_domain); + bool parallel(const Operation* op) const; + Legion::Domain launch_domain(const Operation* op) const; + void set_launch_domain(const Operation* op, const Legion::Domain& launch_domain); public: void insert(const LogicalStore* store, std::shared_ptr partition); @@ -47,20 +46,20 @@ class Strategy { std::unique_ptr get_projection(LogicalStore* store) const; private: - std::unique_ptr launch_domain_; std::unordered_map> assignments_; + std::unordered_map launch_domains_; }; class Partitioner { public: - Partitioner(Runtime* runtime, std::vector&& operations); + Partitioner(Runtime* runtime, std::vector&& operations); public: std::unique_ptr partition_stores(); private: Runtime* runtime_; - std::vector operations_; + std::vector operations_; }; } // namespace legate diff --git a/src/core/runtime/operation.cc b/src/core/runtime/operation.cc index 388db1806d..2ef1099093 100644 --- a/src/core/runtime/operation.cc +++ b/src/core/runtime/operation.cc @@ -40,10 +40,10 @@ void Operation::add_reduction(LogicalStoreP store, Legion::ReductionOpID redop) reductions_.push_back(Reduction(store, redop)); } -std::vector Operation::all_stores() const +std::vector Operation::all_stores() { - std::vector result; - std::unordered_set added; + std::vector result; + std::unordered_set added; auto add_all = [&](auto& stores) { for (auto& store : stores) { @@ -89,8 +89,8 @@ void Task::launch(Strategy* strategy) const } for (auto& scalar : scalars_) launcher.add_scalar(scalar); - if (strategy->parallel()) - launcher.execute(strategy->launch_domain()); + if (strategy->parallel(this)) + launcher.execute(strategy->launch_domain(this)); else launcher.execute_single(); } diff --git a/src/core/runtime/operation.h b/src/core/runtime/operation.h index fdfbff2f29..013d5926b7 100644 --- a/src/core/runtime/operation.h +++ b/src/core/runtime/operation.h @@ -42,7 +42,7 @@ class Operation { void add_reduction(LogicalStoreP store, Legion::ReductionOpID redop); public: - std::vector all_stores() const; + std::vector all_stores(); public: virtual void launch(Strategy* strategy) const = 0; diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index cdb5928a32..330344c089 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -104,7 +104,7 @@ static void toplevel_task(const Legion::Task* task, Legion::Runtime* legion_runtime) { auto runtime = Runtime::get_runtime(); - runtime->set_legion_context(ctx); + runtime->post_startup_initialization(ctx); if (nullptr == Core::main_fn) { log_legate.error( @@ -385,6 +385,52 @@ std::shared_ptr FieldManager::allocate_field() return std::make_shared(runtime_, lr, fid); } +//////////////////////////////////////////////////// +// legate::PartitionManager +//////////////////////////////////////////////////// + +PartitionManager::PartitionManager(Runtime* runtime, const LibraryContext* context) +{ + num_pieces_ = runtime->get_tunable(context, LEGATE_CORE_TUNABLE_NUM_PIECES); + min_shard_volume_ = runtime->get_tunable(context, LEGATE_CORE_TUNABLE_MIN_SHARD_VOLUME); + + int32_t remaining_pieces = num_pieces_; + auto push_factors = [&](auto prime) { + while (remaining_pieces % prime == 0) { + piece_factors_.push_back(prime); + remaining_pieces /= prime; + } + }; + + push_factors(11); + push_factors(7); + push_factors(5); + push_factors(3); + push_factors(2); +} + +std::vector PartitionManager::compute_launch_shape(const LogicalStore* store) +{ + assert(store->dim() == 1); + size_t max_pieces = (store->volume() + min_shard_volume_ - 1) / min_shard_volume_; + std::vector launch_shape; + if (max_pieces > 1) launch_shape.push_back(std::max(1, max_pieces)); + return std::move(launch_shape); +} + +std::vector PartitionManager::compute_tile_shape(const std::vector& extents, + const std::vector& launch_shape) +{ + assert(extents.size() == launch_shape.size()); + std::vector tile_shape; + for (uint32_t idx = 0; idx < extents.size(); ++idx) { + auto x = extents[idx]; + auto y = launch_shape[idx]; + tile_shape.push_back((x + y - 1) / y); + } + return std::move(tile_shape); +} + //////////////////////////////////////////////////// // legate::Runtime //////////////////////////////////////////////////// @@ -426,9 +472,11 @@ LibraryContext* Runtime::create_library(const std::string& library_name, return context; } -void Runtime::set_legion_context(Legion::Context legion_context) +void Runtime::post_startup_initialization(Legion::Context legion_context) { - legion_context_ = legion_context; + legion_context_ = legion_context; + core_context_ = find_library(core_library_name); + partition_manager_ = new PartitionManager(this, core_context_); } // This function should be moved to the library context @@ -451,7 +499,7 @@ void Runtime::submit(std::unique_ptr op) void Runtime::schedule(std::vector> operations) { - std::vector op_pointers{}; + std::vector op_pointers{}; op_pointers.reserve(operations.size()); for (auto& op : operations) op_pointers.push_back(op.get()); @@ -461,14 +509,14 @@ void Runtime::schedule(std::vector> operations) for (auto& op : operations) op->launch(strategy.get()); } -std::shared_ptr Runtime::create_store(std::vector extents, +std::shared_ptr Runtime::create_store(std::vector extents, LegateTypeCode code) { return std::make_shared(this, code, extents); } -std::shared_ptr Runtime::create_region_field( - const std::vector& extents, LegateTypeCode code) +std::shared_ptr Runtime::create_region_field(const std::vector& extents, + LegateTypeCode code) { DomainPoint lo, hi; hi.dim = lo.dim = static_cast(extents.size()); @@ -522,6 +570,8 @@ FieldManager* Runtime::find_or_create_field_manager(const Domain& shape, LegateT } } +PartitionManager* Runtime::get_partition_manager() { return partition_manager_; } + IndexSpace Runtime::find_or_create_index_space(const Domain& shape) { assert(nullptr != legion_context_); diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index ed0c556082..4882774ce3 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -21,6 +21,7 @@ #include "legion.h" #include "core/data/store.h" +#include "core/runtime/context.h" #include "core/utilities/typedefs.h" namespace legate { @@ -48,13 +49,27 @@ class ResourceConfig; class Runtime; class Operation; class Task; -class LibraryContext; class LogicalRegionField; class LogicalStore; class PartitioningFunctor; class RegionManager; class FieldManager; +class PartitionManager { + public: + PartitionManager(Runtime* runtime, const LibraryContext* context); + + public: + std::vector compute_launch_shape(const LogicalStore* store); + std::vector compute_tile_shape(const std::vector& extents, + const std::vector& launch_shape); + + private: + int32_t num_pieces_; + int32_t min_shard_volume_; + std::vector piece_factors_; +}; + class Runtime { public: public: @@ -70,7 +85,11 @@ class Runtime { LibraryContext* create_library(const std::string& library_name, const ResourceConfig& config); public: - void set_legion_context(Legion::Context legion_context); + void post_startup_initialization(Legion::Context legion_context); + + public: + template + T get_tunable(const LibraryContext* context, int64_t tunable_id, int64_t mapper_id = 0); public: std::unique_ptr create_task(LibraryContext* library, @@ -79,8 +98,8 @@ class Runtime { void submit(std::unique_ptr op); public: - std::shared_ptr create_store(std::vector extents, LegateTypeCode code); - std::shared_ptr create_region_field(const std::vector& extents, + std::shared_ptr create_store(std::vector extents, LegateTypeCode code); + std::shared_ptr create_region_field(const std::vector& extents, LegateTypeCode code); RegionField map_region_field(LibraryContext* context, std::shared_ptr region_field); @@ -88,6 +107,7 @@ class Runtime { public: RegionManager* find_or_create_region_manager(const Legion::Domain& shape); FieldManager* find_or_create_field_manager(const Legion::Domain& shape, LegateTypeCode code); + PartitionManager* get_partition_manager(); public: Legion::IndexSpace find_or_create_index_space(const Legion::Domain& shape); @@ -122,10 +142,12 @@ class Runtime { private: Legion::Runtime* legion_runtime_; Legion::Context legion_context_{nullptr}; + LibraryContext* core_context_{nullptr}; private: std::map, FieldManager*> field_managers_; std::map region_managers_; + PartitionManager* partition_manager_{nullptr}; private: std::map index_spaces_; @@ -145,3 +167,5 @@ void set_main_function(LegateMainFnPtr p_main); int32_t start(int32_t argc, char** argv); } // namespace legate + +#include "core/runtime/runtime.inl" diff --git a/src/core/runtime/runtime.inl b/src/core/runtime/runtime.inl new file mode 100644 index 0000000000..7077ea0d83 --- /dev/null +++ b/src/core/runtime/runtime.inl @@ -0,0 +1,27 @@ +/* Copyright 2021 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +namespace legate { + +template +T Runtime::get_tunable(const LibraryContext* context, int64_t tunable_id, int64_t mapper_id /*= 0*/) +{ + Legion::TunableLauncher launcher(tunable_id, context->get_mapper_id(mapper_id), 0, sizeof(T)); + auto future = legion_runtime_->select_tunable_value(legion_context_, launcher); + return future.get_result(); +} + +} // namespace legate From 61768615edf8b2a0b873e31fd24a3ec31e26b62b Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 13 Oct 2021 01:13:26 -0700 Subject: [PATCH 0012/1425] Copy the Python core's heuristic for color shape computation --- src/core/runtime/runtime.cc | 133 ++++++++++++++++++++++++++++++++++-- 1 file changed, 128 insertions(+), 5 deletions(-) diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 330344c089..1c3ef6e2fc 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -14,6 +14,9 @@ * */ +#include +#include + #include "core/data/logical_store.h" #include "core/mapping/core_mapper.h" #include "core/partitioning/partition.h" @@ -394,6 +397,9 @@ PartitionManager::PartitionManager(Runtime* runtime, const LibraryContext* conte num_pieces_ = runtime->get_tunable(context, LEGATE_CORE_TUNABLE_NUM_PIECES); min_shard_volume_ = runtime->get_tunable(context, LEGATE_CORE_TUNABLE_MIN_SHARD_VOLUME); + assert(num_pieces_ > 0); + assert(min_shard_volume_ > 0); + int32_t remaining_pieces = num_pieces_; auto push_factors = [&](auto prime) { while (remaining_pieces % prime == 0) { @@ -411,11 +417,128 @@ PartitionManager::PartitionManager(Runtime* runtime, const LibraryContext* conte std::vector PartitionManager::compute_launch_shape(const LogicalStore* store) { - assert(store->dim() == 1); - size_t max_pieces = (store->volume() + min_shard_volume_ - 1) / min_shard_volume_; - std::vector launch_shape; - if (max_pieces > 1) launch_shape.push_back(std::max(1, max_pieces)); - return std::move(launch_shape); + // Easy case if we only have one piece: no parallel launch space + if (num_pieces_ == 1) return {}; + + // If we only have one point then we never do parallel launches + auto& shape = store->extents(); + if (std::all_of(shape.begin(), shape.end(), [](auto extent) { return 1 == extent; })) return {}; + + // Prune out any dimensions that are 1 + std::vector temp_shape{}; + std::vector temp_dims{}; + int64_t volume = 1; + for (uint32_t dim = 0; dim < shape.size(); ++dim) { + auto extent = shape[dim]; + if (1 == extent) continue; + temp_shape.push_back(extent); + temp_dims.push_back(dim); + volume *= extent; + } + + // Figure out how many shards we can make with this array + int64_t max_pieces = (volume + min_shard_volume_ - 1) / min_shard_volume_; + assert(max_pieces > 0); + // If we can only make one piece return that now + if (1 == max_pieces) return {}; + + // Otherwise we need to compute it ourselves + // TODO: a better heuristic here. + // For now if we can make at least two pieces then we will make N pieces. + max_pieces = num_pieces_; + + // First compute the N-th root of the number of pieces + uint32_t ndim = temp_shape.size(); + assert(ndim > 0); + std::vector temp_result{}; + + if (1 == ndim) { + // Easy one dimensional case + temp_result.push_back(std::min(temp_shape.front(), static_cast(max_pieces))); + } else if (2 == ndim) { + if (volume < max_pieces) { + // TBD: Once the max_pieces heuristic is fixed, this should never happen + temp_result.swap(temp_shape); + } else { + // Two dimensional so we can use square root to try and generate as square a pieces + // as possible since most often we will be doing matrix operations with these + auto nx = temp_shape[0]; + auto ny = temp_shape[1]; + auto swap = nx > ny; + if (swap) std::swap(nx, ny); + auto n = std::sqrt(static_cast(max_pieces) * nx / ny); + + // Need to constraint n to be an integer with numpcs % n == 0 + // try rounding n both up and down + auto n1 = std::max(1, static_cast(std::floor(n + 1e-12))); + while (max_pieces % n1 != 0) --n1; + auto n2 = std::max(1, static_cast(std::floor(n - 1e-12))); + while (max_pieces % n2 != 0) ++n2; + + // pick whichever of n1 and n2 gives blocks closest to square + // i.e. gives the shortest long side + auto side1 = std::max(nx / n1, ny / (max_pieces / n1)); + auto side2 = std::max(nx / n2, ny / (max_pieces / n2)); + auto px = static_cast(side1 <= side2 ? n1 : n2); + auto py = static_cast(max_pieces / px); + + // we need to trim launch space if it is larger than the + // original shape in one of the dimensions (can happen in + // testing) + if (swap) { + temp_result.push_back(std::min(py, temp_shape[0])); + temp_result.push_back(std::min(px, temp_shape[1])); + } else { + temp_result.push_back(std::min(px, temp_shape[1])); + temp_result.push_back(std::min(py, temp_shape[0])); + } + } + } else { + // For higher dimensions we care less about "square"-ness and more about evenly dividing + // things, compute the prime factors for our number of pieces and then round-robin them + // onto the shape, with the goal being to keep the last dimension >= 32 for good memory + // performance on the GPU + temp_result.resize(ndim); + std::fill(temp_result.begin(), temp_result.end(), 1); + size_t factor_prod = 1; + for (auto factor : piece_factors_) { + // Avoid exceeding the maximum number of pieces + if (factor * factor_prod > max_pieces) break; + + factor_prod *= factor; + + std::vector remaining; + for (uint32_t idx = 0; idx < temp_shape.size(); ++idx) + remaining.push_back((temp_shape[idx] + temp_result[idx] - 1) / temp_result[idx]); + uint32_t big_dim = std::max_element(remaining.begin(), remaining.end()) - remaining.begin(); + if (big_dim < ndim - 1) { + // Not the last dimension, so do it + temp_result[big_dim] *= factor; + } else { + // Last dim so see if it still bigger than 32 + if (remaining[big_dim] / factor >= 32) { + // go ahead and do it + temp_result[big_dim] *= factor; + } else { + // Won't be see if we can do it with one of the other dimensions + uint32_t next_big_dim = + std::max_element(remaining.begin(), remaining.end() - 1) - remaining.begin(); + if (remaining[next_big_dim] / factor > 0) + temp_result[next_big_dim] *= factor; + else + // Fine just do it on the last dimension + temp_result[big_dim] *= factor; + } + } + } + } + + // Project back onto the original number of dimensions + assert(temp_result.size() == ndim); + std::vector result(shape.size(), 1); + for (uint32_t idx = 0; idx < ndim; ++idx) result[temp_dims[idx]] = temp_result[idx]; + + return std::move(result); } std::vector PartitionManager::compute_tile_shape(const std::vector& extents, From a9cc9cff7f783a1bd3f60143ae362a8cc9de8340 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 13 Oct 2021 16:56:35 -0700 Subject: [PATCH 0013/1425] Sort class declarations in each file --- src/core/data/logical_store.h | 4 ++-- src/core/partitioning/partitioner.h | 4 ++-- src/core/runtime/context.h | 2 +- src/core/runtime/launcher.h | 10 ++++------ src/core/runtime/operation.h | 4 ++-- src/core/runtime/runtime.h | 10 +++++----- 6 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index 4be95d3b58..a2cf7e1fb2 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -25,11 +25,11 @@ namespace legate { -class Runtime; -class Store; class LibraryContext; class Partition; class Projection; +class Runtime; +class Store; class LogicalRegionField { public: diff --git a/src/core/partitioning/partitioner.h b/src/core/partitioning/partitioner.h index 70e36ef187..b317ad4628 100644 --- a/src/core/partitioning/partitioner.h +++ b/src/core/partitioning/partitioner.h @@ -23,11 +23,11 @@ namespace legate { -class Operation; -class Runtime; class LogicalStore; +class Operation; class Partition; class Projection; +class Runtime; class Strategy { public: diff --git a/src/core/runtime/context.h b/src/core/runtime/context.h index 6c4efc1977..8b7a6ce0a2 100644 --- a/src/core/runtime/context.h +++ b/src/core/runtime/context.h @@ -22,8 +22,8 @@ namespace legate { -class Store; class Scalar; +class Store; struct ResourceConfig { int64_t max_tasks{1000000}; diff --git a/src/core/runtime/launcher.h b/src/core/runtime/launcher.h index b8cbd55841..9bb20889e1 100644 --- a/src/core/runtime/launcher.h +++ b/src/core/runtime/launcher.h @@ -22,16 +22,14 @@ namespace legate { +class ArgWrapper; class BufferBuilder; +class LibraryContext; +class LogicalStore; class RegionReq; class RequirementAnalyzer; -class ArgWrapper; - -class LogicalStore; -class Scalar; - class Runtime; -class LibraryContext; +class Scalar; class Projection { protected: diff --git a/src/core/runtime/operation.h b/src/core/runtime/operation.h index 013d5926b7..20ed208e5b 100644 --- a/src/core/runtime/operation.h +++ b/src/core/runtime/operation.h @@ -22,10 +22,10 @@ namespace legate { +class LibraryContext; class LogicalStore; -class Scalar; class Runtime; -class LibraryContext; +class Scalar; class Strategy; class Operation { diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 4882774ce3..03dc781ec5 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -45,15 +45,15 @@ class Core { #endif }; -class ResourceConfig; -class Runtime; -class Operation; -class Task; +class FieldManager; class LogicalRegionField; class LogicalStore; +class Operation; class PartitioningFunctor; class RegionManager; -class FieldManager; +class ResourceConfig; +class Runtime; +class Task; class PartitionManager { public: From 97540cf58bda04ba9e60024a6a6bb867e1acdeba Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 15 Oct 2021 11:38:28 -0700 Subject: [PATCH 0014/1425] Inteface to specify partitioning constraints. The solver is still incomplete --- src/core.mk | 47 +++++---- src/core/partitioning/constraint.cc | 80 ++++++++++++++ src/core/partitioning/constraint.h | 105 ++++++++++++++++++ src/core/partitioning/constraint_graph.cc | 62 +++++++++++ src/core/partitioning/constraint_graph.h | 48 +++++++++ src/core/partitioning/partition.cc | 32 +++++- src/core/partitioning/partition.h | 6 ++ src/core/partitioning/partitioner.cc | 84 ++++++++++----- src/core/partitioning/partitioner.h | 16 +-- src/core/runtime/context.cc | 12 +++ src/core/runtime/context.h | 7 ++ src/core/runtime/launcher.cc | 8 +- src/core/runtime/launcher.h | 8 +- src/core/runtime/operation.cc | 123 ++++++++++++++-------- src/core/runtime/operation.h | 53 +++++++--- src/core/runtime/runtime.cc | 4 +- src/core/runtime/runtime.h | 1 + src/core/task/task.cc | 1 + src/legate.h | 1 + 19 files changed, 574 insertions(+), 124 deletions(-) create mode 100644 src/core/partitioning/constraint.cc create mode 100644 src/core/partitioning/constraint.h create mode 100644 src/core/partitioning/constraint_graph.cc create mode 100644 src/core/partitioning/constraint_graph.h diff --git a/src/core.mk b/src/core.mk index 0743cff6d2..b88a0d26c2 100644 --- a/src/core.mk +++ b/src/core.mk @@ -15,28 +15,30 @@ # General source files -GEN_CPU_SRC = core/legate_c.cc \ - core/data/logical_store.cc \ - core/data/scalar.cc \ - core/data/store.cc \ - core/data/transform.cc \ - core/mapping/base_mapper.cc \ - core/mapping/core_mapper.cc \ - core/mapping/instance_manager.cc \ - core/mapping/mapping.cc \ - core/mapping/task.cc \ - core/partitioning/partition.cc \ - core/partitioning/partitioner.cc \ - core/runtime/context.cc \ - core/runtime/launcher.cc \ - core/runtime/operation.cc \ - core/runtime/projection.cc \ - core/runtime/runtime.cc \ - core/runtime/shard.cc \ - core/task/return.cc \ - core/task/task.cc \ - core/utilities/deserializer.cc \ - core/utilities/machine.cc \ +GEN_CPU_SRC = core/legate_c.cc \ + core/data/logical_store.cc \ + core/data/scalar.cc \ + core/data/store.cc \ + core/data/transform.cc \ + core/mapping/base_mapper.cc \ + core/mapping/core_mapper.cc \ + core/mapping/instance_manager.cc \ + core/mapping/mapping.cc \ + core/mapping/task.cc \ + core/partitioning/constraint.cc \ + core/partitioning/constraint_graph.cc \ + core/partitioning/partition.cc \ + core/partitioning/partitioner.cc \ + core/runtime/context.cc \ + core/runtime/launcher.cc \ + core/runtime/operation.cc \ + core/runtime/projection.cc \ + core/runtime/runtime.cc \ + core/runtime/shard.cc \ + core/task/return.cc \ + core/task/task.cc \ + core/utilities/deserializer.cc \ + core/utilities/machine.cc \ core/utilities/linearize.cc ifeq ($(strip $(USE_CUDA)),1) @@ -59,6 +61,7 @@ INSTALL_HEADERS = legate.h \ core/mapping/mapping.h \ core/mapping/task.h \ core/mapping/task.inl \ + core/partitioning/constraint.h \ core/runtime/context.h \ core/runtime/operation.h \ core/runtime/runtime.h \ diff --git a/src/core/partitioning/constraint.cc b/src/core/partitioning/constraint.cc new file mode 100644 index 0000000000..c8afd218f5 --- /dev/null +++ b/src/core/partitioning/constraint.cc @@ -0,0 +1,80 @@ +/* Copyright 2021 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "core/data/scalar.h" +#include "core/partitioning/constraint.h" +#include "core/partitioning/partition.h" +#include "core/runtime/operation.h" + +namespace legate { + +Literal::Literal(const std::shared_ptr& partition) : partition_(partition) {} + +std::string Literal::to_string() const { return partition_->to_string(); } + +Variable::Variable(const Operation* op, std::shared_ptr store, int32_t id) + : op_(op), id_(id) +{ +} + +bool operator<(const Variable& lhs, const Variable& rhs) +{ + if (lhs.op_ > rhs.op_) + return false; + else if (lhs.op_ < rhs.op_) + return true; + return lhs.id_ < rhs.id_; +} + +std::string Variable::to_string() const +{ + std::stringstream ss; + ss << "X" << id_ << "{" << op_->to_string() << "}"; + return ss.str(); +} + +struct Alignment : public Constraint { + public: + Alignment(std::shared_ptr lhs, std::shared_ptr rhs); + + public: + virtual std::string to_string() const override; + + private: + std::shared_ptr lhs_; + std::shared_ptr rhs_; +}; + +Alignment::Alignment(std::shared_ptr lhs, std::shared_ptr rhs) + : lhs_(std::move(lhs)), rhs_(std::move(rhs)) +{ +} + +std::string Alignment::to_string() const +{ + std::stringstream ss; + ss << "Align(" << lhs_->to_string() << ", " << rhs_->to_string() << ")"; + return ss.str(); +} + +std::shared_ptr align(std::shared_ptr lhs, std::shared_ptr rhs) +{ + return std::make_shared(std::move(lhs), std::move(rhs)); +} + +} // namespace legate diff --git a/src/core/partitioning/constraint.h b/src/core/partitioning/constraint.h new file mode 100644 index 0000000000..a7dc160567 --- /dev/null +++ b/src/core/partitioning/constraint.h @@ -0,0 +1,105 @@ +/* Copyright 2021 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include +#include +#include + +namespace legate { + +struct Constraint; +struct Literal; +struct LogicalStore; +struct Operation; +struct Partition; +struct Variable; + +struct Expr { + public: + virtual ~Expr() {} + + public: + virtual bool closed() const = 0; + virtual std::string to_string() const = 0; + + public: + virtual const Literal* as_literal() const = 0; + virtual const Variable* as_variable() const = 0; +}; + +struct Literal : public Expr { + public: + Literal(const std::shared_ptr& partition); + + public: + Literal(const Literal&) = default; + Literal& operator=(const Literal&) = default; + + public: + virtual bool closed() const override { return true; } + virtual std::string to_string() const override; + + public: + virtual const Literal* as_literal() const override { return this; } + virtual const Variable* as_variable() const override { return nullptr; } + + public: + std::shared_ptr partition() const { return partition_; } + + private: + std::shared_ptr partition_; +}; + +struct Variable : public Expr { + public: + Variable(const Operation* op, std::shared_ptr store, int32_t id); + + public: + Variable(const Variable&) = default; + Variable& operator=(const Variable&) = default; + + public: + friend bool operator<(const Variable& lhs, const Variable& rhs); + + public: + virtual const Literal* as_literal() const override { return nullptr; } + virtual const Variable* as_variable() const override { return this; } + + public: + virtual bool closed() const override { return false; } + virtual std::string to_string() const override; + + public: + const Operation* operation() const { return op_; } + + private: + const Operation* op_; + int32_t id_; +}; + +struct Constraint { + public: + virtual ~Constraint() {} + + public: + virtual std::string to_string() const = 0; +}; + +std::shared_ptr align(std::shared_ptr lhs, std::shared_ptr rhs); + +} // namespace legate diff --git a/src/core/partitioning/constraint_graph.cc b/src/core/partitioning/constraint_graph.cc new file mode 100644 index 0000000000..4efd2b3c6d --- /dev/null +++ b/src/core/partitioning/constraint_graph.cc @@ -0,0 +1,62 @@ +/* Copyright 2021 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "legion.h" + +#include "core/partitioning/constraint.h" +#include "core/partitioning/constraint_graph.h" + +namespace legate { + +extern Legion::Logger log_legate; + +void ConstraintGraph::add_variable(std::shared_ptr variable) +{ + variables_.push_back(std::move(variable)); +} + +void ConstraintGraph::add_constraint(std::shared_ptr constraint) +{ + constraints_.push_back(std::move(constraint)); +} + +void ConstraintGraph::join(const ConstraintGraph& other) +{ + auto& other_variables = other.variables(); + auto& other_constraints = other.constraints(); + + for (auto& other_variable : other_variables) variables_.push_back(other_variable); + for (auto& other_constraint : other_constraints) constraints_.push_back(other_constraint); +} + +void ConstraintGraph::dump() +{ + for (auto& constraint : constraints_) log_legate.debug("%s", constraint->to_string().c_str()); +} + +const std::vector>& ConstraintGraph::variables() const +{ + return variables_; +} + +const std::vector>& ConstraintGraph::constraints() const +{ + return constraints_; +} + +} // namespace legate diff --git a/src/core/partitioning/constraint_graph.h b/src/core/partitioning/constraint_graph.h new file mode 100644 index 0000000000..16c9b1e6a4 --- /dev/null +++ b/src/core/partitioning/constraint_graph.h @@ -0,0 +1,48 @@ +/* Copyright 2021 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include +#include +#include + +namespace legate { + +struct Constraint; +struct Variable; + +struct ConstraintGraph { + public: + void add_variable(std::shared_ptr variable); + void add_constraint(std::shared_ptr constraint); + + public: + void join(const ConstraintGraph& other); + + public: + void dump(); + + public: + const std::vector>& variables() const; + const std::vector>& constraints() const; + + private: + std::vector> variables_; + std::vector> constraints_; +}; + +} // namespace legate diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index b3271e421c..f97c534b05 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -14,8 +14,10 @@ * */ -#include "core/partitioning/partition.h" +#include + #include "core/data/logical_store.h" +#include "core/partitioning/partition.h" #include "core/runtime/launcher.h" #include "core/runtime/runtime.h" @@ -23,6 +25,15 @@ namespace legate { using Shape = std::vector; +std::string to_string(const Shape& shape) +{ + std::stringstream ss; + ss << "("; + for (auto extent : shape) ss << extent << ","; + ss << ")"; + return ss.str(); +} + class NoPartition : public Partition { public: NoPartition(Runtime* runtime); @@ -41,6 +52,9 @@ class NoPartition : public Partition { virtual bool has_launch_domain() const override; virtual Legion::Domain launch_domain() const override; + public: + virtual std::string to_string() const override; + private: Runtime* runtime_; }; @@ -63,6 +77,9 @@ class Tiling : public Partition { virtual bool has_launch_domain() const override; virtual Legion::Domain launch_domain() const override; + public: + virtual std::string to_string() const override; + private: Runtime* runtime_; Shape tile_shape_; @@ -113,7 +130,7 @@ Legion::LogicalPartition NoPartition::construct(const LogicalStore* store, std::unique_ptr NoPartition::get_projection(LogicalStore* store) const { - return std::make_unique(); + return std::make_unique(); } bool NoPartition::has_launch_domain() const { return false; } @@ -124,6 +141,8 @@ Legion::Domain NoPartition::launch_domain() const return Legion::Domain(); } +std::string NoPartition::to_string() const { return "NoPartition"; } + bool Tiling::is_complete_for(const LogicalStore* store) const { return false; } bool Tiling::is_disjoint_for(const LogicalStore* store) const { return true; } @@ -185,6 +204,15 @@ Legion::Domain Tiling::launch_domain() const return launch_domain; } +std::string Tiling::to_string() const +{ + std::stringstream ss; + ss << "Tiling(tile:" << legate::to_string(tile_shape_) + << ",colors:" << legate::to_string(color_shape_) << ",offset:" << legate::to_string(offsets_) + << ")"; + return ss.str(); +} + PartitionByRestriction::PartitionByRestriction(Legion::DomainTransform transform, Legion::Domain extent) : transform_(transform), extent_(extent) diff --git a/src/core/partitioning/partition.h b/src/core/partitioning/partition.h index 2b797186d8..fba8267046 100644 --- a/src/core/partitioning/partition.h +++ b/src/core/partitioning/partition.h @@ -27,6 +27,9 @@ class Projection; class Runtime; struct Partition { + public: + virtual ~Partition() {} + public: virtual bool is_complete_for(const LogicalStore* store) const = 0; virtual bool is_disjoint_for(const LogicalStore* store) const = 0; @@ -40,6 +43,9 @@ struct Partition { public: virtual bool has_launch_domain() const = 0; virtual Legion::Domain launch_domain() const = 0; + + public: + virtual std::string to_string() const = 0; }; struct PartitioningFunctor { diff --git a/src/core/partitioning/partitioner.cc b/src/core/partitioning/partitioner.cc index 72ec287224..15daee8a3a 100644 --- a/src/core/partitioning/partitioner.cc +++ b/src/core/partitioning/partitioner.cc @@ -17,6 +17,8 @@ #include "core/partitioning/partitioner.h" #include "core/data/logical_store.h" #include "core/data/scalar.h" +#include "core/partitioning/constraint.h" +#include "core/partitioning/constraint_graph.h" #include "core/partitioning/partition.h" #include "core/runtime/launcher.h" #include "core/runtime/operation.h" @@ -29,61 +31,87 @@ Strategy::Strategy() {} bool Strategy::parallel(const Operation* op) const { auto finder = launch_domains_.find(op); - return finder != launch_domains_.end(); + return (launch_domains_.end() != finder) && (nullptr != finder->second); +} + +bool Strategy::has_launch_domain(const Operation* op) const +{ + return launch_domains_.find(op) != launch_domains_.end(); } Legion::Domain Strategy::launch_domain(const Operation* op) const { auto finder = launch_domains_.find(op); assert(finder != launch_domains_.end()); - return finder->second; + return *finder->second; } +void Strategy::set_single_launch(const Operation* op) { launch_domains_[op] = nullptr; } + void Strategy::set_launch_domain(const Operation* op, const Legion::Domain& launch_domain) { - launch_domains_[op] = launch_domain; + launch_domains_[op] = std::make_unique(launch_domain); } -void Strategy::insert(const LogicalStore* store, std::shared_ptr partition) +void Strategy::insert(const Expr* variable, std::shared_ptr partition) { - assert(assignments_.find(store) == assignments_.end()); - assignments_[store] = std::move(partition); + assert(assignments_.find(variable) == assignments_.end()); + assignments_[variable] = std::move(partition); } -std::shared_ptr Strategy::find(const LogicalStore* store) const +std::shared_ptr Strategy::operator[](const std::shared_ptr& variable) const { - auto finder = assignments_.find(store); + auto finder = assignments_.find(variable.get()); assert(finder != assignments_.end()); return finder->second; } -std::unique_ptr Strategy::get_projection(LogicalStore* store) const -{ - auto partition = find(store); - return partition->get_projection(store); -} - Partitioner::Partitioner(Runtime* runtime, std::vector&& operations) : runtime_(runtime), operations_(std::forward>(operations)) { } -std::unique_ptr Partitioner::partition_stores() +std::unique_ptr Partitioner::solve() { - auto strategy = std::make_unique(); - - for (auto op : operations_) { - bool determined_launch_domain = false; - auto all_stores = op->all_stores(); - for (auto store : all_stores) { - auto key_partition = store->find_or_create_key_partition(); - if (!determined_launch_domain) { - determined_launch_domain = true; - if (key_partition->has_launch_domain()) - strategy->set_launch_domain(op, key_partition->launch_domain()); - } - strategy->insert(store, std::move(key_partition)); + ConstraintGraph constraints; + + for (auto op : operations_) constraints.join(*op->constraints()); + + constraints.dump(); + + // We need to find a mapping from every partition variable to a concrete partition + // Substitution mapping; + // for (auto& expr : all_symbols) { + // auto* var = expr->as_variable(); + // assert(nullptr != sym); + // if (mapping.find(var) != mapping.end()) continue; + + // auto store = store_mapping[expr]; + // auto partition = store->find_or_create_key_partition(); + // mapping[sym] = std::move(partition); + + // std::vector> next_constraints; + // for (auto& constraint : all_constraints) { + // auto substituted = constraint.subst(mapping); + // bool resolved = substituted.resolve(mapping); + // if (!resolved) next_constraints.push_back(std::move(substituted)); + // } + //} + + auto strategy = std::make_unique(); + auto& variables = constraints.variables(); + + for (auto& variable : variables) { + auto* op = variable->operation(); + auto store = op->find_store(variable); + auto partition = store->find_or_create_key_partition(); + if (!strategy->has_launch_domain(op)) { + if (partition->has_launch_domain()) + strategy->set_launch_domain(op, partition->launch_domain()); + else + strategy->set_single_launch(op); } + strategy->insert(variable.get(), std::move(partition)); } return std::move(strategy); diff --git a/src/core/partitioning/partitioner.h b/src/core/partitioning/partitioner.h index b317ad4628..1d152ec407 100644 --- a/src/core/partitioning/partitioner.h +++ b/src/core/partitioning/partitioner.h @@ -23,6 +23,7 @@ namespace legate { +class Expr; class LogicalStore; class Operation; class Partition; @@ -35,19 +36,18 @@ class Strategy { public: bool parallel(const Operation* op) const; + bool has_launch_domain(const Operation* op) const; Legion::Domain launch_domain(const Operation* op) const; + void set_single_launch(const Operation* op); void set_launch_domain(const Operation* op, const Legion::Domain& launch_domain); public: - void insert(const LogicalStore* store, std::shared_ptr partition); - std::shared_ptr find(const LogicalStore* store) const; - - public: - std::unique_ptr get_projection(LogicalStore* store) const; + void insert(const Expr* variable, std::shared_ptr partition); + std::shared_ptr operator[](const std::shared_ptr& variable) const; private: - std::unordered_map> assignments_; - std::unordered_map launch_domains_; + std::unordered_map> assignments_; + std::unordered_map> launch_domains_; }; class Partitioner { @@ -55,7 +55,7 @@ class Partitioner { Partitioner(Runtime* runtime, std::vector&& operations); public: - std::unique_ptr partition_stores(); + std::unique_ptr solve(); private: Runtime* runtime_; diff --git a/src/core/runtime/context.cc b/src/core/runtime/context.cc index 7a9586e442..35e835dc0b 100644 --- a/src/core/runtime/context.cc +++ b/src/core/runtime/context.cc @@ -139,6 +139,18 @@ bool LibraryContext::valid_sharding_id(Legion::ShardingID shard_id) const return shard_scope_.in_scope(shard_id); } +void LibraryContext::record_task_name(int64_t local_task_id, const std::string& task_name) +{ + task_names_[local_task_id] = task_name; +} + +const std::string& LibraryContext::get_task_name(int64_t local_task_id) const +{ + auto finder = task_names_.find(local_task_id); + assert(task_names_.end() != finder); + return finder->second; +} + TaskContext::TaskContext(const Legion::Task* task, const std::vector& regions, Legion::Context context, diff --git a/src/core/runtime/context.h b/src/core/runtime/context.h index 8b7a6ce0a2..1b2c6b7881 100644 --- a/src/core/runtime/context.h +++ b/src/core/runtime/context.h @@ -16,6 +16,8 @@ #pragma once +#include + #include "legion.h" #include "core/task/return.h" @@ -94,6 +96,10 @@ class LibraryContext { bool valid_projection_id(Legion::ProjectionID proj_id) const; bool valid_sharding_id(Legion::ShardingID shard_id) const; + public: + void record_task_name(int64_t local_task_id, const std::string& task_name); + const std::string& get_task_name(int64_t local_task_id) const; + private: const std::string library_name_; ResourceScope task_scope_; @@ -101,6 +107,7 @@ class LibraryContext { ResourceScope redop_scope_; ResourceScope proj_scope_; ResourceScope shard_scope_; + std::unordered_map task_names_; }; // A thin context layer on top of the Legion runtime, primarily designed to hide verbosity diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/launcher.cc index f5fa874aef..30d31192ac 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -207,11 +207,11 @@ void Projection::set_reduction_op(Legion::ReductionOpID r) redop = std::make_unique(r); } -Broadcast::Broadcast() : Projection() {} +Replicate::Replicate() : Projection() {} -Broadcast::Broadcast(Legion::ReductionOpID redop) : Projection(redop) {} +Replicate::Replicate(Legion::ReductionOpID redop) : Projection(redop) {} -void Broadcast::populate_launcher(Legion::TaskLauncher* task, +void Replicate::populate_launcher(Legion::TaskLauncher* task, const RegionReq& req, const std::vector& fields) const { @@ -226,7 +226,7 @@ void Broadcast::populate_launcher(Legion::TaskLauncher* task, } } -void Broadcast::populate_launcher(Legion::IndexTaskLauncher* task, +void Replicate::populate_launcher(Legion::IndexTaskLauncher* task, const RegionReq& req, const std::vector& fields) const { diff --git a/src/core/runtime/launcher.h b/src/core/runtime/launcher.h index 9bb20889e1..12b4f4d1ef 100644 --- a/src/core/runtime/launcher.h +++ b/src/core/runtime/launcher.h @@ -54,13 +54,13 @@ class Projection { std::unique_ptr redop{nullptr}; }; -class Broadcast : public Projection { +class Replicate : public Projection { public: - Broadcast(); - Broadcast(Legion::ReductionOpID redop); + Replicate(); + Replicate(Legion::ReductionOpID redop); public: - virtual ~Broadcast() {} + virtual ~Replicate() {} public: virtual void populate_launcher(Legion::TaskLauncher* task, diff --git a/src/core/runtime/operation.cc b/src/core/runtime/operation.cc index 2ef1099093..f024c1d3d2 100644 --- a/src/core/runtime/operation.cc +++ b/src/core/runtime/operation.cc @@ -14,10 +14,14 @@ * */ +#include #include #include "core/data/logical_store.h" #include "core/data/scalar.h" +#include "core/partitioning/constraint.h" +#include "core/partitioning/constraint_graph.h" +#include "core/partitioning/partition.h" #include "core/partitioning/partitioner.h" #include "core/runtime/context.h" #include "core/runtime/launcher.h" @@ -26,73 +30,110 @@ namespace legate { -Operation::Operation(Runtime* runtime, LibraryContext* library, int64_t mapper_id) - : runtime_(runtime), library_(library), mapper_id_(mapper_id) +Operation::Operation(Runtime* runtime, + LibraryContext* library, + uint64_t unique_id, + int64_t mapper_id) + : runtime_(runtime), + library_(library), + unique_id_(unique_id), + mapper_id_(mapper_id), + constraints_(std::make_shared()) { } -void Operation::add_input(LogicalStoreP store) { inputs_.push_back(store); } +void Operation::add_input(std::shared_ptr store, std::shared_ptr partition) +{ + constraints_->add_variable(partition); + inputs_.push_back(Store(std::move(store), std::move(partition))); +} + +void Operation::add_output(std::shared_ptr store, std::shared_ptr partition) +{ + constraints_->add_variable(partition); + outputs_.push_back(Store(std::move(store), std::move(partition))); +} -void Operation::add_output(LogicalStoreP store) { outputs_.push_back(store); } +void Operation::add_reduction(std::shared_ptr store, + Legion::ReductionOpID redop, + std::shared_ptr partition) +{ + constraints_->add_variable(partition); + reductions_.push_back(Store(std::move(store), std::move(partition))); + reduction_ops_.push_back(redop); +} -void Operation::add_reduction(LogicalStoreP store, Legion::ReductionOpID redop) +std::shared_ptr Operation::declare_partition(std::shared_ptr store) { - reductions_.push_back(Reduction(store, redop)); + auto variable = std::make_shared(this, store, next_part_id_++); + store_mappings_[variable] = std::move(store); + return std::move(variable); } -std::vector Operation::all_stores() +std::shared_ptr Operation::find_store(std::shared_ptr variable) const { - std::vector result; - std::unordered_set added; - - auto add_all = [&](auto& stores) { - for (auto& store : stores) { - auto p_store = store.get(); - if (added.find(p_store) == added.end()) { - result.push_back(p_store); - added.insert(p_store); - } - } - }; - - add_all(inputs_); - add_all(outputs_); - for (auto& reduction : reductions_) { - auto& store = reduction.first; - auto p_store = store.get(); - if (added.find(p_store) == added.end()) { - result.push_back(p_store); - added.insert(p_store); - } - } + auto finder = store_mappings_.find(variable); + assert(store_mappings_.end() != finder); + return finder->second; +} - return std::move(result); +void Operation::add_constraint(std::shared_ptr constraint) +{ + constraints_->add_constraint(constraint); } -Task::Task(Runtime* runtime, LibraryContext* library, int64_t task_id, int64_t mapper_id /*=0*/) - : Operation(runtime, library, mapper_id), task_id_(task_id) +std::shared_ptr Operation::constraints() const { return constraints_; } + +Task::Task(Runtime* runtime, + LibraryContext* library, + int64_t task_id, + uint64_t unique_id, + int64_t mapper_id /*=0*/) + : Operation(runtime, library, unique_id, mapper_id), task_id_(task_id) { } void Task::add_scalar_arg(const Scalar& scalar) { scalars_.push_back(scalar); } -void Task::launch(Strategy* strategy) const +void Task::launch(Strategy* p_strategy) const { + auto& strategy = *p_strategy; TaskLauncher launcher(runtime_, library_, task_id_, mapper_id_); - for (auto& input : inputs_) launcher.add_input(input, strategy->get_projection(input.get())); - for (auto& output : outputs_) launcher.add_output(output, strategy->get_projection(output.get())); + for (auto& pair : inputs_) { + auto& store = pair.first; + auto& var = pair.second; + auto proj = strategy[var]->get_projection(store.get()); + launcher.add_input(store, std::move(proj)); + } + for (auto& pair : outputs_) { + auto& store = pair.first; + auto& var = pair.second; + auto proj = strategy[var]->get_projection(store.get()); + launcher.add_output(store, std::move(proj)); + } + uint32_t idx = 0; for (auto& pair : reductions_) { - auto projection = strategy->get_projection(pair.first.get()); - projection->set_reduction_op(pair.second); - launcher.add_reduction(pair.first, std::move(projection)); + auto& store = pair.first; + auto& var = pair.second; + auto proj = strategy[var]->get_projection(store.get()); + auto redop = reduction_ops_[idx++]; + proj->set_reduction_op(redop); + launcher.add_reduction(store, std::move(proj)); } for (auto& scalar : scalars_) launcher.add_scalar(scalar); - if (strategy->parallel(this)) - launcher.execute(strategy->launch_domain(this)); + if (strategy.parallel(this)) + launcher.execute(strategy.launch_domain(this)); else launcher.execute_single(); } +std::string Task::to_string() const +{ + std::stringstream ss; + ss << library_->get_task_name(task_id_) << ":" << unique_id_; + return ss.str(); +} + } // namespace legate diff --git a/src/core/runtime/operation.h b/src/core/runtime/operation.h index 20ed208e5b..d79653acda 100644 --- a/src/core/runtime/operation.h +++ b/src/core/runtime/operation.h @@ -17,50 +17,74 @@ #pragma once #include +#include #include "legion.h" namespace legate { +class Constraint; +class ConstraintGraph; class LibraryContext; class LogicalStore; class Runtime; class Scalar; class Strategy; +class Variable; class Operation { - private: - using LogicalStoreP = std::shared_ptr; - using Reduction = std::pair; - public: - Operation(Runtime* runtime, LibraryContext* library, int64_t mapper_id); + Operation(Runtime* runtime, LibraryContext* library, uint64_t unique_id, int64_t mapper_id); public: - void add_input(LogicalStoreP store); - void add_output(LogicalStoreP store); - void add_reduction(LogicalStoreP store, Legion::ReductionOpID redop); + void add_input(std::shared_ptr store, std::shared_ptr partition); + void add_output(std::shared_ptr store, std::shared_ptr partition); + void add_reduction(std::shared_ptr store, + Legion::ReductionOpID redop, + std::shared_ptr partition); public: - std::vector all_stores(); + std::shared_ptr declare_partition(std::shared_ptr store); + std::shared_ptr find_store(std::shared_ptr variable) const; + void add_constraint(std::shared_ptr constraint); + std::shared_ptr constraints() const; public: virtual void launch(Strategy* strategy) const = 0; + public: + virtual std::string to_string() const = 0; + protected: Runtime* runtime_; LibraryContext* library_; + uint64_t unique_id_; int64_t mapper_id_; protected: - std::vector inputs_; - std::vector outputs_; - std::vector reductions_; + using Store = std::pair, std::shared_ptr>; + + protected: + std::vector inputs_{}; + std::vector outputs_{}; + std::vector reductions_{}; + std::vector reduction_ops_{}; + + private: + uint32_t next_part_id_{0}; + + private: + std::unordered_map, std::shared_ptr> store_mappings_; + std::shared_ptr constraints_; }; class Task : public Operation { public: - Task(Runtime* runtime, LibraryContext* library, int64_t task_id, int64_t mapper_id = 0); + Task(Runtime* runtime, + LibraryContext* library, + int64_t task_id, + uint64_t unique_id, + int64_t mapper_id = 0); public: void add_scalar_arg(const Scalar& scalar); @@ -68,6 +92,9 @@ class Task : public Operation { public: virtual void launch(Strategy* strategy) const override; + public: + virtual std::string to_string() const override; + private: int64_t task_id_; std::vector scalars_{}; diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 1c3ef6e2fc..faf8526153 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -607,7 +607,7 @@ std::unique_ptr Runtime::create_task(LibraryContext* library, int64_t task_id, int64_t mapper_id /*=0*/) { - return std::make_unique(this, library, task_id, mapper_id); + return std::make_unique(this, library, task_id, next_unique_id_++, mapper_id); } void Runtime::submit(std::unique_ptr op) @@ -627,7 +627,7 @@ void Runtime::schedule(std::vector> operations) for (auto& op : operations) op_pointers.push_back(op.get()); Partitioner partitioner(this, std::move(op_pointers)); - auto strategy = partitioner.partition_stores(); + auto strategy = partitioner.solve(); for (auto& op : operations) op->launch(strategy.get()); } diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 03dc781ec5..91e589d01e 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -155,6 +155,7 @@ class Runtime { private: std::vector> operations_; size_t window_size_{1}; + uint64_t next_unique_id_{0}; private: std::map libraries_; diff --git a/src/core/task/task.cc b/src/core/task/task.cc index 4a8a6d7231..7b44830ab5 100644 --- a/src/core/task/task.cc +++ b/src/core/task/task.cc @@ -58,6 +58,7 @@ void LegateTaskRegistrar::register_all_tasks(Legion::Runtime* runtime, LibraryCo { // Do all our registrations for (auto& task : pending_task_variants_) { + context.record_task_name(task.task_id, task.task_name); task.task_id = context.get_task_id(task.task_id); // Convert a task local task id to a global id // Attach the task name too for debugging diff --git a/src/legate.h b/src/legate.h index d1a2b5e62a..9f597b2978 100644 --- a/src/legate.h +++ b/src/legate.h @@ -22,6 +22,7 @@ #include "core/data/scalar.h" #include "core/data/store.h" #include "core/legate_c.h" +#include "core/partitioning/constraint.h" #include "core/runtime/operation.h" #include "core/runtime/runtime.h" #include "core/task/task.h" From ddd8dc77455f8f1e16f79014b39331df66e368a3 Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Wed, 27 Oct 2021 14:18:39 -0700 Subject: [PATCH 0015/1425] Change the pip package name to match the conda package and update version. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 749f42373b..ac6f7dc02c 100755 --- a/setup.py +++ b/setup.py @@ -69,8 +69,8 @@ def run(self): # Remove the recurse argument from the list sys.argv.remove("--recurse") setup( - name="legate.core", - version="0.1", + name="legate-core", + version="21.10.00", packages=["legate", "legate.core", "legate.timing"], cmdclass={"build_py": my_build_py}, ) From ef677e6448b11b76d8b91e8ebd17652dc8d992a6 Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Wed, 27 Oct 2021 23:46:45 -0700 Subject: [PATCH 0016/1425] Fix the version of Legion to a particular commit --- install.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/install.py b/install.py index 8f9a32db88..0b3ca61b5a 100755 --- a/install.py +++ b/install.py @@ -142,6 +142,7 @@ def git_clone(repo_dir, url, branch=None, tag=None, commit=None): verbose_check_call( ["git", "submodule", "update", "--init"], cwd=repo_dir ) + git_reset(repo_dir, commit) else: verbose_check_call( [ @@ -162,10 +163,13 @@ def git_reset(repo_dir, refspec): verbose_check_call(["git", "reset", "--hard", refspec], cwd=repo_dir) -def git_update(repo_dir, branch=None): - verbose_check_call(["git", "pull", "--ff-only"], cwd=repo_dir) +def git_update(repo_dir, branch=None, tag=None, commit=None): if branch is not None: verbose_check_call(["git", "checkout", branch], cwd=repo_dir) + verbose_check_call(["git", "pull", "--ff-only"], cwd=repo_dir) + else: + verbose_check_call(["git", "fetch"], cwd=repo_dir) + verbose_check_call(["git", "checkout", commit or tag], cwd=repo_dir) def load_json_config(filename): @@ -209,13 +213,14 @@ def install_gasnet(gasnet_dir, conduit, thread_count): shutil.rmtree(temp_dir) -def install_legion(legion_src_dir, branch="legate_stable"): +def install_legion(legion_src_dir, branch, commit="3141d7c0"): print("Legate is installing Legion into a local directory...") # For now all we have to do is clone legion since we build it with Legate git_clone( legion_src_dir, url="https://gitlab.com/StanfordLegion/legion.git", branch=branch, + commit=commit ) @@ -228,9 +233,9 @@ def install_thrust(thrust_dir): ) -def update_legion(legion_src_dir, branch="legate_stable"): +def update_legion(legion_src_dir, branch, commit="3141d7c0"): # Make sure we are on the right branch for single/multi-node - git_update(legion_src_dir, branch=branch) + git_update(legion_src_dir, branch=branch, commit=commit) def build_legion( From ba955e280ef575bcb93181bc5ce22787f68625f3 Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Thu, 28 Oct 2021 00:24:15 -0700 Subject: [PATCH 0017/1425] Do not find a default branch for the release --- install.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/install.py b/install.py index 0b3ca61b5a..f6ddbb75cc 100755 --- a/install.py +++ b/install.py @@ -220,7 +220,7 @@ def install_legion(legion_src_dir, branch, commit="3141d7c0"): legion_src_dir, url="https://gitlab.com/StanfordLegion/legion.git", branch=branch, - commit=commit + commit=commit, ) @@ -562,8 +562,10 @@ def install( legate_core_dir = os.path.dirname(os.path.realpath(__file__)) - if legion_branch is None: - legion_branch = find_default_legion_branch(legate_core_dir) + # For the release, we will use a hardcoded commit unless user asks for + # a branch + # if legion_branch is None: + # legion_branch = find_default_legion_branch(legate_core_dir) cmake_config = os.path.join(legate_core_dir, ".cmake.json") dump_json_config(cmake_config, cmake) From de337cfe033d5c332dda6bf8ac69b63956fcb2b4 Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Thu, 28 Oct 2021 01:03:19 -0700 Subject: [PATCH 0018/1425] Change the Legion checkout target --- install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install.py b/install.py index f6ddbb75cc..d18a44a42c 100755 --- a/install.py +++ b/install.py @@ -213,7 +213,7 @@ def install_gasnet(gasnet_dir, conduit, thread_count): shutil.rmtree(temp_dir) -def install_legion(legion_src_dir, branch, commit="3141d7c0"): +def install_legion(legion_src_dir, branch, commit="d0907f4c"): print("Legate is installing Legion into a local directory...") # For now all we have to do is clone legion since we build it with Legate git_clone( @@ -233,7 +233,7 @@ def install_thrust(thrust_dir): ) -def update_legion(legion_src_dir, branch, commit="3141d7c0"): +def update_legion(legion_src_dir, branch, commit="d0907f4c"): # Make sure we are on the right branch for single/multi-node git_update(legion_src_dir, branch=branch, commit=commit) From 78335cc7682b961910119d2b899d6e49328db3e3 Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Thu, 28 Oct 2021 09:13:48 -0700 Subject: [PATCH 0019/1425] Bumped up the version of pyarrow --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 80f3fbd55a..d72135b2e2 100644 --- a/README.md +++ b/README.md @@ -221,7 +221,7 @@ no support for Windows. The Legate Core currently requires Python >= 3.7 and the following packages: - - `pyarrow=1.0.1` + - `pyarrow=5.0.0` - `numpy` - `cffi` - [CUDA](https://developer.nvidia.com/cuda-downloads) >= 8.0 From ab0c0448474058bdd71e525a261affa049a2b3d8 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Mon, 1 Nov 2021 11:35:03 -0700 Subject: [PATCH 0020/1425] Remove back edges from partition symbols back to operations to avoid object cycles --- legate/core/constraints.py | 13 +++++++++---- legate/core/operation.py | 3 ++- legate/core/solver.py | 18 +++++++++--------- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/legate/core/constraints.py b/legate/core/constraints.py index cd81306e56..39e85bdcc9 100644 --- a/legate/core/constraints.py +++ b/legate/core/constraints.py @@ -55,8 +55,9 @@ def reduce(self): class PartSym(Expr): - def __init__(self, op, store, id, disjoint, complete): - self._op = op + def __init__(self, op_hash, op_name, store, id, disjoint, complete): + self._op_hash = op_hash + self._op_name = op_name self._store = store self._id = id self._disjoint = disjoint @@ -66,6 +67,10 @@ def __init__(self, op, store, id, disjoint, complete): def ndim(self): return self._store.ndim + @property + def store(self): + return self._store + @property def closed(self): return False @@ -73,10 +78,10 @@ def closed(self): def __repr__(self): disj = "D" if self._disjoint else "A" comp = "C" if self._complete else "I" - return f"X{self._id}({disj},{comp})@{self._op.get_name()}" + return f"X{self._id}({disj},{comp})@{self._op_name}" def __hash__(self): - return hash((self._op, self._id)) + return hash((self._op_hash, self._id)) def subst(self, mapping): return Lit(mapping[self]) diff --git a/legate/core/operation.py b/legate/core/operation.py index c2f13c5242..5e8283dd50 100644 --- a/legate/core/operation.py +++ b/legate/core/operation.py @@ -158,7 +158,8 @@ def _get_symbol_id(self): def declare_partition(self, store, disjoint=True, complete=True): sym = PartSym( - self, + self._op_id, + self.get_name(), store, self._get_symbol_id(), disjoint=disjoint, diff --git a/legate/core/solver.py b/legate/core/solver.py index 58321c00fb..b4d5c5e80c 100644 --- a/legate/core/solver.py +++ b/legate/core/solver.py @@ -114,16 +114,16 @@ def launch_domain(self): def get_projection(self, part): partition = self.get_partition(part) - return partition.get_requirement(self._launch_shape, part._store) + return partition.get_requirement(self._launch_shape, part.store) def get_partition(self, part): - assert not part._store.unbound + assert not part.store.unbound if part not in self._strategy: raise ValueError(f"No strategy is found for {part}") return self._strategy[part] def get_field_space(self, part): - assert part._store.unbound + assert part.store.unbound if part not in self._fspaces: raise ValueError(f"No strategy is found for {part}") return self._fspaces[part] @@ -160,7 +160,7 @@ def _solve_broadcast_constraints( ): to_remove = OrderedSet() for unknown in unknowns: - store = unknown._store + store = unknown.store if not (store.kind is Future or unknown in broadcasts): continue @@ -183,7 +183,7 @@ def _solve_unbound_constraints( ): to_remove = OrderedSet() for unknown in unknowns: - store = unknown._store + store = unknown.store if not store.unbound: continue @@ -193,7 +193,7 @@ def _solve_unbound_constraints( continue cls = constraints.find(unknown) - assert all(to_align._store.unbound for to_align in cls) + assert all(to_align.store.unbound for to_align in cls) fspace = self._runtime.create_field_space() for to_align in cls: @@ -206,7 +206,7 @@ def _solve_unbound_constraints( def _find_restrictions(cls): merged = None for unknown in cls: - store = unknown._store + store = unknown.store restrictions = store.find_restrictions() if merged is None: merged = restrictions @@ -268,7 +268,7 @@ def partition_stores(self): all_restrictions = self._find_all_restrictions(unknowns, constraints) def cost(unknown): - store = unknown._store + store = unknown.store return ( -store.comm_volume(), not store.has_key_partition(all_restrictions[unknown]), @@ -284,7 +284,7 @@ def cost(unknown): elif unknown in dependent: continue - store = unknown._store + store = unknown.store restrictions = all_restrictions[unknown] if isinstance(prev_part, NoPartition): From 54d3bb8fc20baf186246bd7b97271f1a01142459 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Mon, 1 Nov 2021 15:39:06 -0700 Subject: [PATCH 0021/1425] Make sure we don't create cycles between region fields and attachments --- legate/core/runtime.py | 60 +++++++++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/legate/core/runtime.py b/legate/core/runtime.py index 6bd9a58393..c06de76e12 100644 --- a/legate/core/runtime.py +++ b/legate/core/runtime.py @@ -16,6 +16,7 @@ import gc import math import struct +import weakref from collections import deque from functools import reduce @@ -319,11 +320,19 @@ def __init__(self, ptr, extent, region_field): self.ptr = ptr self.extent = extent self.end = ptr + extent - 1 - self.region_field = region_field + self._region_field = weakref.ref(region_field) def overlaps(self, other): return not (self.end < other.ptr or other.end < self.ptr) + @property + def region_field(self): + return self._region_field() + + @region_field.setter + def region_field(self, region_field): + self._region_field = weakref.ref(region_field) + class AttachmentManager(object): def __init__(self, runtime): @@ -359,22 +368,35 @@ def attachment_key(alloc): def has_attachment(self, alloc): key = self.attachment_key(alloc) - return key in self._attachments + attachment = self._attachments.get(key, None) + return attachment is not None and attachment.region_field def reuse_existing_attachment(self, alloc): key = self.attachment_key(alloc) - if key not in self._attachments: + attachment = self._attachments.get(key, None) + if attachment is None: return None - attachment = self._attachments[key] - return attachment.region_field + rf = attachment.region_field + # If the region field is already collected, we don't need to keep + # track of it for de-duplication. + if rf is None: + del self._attachments[key] + return rf def attach_external_allocation(self, alloc, region_field): key = self.attachment_key(alloc) - if key in self._attachments: + attachment = self._attachments.get(key, None) + if not (attachment is None or attachment.region_field is None): raise RuntimeError( "Cannot attach two different RegionFields to the same buffer" ) - attachment = Attachment(*key, region_field) + if attachment is None: + attachment = Attachment(*key, region_field) + else: + attachment.region_field = region_field + # We temporary remove the attachment from the map for + # the following alias checking + del self._attachments[key] for other in self._attachments.values(): if other.overlaps(attachment): raise RuntimeError( @@ -382,7 +404,19 @@ def attach_external_allocation(self, alloc, region_field): ) self._attachments[key] = attachment - def detach_external_allocation(self, alloc, detach, defer): + def _remove_allocation(self, alloc): + key = self.attachment_key(alloc) + if key not in self._attachments: + raise RuntimeError("Unable to find attachment to remove") + del self._attachments[key] + + def detach_external_allocation( + self, alloc, detach, defer=False, previously_deferred=False + ): + # If the detachment was previously deferred, then we don't + # need to remove the allocation from the map again. + if not previously_deferred: + self._remove_allocation(alloc) if defer: # If we need to defer this until later do that now self._deferred_detachments.append((alloc, detach)) @@ -391,12 +425,6 @@ def detach_external_allocation(self, alloc, detach, defer): # Dangle a reference to the field off the future to prevent the # field from being recycled until the detach is done future.field_reference = detach.field - # We also need to tell the core legate library that this buffer - # is no longer attached - key = self.attachment_key(alloc) - if key not in self._attachments: - raise RuntimeError("Unable to find attachment to remove") - del self._attachments[key] # If the future is already ready, then no need to track it if future.is_ready(): return @@ -417,7 +445,9 @@ def perform_detachments(self): detachments = self._deferred_detachments self._deferred_detachments = list() for alloc, detach in detachments: - self.detach_external_allocation(alloc, detach, defer=False) + self.detach_external_allocation( + alloc, detach, defer=False, previously_deferred=True + ) def prune_detachments(self): to_remove = [] From 2e1affd0b2f3c2aa71a18cd4ad353dd0ce1a2bdb Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 3 Nov 2021 02:01:23 -0700 Subject: [PATCH 0022/1425] Minimum shard volume is now a 64-bit value --- src/core/runtime/runtime.cc | 2 +- src/core/runtime/runtime.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index faf8526153..7a75511577 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -395,7 +395,7 @@ std::shared_ptr FieldManager::allocate_field() PartitionManager::PartitionManager(Runtime* runtime, const LibraryContext* context) { num_pieces_ = runtime->get_tunable(context, LEGATE_CORE_TUNABLE_NUM_PIECES); - min_shard_volume_ = runtime->get_tunable(context, LEGATE_CORE_TUNABLE_MIN_SHARD_VOLUME); + min_shard_volume_ = runtime->get_tunable(context, LEGATE_CORE_TUNABLE_MIN_SHARD_VOLUME); assert(num_pieces_ > 0); assert(min_shard_volume_ > 0); diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 91e589d01e..3d4631b00d 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -66,7 +66,7 @@ class PartitionManager { private: int32_t num_pieces_; - int32_t min_shard_volume_; + int64_t min_shard_volume_; std::vector piece_factors_; }; From d6ccdd2747d9f2bb72de21658f0a7906810991aa Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 3 Nov 2021 15:25:17 -0700 Subject: [PATCH 0023/1425] Handle cases where one instance is used by multiple mappings --- src/core/mapping/base_mapper.cc | 46 ++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/src/core/mapping/base_mapper.cc b/src/core/mapping/base_mapper.cc index fd4cfa04c6..9f679de9ec 100644 --- a/src/core/mapping/base_mapper.cc +++ b/src/core/mapping/base_mapper.cc @@ -429,7 +429,7 @@ void BaseMapper::map_task(const MapperContext ctx, // Map each field separately for each of the logical regions std::vector needed_acquires; - std::map instances_to_mappings; + std::map> instances_to_mappings; for (uint32_t mapping_idx = 0; mapping_idx < mappings.size(); ++mapping_idx) { auto& mapping = mappings[mapping_idx]; auto req_indices = mapping.requirement_indices(); @@ -457,7 +457,7 @@ void BaseMapper::map_task(const MapperContext ctx, needed_acquires.push_back(result); for (auto req_idx : req_indices) output.chosen_instances[req_idx].push_back(result); - instances_to_mappings[result] = mapping_idx; + instances_to_mappings[result].insert(mapping_idx); } // Do an acquire on all the instances so we have our result @@ -471,27 +471,31 @@ void BaseMapper::map_task(const MapperContext ctx, filter_failed_acquires(needed_acquires, failed_acquires); for (auto failed_acquire : failed_acquires) { - auto mapping_idx = instances_to_mappings[failed_acquire]; - auto& mapping = mappings[mapping_idx]; - auto req_indices = mapping.requirement_indices(); - - std::vector> reqs; - for (auto req_idx : req_indices) reqs.push_back(std::cref(task.regions[req_idx])); - - for (auto req_idx : req_indices) { - auto& instances = output.chosen_instances[req_idx]; - uint32_t inst_idx = 0; - for (; inst_idx < instances.size(); ++inst_idx) - if (instances[inst_idx] == failed_acquire) break; - instances.erase(instances.begin() + inst_idx); - } + auto affected_mappings = instances_to_mappings[failed_acquire]; + instances_to_mappings.erase(failed_acquire); + + for (auto& mapping_idx : affected_mappings) { + auto& mapping = mappings[mapping_idx]; + auto req_indices = mapping.requirement_indices(); + + std::vector> reqs; + for (auto req_idx : req_indices) reqs.push_back(std::cref(task.regions[req_idx])); + + for (auto req_idx : req_indices) { + auto& instances = output.chosen_instances[req_idx]; + uint32_t inst_idx = 0; + for (; inst_idx < instances.size(); ++inst_idx) + if (instances[inst_idx] == failed_acquire) break; + instances.erase(instances.begin() + inst_idx); + } - PhysicalInstance result; - if (map_legate_store(ctx, task, mapping, reqs, task.target_proc, result)) - needed_acquires.push_back(result); + PhysicalInstance result; + if (map_legate_store(ctx, task, mapping, reqs, task.target_proc, result)) + needed_acquires.push_back(result); - for (auto req_idx : req_indices) output.chosen_instances[req_idx].push_back(result); - instances_to_mappings[result] = mapping_idx; + for (auto req_idx : req_indices) output.chosen_instances[req_idx].push_back(result); + instances_to_mappings[result].insert(mapping_idx); + } } } } From 51dd00f6acf0ea32569498a131ceebd44f76ccaf Mon Sep 17 00:00:00 2001 From: Manolis Papadakis Date: Fri, 5 Nov 2021 13:29:35 -0700 Subject: [PATCH 0024/1425] Fix import of legion CFFI --- legate/core/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/legate/core/__init__.py b/legate/core/__init__.py index 8c607466f7..b3fa32b9cd 100644 --- a/legate/core/__init__.py +++ b/legate/core/__init__.py @@ -19,10 +19,10 @@ # Perform a check to see if we're running inside of Legion Python # If we're not then we should raise an error message try: - from legion_cffi import ffi, lib as legion + from legion_cffi import lib as _legion # Now confirm that we are actually inside of a task - if legion.legion_runtime_has_context(): + if _legion.legion_runtime_has_context(): using_legion_python = True else: using_legion_python = False @@ -115,6 +115,10 @@ ReductionOp, ) +# NOTE: This needs to come after the imports from legate.core.legion, as we +# are overriding that module's name. +from legion_cffi import ffi, lib as legion + # Import the PyArrow type system from pyarrow import ( DataType, From f388f07bb36594d9359676184a9e64c911b2309c Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 5 Nov 2021 16:25:15 -0700 Subject: [PATCH 0025/1425] Make sure we flush deferred detachments --- legate/core/runtime.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/legate/core/runtime.py b/legate/core/runtime.py index c06de76e12..957019d4e8 100644 --- a/legate/core/runtime.py +++ b/legate/core/runtime.py @@ -851,6 +851,8 @@ def destroy(self): self.destroyed = True def dispatch(self, op, redop=None): + self._attachment_manager.perform_detachments() + self._attachment_manager.prune_detachments() if redop: return op.launch(self.legion_runtime, self.legion_context, redop) else: From 2aad58194f4514458be484010a7e8c9f86afa2d2 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Tue, 9 Nov 2021 16:21:13 -0800 Subject: [PATCH 0026/1425] Pimpl for logical store to make it easier to keep a shared pointer of the parent in a child --- src/core/data/logical_store.cc | 116 ++++++++++++++++++++++++--- src/core/data/logical_store.h | 26 +++--- src/core/partitioning/constraint.cc | 5 +- src/core/partitioning/constraint.h | 3 +- src/core/partitioning/partition.cc | 19 +++-- src/core/partitioning/partition.h | 6 +- src/core/partitioning/partitioner.cc | 2 +- src/core/runtime/launcher.cc | 23 +++--- src/core/runtime/launcher.h | 11 ++- src/core/runtime/operation.cc | 21 +++-- src/core/runtime/operation.h | 20 ++--- src/core/runtime/runtime.cc | 8 +- src/core/runtime/runtime.h | 4 +- 13 files changed, 173 insertions(+), 91 deletions(-) diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index b6ac195736..7c745ec498 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -19,11 +19,14 @@ #include "core/partitioning/partition.h" #include "core/runtime/launcher.h" #include "core/runtime/runtime.h" +#include "legate_defines.h" using namespace Legion; namespace legate { +extern Logger log_legate; + LogicalRegionField::LogicalRegionField(Runtime* runtime, const LogicalRegion& lr, FieldID fid) : runtime_(runtime), lr_(lr), fid_(fid) { @@ -36,22 +39,77 @@ Domain LogicalRegionField::domain() const return runtime_->get_index_space_domain(lr_.get_index_space()); } +namespace detail { + +class LogicalStore { + public: + LogicalStore(); + LogicalStore(Runtime* runtime, + LegateTypeCode code, + std::vector extents, + std::shared_ptr parent, + std::shared_ptr transform); + + public: + LogicalStore(const LogicalStore& other) = default; + LogicalStore& operator=(const LogicalStore& other) = default; + + public: + LogicalStore(LogicalStore&& other) = default; + LogicalStore& operator=(LogicalStore&& other) = default; + + public: + int32_t dim() const; + LegateTypeCode code() const; + Legion::Domain domain() const; + const std::vector& extents() const; + size_t volume() const; + + public: + bool has_storage() const; + std::shared_ptr get_storage(); + + private: + void create_storage(); + + public: + std::shared_ptr get_physical_store(LibraryContext* context); + + public: + std::unique_ptr find_or_create_partition(const Partition* partition); + std::unique_ptr find_or_create_key_partition(); + + private: + Runtime* runtime_{nullptr}; + LegateTypeCode code_{MAX_TYPE_NUMBER}; + std::vector extents_; + std::shared_ptr region_field_{nullptr}; + std::shared_ptr parent_{nullptr}; + std::shared_ptr transform_{nullptr}; + std::shared_ptr mapped_{nullptr}; +}; + LogicalStore::LogicalStore(Runtime* runtime, LegateTypeCode code, std::vector extents, - std::shared_ptr transform /*= nullptr*/) + std::shared_ptr parent, + std::shared_ptr transform) : runtime_(runtime), code_(code), extents_(std::move(extents)), transform_(std::move(transform)) { } int32_t LogicalStore::dim() const { return static_cast(extents_.size()); } +LegateTypeCode LogicalStore::code() const { return code_; } + Domain LogicalStore::domain() const { assert(nullptr != region_field_); return region_field_->domain(); } +const std::vector& LogicalStore::extents() const { return extents_; } + size_t LogicalStore::volume() const { size_t vol = 1; @@ -59,17 +117,14 @@ size_t LogicalStore::volume() const return vol; } +bool LogicalStore::has_storage() const { return nullptr != region_field_; } + std::shared_ptr LogicalStore::get_storage() { if (!has_storage()) create_storage(); return region_field_; } -std::shared_ptr LogicalStore::get_storage_unsafe() const -{ - return region_field_; -} - void LogicalStore::create_storage() { region_field_ = runtime_->create_region_field(extents_, code_); @@ -87,16 +142,16 @@ std::unique_ptr LogicalStore::find_or_create_partition(const Partiti { // We're about to create a legion partition for this store, so the store should have its region // created. - if (!has_storage()) create_storage(); - auto lp = - partition->construct(this, partition->is_disjoint_for(this), partition->is_complete_for(this)); + auto lr = get_storage()->region(); + auto lp = partition->construct( + lr, partition->is_disjoint_for(nullptr), partition->is_complete_for(nullptr)); return std::make_unique(lp, 0); } std::unique_ptr LogicalStore::find_or_create_key_partition() { auto part_mgr = runtime_->get_partition_manager(); - auto launch_shape = part_mgr->compute_launch_shape(this); + auto launch_shape = part_mgr->compute_launch_shape(extents()); if (launch_shape.empty()) return create_no_partition(runtime_); else { @@ -105,4 +160,45 @@ std::unique_ptr LogicalStore::find_or_create_key_partition() } } +} // namespace detail + +LogicalStore::LogicalStore() {} + +LogicalStore::LogicalStore(Runtime* runtime, + LegateTypeCode code, + std::vector extents, + LogicalStore parent, /* = LogicalStore() */ + std::shared_ptr transform /*= nullptr*/) + : impl_(std::make_shared( + runtime, code, std::move(extents), parent.impl_, std::move(transform))) +{ +} + +int32_t LogicalStore::dim() const { return impl_->dim(); } + +LegateTypeCode LogicalStore::code() const { return impl_->code(); } + +Domain LogicalStore::domain() const { return impl_->domain(); } + +const std::vector& LogicalStore::extents() const { return impl_->extents(); } + +size_t LogicalStore::volume() const { return impl_->volume(); } + +std::shared_ptr LogicalStore::get_storage() { return impl_->get_storage(); } + +std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) +{ + return impl_->get_physical_store(context); +} + +std::unique_ptr LogicalStore::find_or_create_partition(const Partition* partition) +{ + return impl_->find_or_create_partition(partition); +} + +std::unique_ptr LogicalStore::find_or_create_key_partition() +{ + return impl_->find_or_create_key_partition(); +} + } // namespace legate diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index a2cf7e1fb2..a27d65a9cf 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -25,6 +25,12 @@ namespace legate { +namespace detail { + +class LogicalStore; + +} // namespace detail + class LibraryContext; class Partition; class Projection; @@ -61,10 +67,11 @@ class LogicalRegionField { class LogicalStore { public: - LogicalStore() {} + LogicalStore(); LogicalStore(Runtime* runtime, LegateTypeCode code, std::vector extents, + LogicalStore parent = LogicalStore(), std::shared_ptr transform = nullptr); public: @@ -77,18 +84,14 @@ class LogicalStore { public: int32_t dim() const; - LegateTypeCode code() const { return code_; } + LegateTypeCode code() const; Legion::Domain domain() const; - const std::vector& extents() const { return extents_; } + const std::vector& extents() const; size_t volume() const; public: - bool has_storage() const { return nullptr != region_field_; } + bool has_storage() const; std::shared_ptr get_storage(); - std::shared_ptr get_storage_unsafe() const; - - private: - void create_storage(); public: std::shared_ptr get_physical_store(LibraryContext* context); @@ -98,12 +101,7 @@ class LogicalStore { std::unique_ptr find_or_create_key_partition(); private: - Runtime* runtime_{nullptr}; - LegateTypeCode code_{MAX_TYPE_NUMBER}; - std::vector extents_; - std::shared_ptr region_field_{nullptr}; - std::shared_ptr transform_{nullptr}; - std::shared_ptr mapped_{nullptr}; + std::shared_ptr impl_{nullptr}; }; } // namespace legate diff --git a/src/core/partitioning/constraint.cc b/src/core/partitioning/constraint.cc index c8afd218f5..f08a344c8a 100644 --- a/src/core/partitioning/constraint.cc +++ b/src/core/partitioning/constraint.cc @@ -27,10 +27,7 @@ Literal::Literal(const std::shared_ptr& partition) : partition_(parti std::string Literal::to_string() const { return partition_->to_string(); } -Variable::Variable(const Operation* op, std::shared_ptr store, int32_t id) - : op_(op), id_(id) -{ -} +Variable::Variable(const Operation* op, int32_t id) : op_(op), id_(id) {} bool operator<(const Variable& lhs, const Variable& rhs) { diff --git a/src/core/partitioning/constraint.h b/src/core/partitioning/constraint.h index a7dc160567..120edf7e93 100644 --- a/src/core/partitioning/constraint.h +++ b/src/core/partitioning/constraint.h @@ -24,7 +24,6 @@ namespace legate { struct Constraint; struct Literal; -struct LogicalStore; struct Operation; struct Partition; struct Variable; @@ -67,7 +66,7 @@ struct Literal : public Expr { struct Variable : public Expr { public: - Variable(const Operation* op, std::shared_ptr store, int32_t id); + Variable(const Operation* op, int32_t id); public: Variable(const Variable&) = default; diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index f97c534b05..a54288c872 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -43,10 +43,10 @@ class NoPartition : public Partition { virtual bool is_disjoint_for(const LogicalStore* store) const override; public: - virtual Legion::LogicalPartition construct(const LogicalStore* store, + virtual Legion::LogicalPartition construct(Legion::LogicalRegion region, bool disjoint, bool complete) const override; - virtual std::unique_ptr get_projection(LogicalStore* store) const override; + virtual std::unique_ptr get_projection(LogicalStore store) const override; public: virtual bool has_launch_domain() const override; @@ -68,10 +68,10 @@ class Tiling : public Partition { virtual bool is_disjoint_for(const LogicalStore* store) const override; public: - virtual Legion::LogicalPartition construct(const LogicalStore* store, + virtual Legion::LogicalPartition construct(Legion::LogicalRegion region, bool disjoint, bool complete) const override; - virtual std::unique_ptr get_projection(LogicalStore* store) const override; + virtual std::unique_ptr get_projection(LogicalStore store) const override; public: virtual bool has_launch_domain() const override; @@ -121,14 +121,14 @@ bool NoPartition::is_complete_for(const LogicalStore* store) const { return fals bool NoPartition::is_disjoint_for(const LogicalStore* store) const { return false; } -Legion::LogicalPartition NoPartition::construct(const LogicalStore* store, +Legion::LogicalPartition NoPartition::construct(Legion::LogicalRegion region, bool disjoint, bool complete) const { return Legion::LogicalPartition::NO_PART; } -std::unique_ptr NoPartition::get_projection(LogicalStore* store) const +std::unique_ptr NoPartition::get_projection(LogicalStore store) const { return std::make_unique(); } @@ -147,7 +147,7 @@ bool Tiling::is_complete_for(const LogicalStore* store) const { return false; } bool Tiling::is_disjoint_for(const LogicalStore* store) const { return true; } -Legion::LogicalPartition Tiling::construct(const LogicalStore* store, +Legion::LogicalPartition Tiling::construct(Legion::LogicalRegion region, bool disjoint, bool complete) const { @@ -173,7 +173,6 @@ Legion::LogicalPartition Tiling::construct(const LogicalStore* store, color_domain.rect_data[idx + ndim] = color_shape_[idx] - 1; } - auto region = store->get_storage_unsafe()->region(); auto color_space = runtime_->find_or_create_index_space(color_domain); auto index_space = region.get_index_space(); @@ -185,9 +184,9 @@ Legion::LogicalPartition Tiling::construct(const LogicalStore* store, return runtime_->create_logical_partition(region, index_partition); } -std::unique_ptr Tiling::get_projection(LogicalStore* store) const +std::unique_ptr Tiling::get_projection(LogicalStore store) const { - return store->find_or_create_partition(this); + return store.find_or_create_partition(this); } bool Tiling::has_launch_domain() const { return true; } diff --git a/src/core/partitioning/partition.h b/src/core/partitioning/partition.h index fba8267046..d9e4b40610 100644 --- a/src/core/partitioning/partition.h +++ b/src/core/partitioning/partition.h @@ -35,10 +35,10 @@ struct Partition { virtual bool is_disjoint_for(const LogicalStore* store) const = 0; public: - virtual Legion::LogicalPartition construct(const LogicalStore* store, + virtual Legion::LogicalPartition construct(Legion::LogicalRegion region, bool disjoint = false, - bool complete = false) const = 0; - virtual std::unique_ptr get_projection(LogicalStore* store) const = 0; + bool complete = false) const = 0; + virtual std::unique_ptr get_projection(LogicalStore store) const = 0; public: virtual bool has_launch_domain() const = 0; diff --git a/src/core/partitioning/partitioner.cc b/src/core/partitioning/partitioner.cc index 15daee8a3a..5ce0bf2da3 100644 --- a/src/core/partitioning/partitioner.cc +++ b/src/core/partitioning/partitioner.cc @@ -104,7 +104,7 @@ std::unique_ptr Partitioner::solve() for (auto& variable : variables) { auto* op = variable->operation(); auto store = op->find_store(variable); - auto partition = store->find_or_create_key_partition(); + auto partition = store.find_or_create_key_partition(); if (!strategy->has_launch_domain(op)) { if (partition->has_launch_domain()) strategy->set_launch_domain(op, partition->launch_domain()); diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/launcher.cc index 30d31192ac..9f3eb365f3 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -84,12 +84,9 @@ struct UntypedScalarArg : public ArgWrapper { }; struct RegionFieldArg : public ArgWrapper { - private: - using LogicalStoreP = std::shared_ptr; - public: RegionFieldArg(RequirementAnalyzer* analyzer, - LogicalStoreP store, + LogicalStore store, int32_t dim, RegionReq* req, Legion::FieldID field_id, @@ -100,7 +97,7 @@ struct RegionFieldArg : public ArgWrapper { private: RequirementAnalyzer* analyzer_; - LogicalStoreP store_; + LogicalStore store_; int32_t dim_; RegionReq* req_; Legion::FieldID field_id_; @@ -176,7 +173,7 @@ void UntypedScalarArg::pack(BufferBuilder& buffer) const } RegionFieldArg::RegionFieldArg(RequirementAnalyzer* analyzer, - LogicalStoreP store, + LogicalStore store, int32_t dim, RegionReq* req, Legion::FieldID field_id, @@ -188,8 +185,8 @@ RegionFieldArg::RegionFieldArg(RequirementAnalyzer* analyzer, void RegionFieldArg::pack(BufferBuilder& buffer) const { buffer.pack(false); - buffer.pack(store_->dim()); - buffer.pack(store_->code()); + buffer.pack(store_.dim()); + buffer.pack(store_.code()); buffer.pack(-1); buffer.pack(redop_); @@ -324,17 +321,17 @@ void TaskLauncher::add_scalar(const Scalar& scalar) scalars_.push_back(new UntypedScalarArg(scalar)); } -void TaskLauncher::add_input(LogicalStoreP store, ProjectionP proj, uint64_t tag /*= 0*/) +void TaskLauncher::add_input(LogicalStore store, ProjectionP proj, uint64_t tag /*= 0*/) { add_store(inputs_, std::move(store), std::move(proj), READ_ONLY, tag); } -void TaskLauncher::add_output(LogicalStoreP store, ProjectionP proj, uint64_t tag /*= 0*/) +void TaskLauncher::add_output(LogicalStore store, ProjectionP proj, uint64_t tag /*= 0*/) { add_store(outputs_, std::move(store), std::move(proj), WRITE_ONLY, tag); } -void TaskLauncher::add_reduction(LogicalStoreP store, +void TaskLauncher::add_reduction(LogicalStore store, ProjectionP proj, uint64_t tag /*= 0*/, bool read_write /*= false*/) @@ -358,12 +355,12 @@ void TaskLauncher::execute_single() } void TaskLauncher::add_store(std::vector& args, - LogicalStoreP store, + LogicalStore store, ProjectionP proj, Legion::PrivilegeMode privilege, uint64_t tag) { - auto storage = store->get_storage(); + auto storage = store.get_storage(); auto region = storage->region(); auto field_id = storage->field_id(); diff --git a/src/core/runtime/launcher.h b/src/core/runtime/launcher.h index 12b4f4d1ef..8d93b67624 100644 --- a/src/core/runtime/launcher.h +++ b/src/core/runtime/launcher.h @@ -113,8 +113,7 @@ class RegionReq { class TaskLauncher { private: - using LogicalStoreP = std::shared_ptr; - using ProjectionP = std::unique_ptr; + using ProjectionP = std::unique_ptr; public: TaskLauncher(Runtime* runtime, @@ -130,16 +129,16 @@ class TaskLauncher { public: void add_scalar(const Scalar& scalar); - void add_input(LogicalStoreP store, ProjectionP proj, uint64_t tag = 0); - void add_output(LogicalStoreP store, ProjectionP proj, uint64_t tag = 0); - void add_reduction(LogicalStoreP store, + void add_input(LogicalStore store, ProjectionP proj, uint64_t tag = 0); + void add_output(LogicalStore store, ProjectionP proj, uint64_t tag = 0); + void add_reduction(LogicalStore store, ProjectionP proj, uint64_t tag = 0, bool read_write = false); private: void add_store(std::vector& args, - LogicalStoreP store, + LogicalStore store, ProjectionP proj, Legion::PrivilegeMode privilege, uint64_t tag); diff --git a/src/core/runtime/operation.cc b/src/core/runtime/operation.cc index f024c1d3d2..cec2b4776a 100644 --- a/src/core/runtime/operation.cc +++ b/src/core/runtime/operation.cc @@ -17,7 +17,6 @@ #include #include -#include "core/data/logical_store.h" #include "core/data/scalar.h" #include "core/partitioning/constraint.h" #include "core/partitioning/constraint_graph.h" @@ -42,19 +41,19 @@ Operation::Operation(Runtime* runtime, { } -void Operation::add_input(std::shared_ptr store, std::shared_ptr partition) +void Operation::add_input(LogicalStore store, std::shared_ptr partition) { constraints_->add_variable(partition); inputs_.push_back(Store(std::move(store), std::move(partition))); } -void Operation::add_output(std::shared_ptr store, std::shared_ptr partition) +void Operation::add_output(LogicalStore store, std::shared_ptr partition) { constraints_->add_variable(partition); outputs_.push_back(Store(std::move(store), std::move(partition))); } -void Operation::add_reduction(std::shared_ptr store, +void Operation::add_reduction(LogicalStore store, Legion::ReductionOpID redop, std::shared_ptr partition) { @@ -63,14 +62,14 @@ void Operation::add_reduction(std::shared_ptr store, reduction_ops_.push_back(redop); } -std::shared_ptr Operation::declare_partition(std::shared_ptr store) +std::shared_ptr Operation::declare_partition(LogicalStore store) { - auto variable = std::make_shared(this, store, next_part_id_++); + auto variable = std::make_shared(this, next_part_id_++); store_mappings_[variable] = std::move(store); return std::move(variable); } -std::shared_ptr Operation::find_store(std::shared_ptr variable) const +LogicalStore Operation::find_store(std::shared_ptr variable) const { auto finder = store_mappings_.find(variable); assert(store_mappings_.end() != finder); @@ -95,7 +94,7 @@ Task::Task(Runtime* runtime, void Task::add_scalar_arg(const Scalar& scalar) { scalars_.push_back(scalar); } -void Task::launch(Strategy* p_strategy) const +void Task::launch(Strategy* p_strategy) { auto& strategy = *p_strategy; TaskLauncher launcher(runtime_, library_, task_id_, mapper_id_); @@ -103,20 +102,20 @@ void Task::launch(Strategy* p_strategy) const for (auto& pair : inputs_) { auto& store = pair.first; auto& var = pair.second; - auto proj = strategy[var]->get_projection(store.get()); + auto proj = strategy[var]->get_projection(store); launcher.add_input(store, std::move(proj)); } for (auto& pair : outputs_) { auto& store = pair.first; auto& var = pair.second; - auto proj = strategy[var]->get_projection(store.get()); + auto proj = strategy[var]->get_projection(store); launcher.add_output(store, std::move(proj)); } uint32_t idx = 0; for (auto& pair : reductions_) { auto& store = pair.first; auto& var = pair.second; - auto proj = strategy[var]->get_projection(store.get()); + auto proj = strategy[var]->get_projection(store); auto redop = reduction_ops_[idx++]; proj->set_reduction_op(redop); launcher.add_reduction(store, std::move(proj)); diff --git a/src/core/runtime/operation.h b/src/core/runtime/operation.h index d79653acda..d3969f3b3b 100644 --- a/src/core/runtime/operation.h +++ b/src/core/runtime/operation.h @@ -19,6 +19,7 @@ #include #include +#include "core/data/logical_store.h" #include "legion.h" namespace legate { @@ -26,7 +27,6 @@ namespace legate { class Constraint; class ConstraintGraph; class LibraryContext; -class LogicalStore; class Runtime; class Scalar; class Strategy; @@ -37,20 +37,20 @@ class Operation { Operation(Runtime* runtime, LibraryContext* library, uint64_t unique_id, int64_t mapper_id); public: - void add_input(std::shared_ptr store, std::shared_ptr partition); - void add_output(std::shared_ptr store, std::shared_ptr partition); - void add_reduction(std::shared_ptr store, + void add_input(LogicalStore store, std::shared_ptr partition); + void add_output(LogicalStore store, std::shared_ptr partition); + void add_reduction(LogicalStore store, Legion::ReductionOpID redop, std::shared_ptr partition); public: - std::shared_ptr declare_partition(std::shared_ptr store); - std::shared_ptr find_store(std::shared_ptr variable) const; + std::shared_ptr declare_partition(LogicalStore store); + LogicalStore find_store(std::shared_ptr variable) const; void add_constraint(std::shared_ptr constraint); std::shared_ptr constraints() const; public: - virtual void launch(Strategy* strategy) const = 0; + virtual void launch(Strategy* strategy) = 0; public: virtual std::string to_string() const = 0; @@ -62,7 +62,7 @@ class Operation { int64_t mapper_id_; protected: - using Store = std::pair, std::shared_ptr>; + using Store = std::pair>; protected: std::vector inputs_{}; @@ -74,7 +74,7 @@ class Operation { uint32_t next_part_id_{0}; private: - std::unordered_map, std::shared_ptr> store_mappings_; + std::unordered_map, LogicalStore> store_mappings_; std::shared_ptr constraints_; }; @@ -90,7 +90,7 @@ class Task : public Operation { void add_scalar_arg(const Scalar& scalar); public: - virtual void launch(Strategy* strategy) const override; + virtual void launch(Strategy* strategy) override; public: virtual std::string to_string() const override; diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 7a75511577..9bafe48108 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -415,13 +415,12 @@ PartitionManager::PartitionManager(Runtime* runtime, const LibraryContext* conte push_factors(2); } -std::vector PartitionManager::compute_launch_shape(const LogicalStore* store) +std::vector PartitionManager::compute_launch_shape(const std::vector& shape) { // Easy case if we only have one piece: no parallel launch space if (num_pieces_ == 1) return {}; // If we only have one point then we never do parallel launches - auto& shape = store->extents(); if (std::all_of(shape.begin(), shape.end(), [](auto extent) { return 1 == extent; })) return {}; // Prune out any dimensions that are 1 @@ -632,10 +631,9 @@ void Runtime::schedule(std::vector> operations) for (auto& op : operations) op->launch(strategy.get()); } -std::shared_ptr Runtime::create_store(std::vector extents, - LegateTypeCode code) +LogicalStore Runtime::create_store(std::vector extents, LegateTypeCode code) { - return std::make_shared(this, code, extents); + return LogicalStore(this, code, extents); } std::shared_ptr Runtime::create_region_field(const std::vector& extents, diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 3d4631b00d..a234e79f95 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -60,7 +60,7 @@ class PartitionManager { PartitionManager(Runtime* runtime, const LibraryContext* context); public: - std::vector compute_launch_shape(const LogicalStore* store); + std::vector compute_launch_shape(const std::vector& shape); std::vector compute_tile_shape(const std::vector& extents, const std::vector& launch_shape); @@ -98,7 +98,7 @@ class Runtime { void submit(std::unique_ptr op); public: - std::shared_ptr create_store(std::vector extents, LegateTypeCode code); + LogicalStore create_store(std::vector extents, LegateTypeCode code); std::shared_ptr create_region_field(const std::vector& extents, LegateTypeCode code); RegionField map_region_field(LibraryContext* context, From bbedbacd4c244c3e18590874da459de7963a8d2e Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Tue, 9 Nov 2021 21:23:28 -0800 Subject: [PATCH 0027/1425] Add promotion to logical store --- src/core/data/logical_store.cc | 48 +++++++++++++++++++++++++++++++--- src/core/data/logical_store.h | 6 +++++ 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index 7c745ec498..c00d5ddef5 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -50,6 +50,9 @@ class LogicalStore { std::shared_ptr parent, std::shared_ptr transform); + private: + LogicalStore(std::shared_ptr impl); + public: LogicalStore(const LogicalStore& other) = default; LogicalStore& operator=(const LogicalStore& other) = default; @@ -72,6 +75,11 @@ class LogicalStore { private: void create_storage(); + public: + std::shared_ptr promote(int32_t extra_dim, + size_t dim_size, + std::shared_ptr parent) const; + public: std::shared_ptr get_physical_store(LibraryContext* context); @@ -94,7 +102,11 @@ LogicalStore::LogicalStore(Runtime* runtime, std::vector extents, std::shared_ptr parent, std::shared_ptr transform) - : runtime_(runtime), code_(code), extents_(std::move(extents)), transform_(std::move(transform)) + : runtime_(runtime), + code_(code), + extents_(std::move(extents)), + parent_(std::move(parent)), + transform_(std::move(transform)) { } @@ -121,8 +133,11 @@ bool LogicalStore::has_storage() const { return nullptr != region_field_; } std::shared_ptr LogicalStore::get_storage() { - if (!has_storage()) create_storage(); - return region_field_; + if (nullptr == parent_) { + if (!has_storage()) create_storage(); + return region_field_; + } else + return parent_->get_storage(); } void LogicalStore::create_storage() @@ -130,6 +145,26 @@ void LogicalStore::create_storage() region_field_ = runtime_->create_region_field(extents_, code_); } +std::shared_ptr LogicalStore::promote(int32_t extra_dim, + size_t dim_size, + std::shared_ptr parent) const +{ + if (extra_dim < 0 || static_cast(extra_dim) > extents_.size()) { + log_legate.error( + "Invalid promotion on dimension %d for a %zd-D store", extra_dim, extents_.size()); + LEGATE_ABORT + } + std::vector new_extents; + for (int32_t dim = 0; dim < extra_dim; ++dim) new_extents.push_back(extents_[dim]); + new_extents.push_back(dim_size); + for (int32_t dim = extra_dim; dim < static_cast(extents_.size()); ++dim) + new_extents.push_back(extents_[dim]); + + auto transform = std::make_shared(extra_dim, dim_size); + return std::make_shared( + runtime_, code_, std::move(new_extents), std::move(parent), std::move(transform)); +} + std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) { if (nullptr != mapped_) return mapped_; @@ -174,6 +209,8 @@ LogicalStore::LogicalStore(Runtime* runtime, { } +LogicalStore::LogicalStore(std::shared_ptr impl) : impl_(std::move(impl)) {} + int32_t LogicalStore::dim() const { return impl_->dim(); } LegateTypeCode LogicalStore::code() const { return impl_->code(); } @@ -186,6 +223,11 @@ size_t LogicalStore::volume() const { return impl_->volume(); } std::shared_ptr LogicalStore::get_storage() { return impl_->get_storage(); } +LogicalStore LogicalStore::promote(int32_t extra_dim, size_t dim_size) const +{ + return LogicalStore(impl_->promote(extra_dim, dim_size, impl_)); +} + std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) { return impl_->get_physical_store(context); diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index a27d65a9cf..b5130009a7 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -74,6 +74,9 @@ class LogicalStore { LogicalStore parent = LogicalStore(), std::shared_ptr transform = nullptr); + private: + LogicalStore(std::shared_ptr impl); + public: LogicalStore(const LogicalStore& other) = default; LogicalStore& operator=(const LogicalStore& other) = default; @@ -93,6 +96,9 @@ class LogicalStore { bool has_storage() const; std::shared_ptr get_storage(); + public: + LogicalStore promote(int32_t extra_dim, size_t dim_size) const; + public: std::shared_ptr get_physical_store(LibraryContext* context); From 0dc791b1f69b817f7e29f50cff8c03fdb1821ca1 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 10 Nov 2021 13:53:19 -0800 Subject: [PATCH 0028/1425] Future-backed logical stores --- src/core/data/logical_store.cc | 49 +++++++++++++++++++++ src/core/data/logical_store.h | 4 ++ src/core/runtime/launcher.cc | 80 ++++++++++++++++++++++++++++++---- src/core/runtime/launcher.h | 1 + src/core/runtime/runtime.cc | 10 +++++ src/core/runtime/runtime.h | 2 + 6 files changed, 137 insertions(+), 9 deletions(-) diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index c00d5ddef5..68237c19be 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -19,6 +19,8 @@ #include "core/partitioning/partition.h" #include "core/runtime/launcher.h" #include "core/runtime/runtime.h" +#include "core/utilities/dispatch.h" +#include "core/utilities/type_traits.h" #include "legate_defines.h" using namespace Legion; @@ -49,6 +51,7 @@ class LogicalStore { std::vector extents, std::shared_ptr parent, std::shared_ptr transform); + LogicalStore(Runtime* runtime, LegateTypeCode code, const void* data); private: LogicalStore(std::shared_ptr impl); @@ -62,6 +65,7 @@ class LogicalStore { LogicalStore& operator=(LogicalStore&& other) = default; public: + bool scalar() const; int32_t dim() const; LegateTypeCode code() const; Legion::Domain domain() const; @@ -71,6 +75,7 @@ class LogicalStore { public: bool has_storage() const; std::shared_ptr get_storage(); + Legion::Future get_future(); private: void create_storage(); @@ -88,10 +93,12 @@ class LogicalStore { std::unique_ptr find_or_create_key_partition(); private: + bool scalar_{false}; Runtime* runtime_{nullptr}; LegateTypeCode code_{MAX_TYPE_NUMBER}; std::vector extents_; std::shared_ptr region_field_{nullptr}; + Legion::Future future_{}; std::shared_ptr parent_{nullptr}; std::shared_ptr transform_{nullptr}; std::shared_ptr mapped_{nullptr}; @@ -110,6 +117,23 @@ LogicalStore::LogicalStore(Runtime* runtime, { } +struct datalen_fn { + template + size_t operator()() + { + return sizeof(legate_type_of); + } +}; + +LogicalStore::LogicalStore(Runtime* runtime, LegateTypeCode code, const void* data) + : scalar_(true), runtime_(runtime), code_(code), extents_({1}) +{ + auto datalen = type_dispatch(code, datalen_fn{}); + future_ = runtime_->create_future(data, datalen); +} + +bool LogicalStore::scalar() const { return scalar_; } + int32_t LogicalStore::dim() const { return static_cast(extents_.size()); } LegateTypeCode LogicalStore::code() const { return code_; } @@ -133,6 +157,7 @@ bool LogicalStore::has_storage() const { return nullptr != region_field_; } std::shared_ptr LogicalStore::get_storage() { + assert(!scalar_); if (nullptr == parent_) { if (!has_storage()) create_storage(); return region_field_; @@ -140,6 +165,15 @@ std::shared_ptr LogicalStore::get_storage() return parent_->get_storage(); } +Legion::Future LogicalStore::get_future() +{ + assert(scalar_); + if (nullptr == parent_) { + return future_; + } else + return parent_->get_future(); +} + void LogicalStore::create_storage() { region_field_ = runtime_->create_region_field(extents_, code_); @@ -167,6 +201,8 @@ std::shared_ptr LogicalStore::promote(int32_t extra_dim, std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) { + // TODO: Need to support inline mapping for scalars + assert(!scalar_); if (nullptr != mapped_) return mapped_; auto rf = runtime_->map_region_field(context, region_field_); mapped_ = std::make_shared(dim(), code_, -1, std::move(rf), transform_); @@ -175,6 +211,8 @@ std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) std::unique_ptr LogicalStore::find_or_create_partition(const Partition* partition) { + if (scalar_) return std::make_unique(); + // We're about to create a legion partition for this store, so the store should have its region // created. auto lr = get_storage()->region(); @@ -185,6 +223,8 @@ std::unique_ptr LogicalStore::find_or_create_partition(const Partiti std::unique_ptr LogicalStore::find_or_create_key_partition() { + if (scalar_) return create_no_partition(runtime_); + auto part_mgr = runtime_->get_partition_manager(); auto launch_shape = part_mgr->compute_launch_shape(extents()); if (launch_shape.empty()) @@ -209,8 +249,15 @@ LogicalStore::LogicalStore(Runtime* runtime, { } +LogicalStore::LogicalStore(Runtime* runtime, LegateTypeCode code, const void* data) + : impl_(std::make_shared(runtime, code, data)) +{ +} + LogicalStore::LogicalStore(std::shared_ptr impl) : impl_(std::move(impl)) {} +bool LogicalStore::scalar() const { return impl_->scalar(); } + int32_t LogicalStore::dim() const { return impl_->dim(); } LegateTypeCode LogicalStore::code() const { return impl_->code(); } @@ -223,6 +270,8 @@ size_t LogicalStore::volume() const { return impl_->volume(); } std::shared_ptr LogicalStore::get_storage() { return impl_->get_storage(); } +Legion::Future LogicalStore::get_future() { return impl_->get_future(); } + LogicalStore LogicalStore::promote(int32_t extra_dim, size_t dim_size) const { return LogicalStore(impl_->promote(extra_dim, dim_size, impl_)); diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index b5130009a7..7606fefb82 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -73,6 +73,8 @@ class LogicalStore { std::vector extents, LogicalStore parent = LogicalStore(), std::shared_ptr transform = nullptr); + // Creates a read-only store from a scalar + LogicalStore(Runtime* runtime, LegateTypeCode code, const void* data); private: LogicalStore(std::shared_ptr impl); @@ -86,6 +88,7 @@ class LogicalStore { LogicalStore& operator=(LogicalStore&& other) = default; public: + bool scalar() const; int32_t dim() const; LegateTypeCode code() const; Legion::Domain domain() const; @@ -95,6 +98,7 @@ class LogicalStore { public: bool has_storage() const; std::shared_ptr get_storage(); + Legion::Future get_future(); public: LogicalStore promote(int32_t extra_dim, size_t dim_size) const; diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/launcher.cc index 9f3eb365f3..8897b7dfc5 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -19,6 +19,7 @@ #include "core/data/scalar.h" #include "core/runtime/context.h" #include "core/runtime/runtime.h" +#include "core/utilities/dispatch.h" namespace legate { @@ -29,6 +30,8 @@ class BufferBuilder { public: template void pack(const T& value); + template + void pack(const std::vector& values); void pack_buffer(const void* buffer, size_t size); public: @@ -104,6 +107,18 @@ struct RegionFieldArg : public ArgWrapper { Legion::ReductionOpID redop_; }; +struct FutureStoreArg : public ArgWrapper { + public: + FutureStoreArg(LogicalStore store, bool read_only); + + public: + virtual void pack(BufferBuilder& buffer) const override; + + private: + LogicalStore store_; + bool read_only_; +}; + BufferBuilder::BufferBuilder() { // Reserve 4KB to minimize resizing while packing the arguments. @@ -116,6 +131,14 @@ void BufferBuilder::pack(const T& value) pack_buffer(reinterpret_cast(&value), sizeof(T)); } +template +void BufferBuilder::pack(const std::vector& values) +{ + uint32_t size = values.size(); + pack(size); + pack_buffer(values.data(), size * sizeof(T)); +} + void BufferBuilder::pack_buffer(const void* src, size_t size) { auto tgt = buffer_.data() + buffer_.size(); @@ -178,7 +201,12 @@ RegionFieldArg::RegionFieldArg(RequirementAnalyzer* analyzer, RegionReq* req, Legion::FieldID field_id, Legion::ReductionOpID redop) - : analyzer_(analyzer), store_(store), dim_(dim), req_(req), field_id_(field_id), redop_(redop) + : analyzer_(analyzer), + store_(std::move(store)), + dim_(dim), + req_(req), + field_id_(field_id), + redop_(redop) { } @@ -195,6 +223,32 @@ void RegionFieldArg::pack(BufferBuilder& buffer) const buffer.pack(field_id_); } +FutureStoreArg::FutureStoreArg(LogicalStore store, bool read_only) + : store_(std::move(store)), read_only_(read_only) +{ +} + +struct datalen_fn { + template + size_t operator()() + { + return sizeof(legate_type_of); + } +}; + +void FutureStoreArg::pack(BufferBuilder& buffer) const +{ + buffer.pack(true); + buffer.pack(store_.dim()); + buffer.pack(store_.code()); + buffer.pack(-1); + + buffer.pack(read_only_); + buffer.pack(true); + buffer.pack(type_dispatch(store_.code(), datalen_fn{})); + buffer.pack(store_.extents()); +} + Projection::Projection(Legion::ReductionOpID r) : redop(std::make_unique(r)) { } @@ -360,16 +414,22 @@ void TaskLauncher::add_store(std::vector& args, Legion::PrivilegeMode privilege, uint64_t tag) { - auto storage = store.get_storage(); - auto region = storage->region(); - auto field_id = storage->field_id(); + if (store.scalar()) { + futures_.push_back(store.get_future()); + auto read_only = privilege == READ_ONLY; + args.push_back(new FutureStoreArg(std::move(store), read_only)); + } else { + auto storage = store.get_storage(); + auto region = storage->region(); + auto field_id = storage->field_id(); - auto redop = nullptr != proj->redop ? *proj->redop : -1; - auto req = new RegionReq(region, privilege, std::move(proj), tag); + auto redop = nullptr != proj->redop ? *proj->redop : -1; + auto req = new RegionReq(region, privilege, std::move(proj), tag); - req_analyzer_->insert(req, field_id); - args.push_back( - new RegionFieldArg(req_analyzer_, std::move(store), region.get_dim(), req, field_id, redop)); + req_analyzer_->insert(req, field_id); + args.push_back( + new RegionFieldArg(req_analyzer_, std::move(store), region.get_dim(), req, field_id, redop)); + } } void TaskLauncher::pack_args(const std::vector& args) @@ -390,6 +450,7 @@ Legion::TaskLauncher* TaskLauncher::build_single_task() Legion::Predicate::TRUE_PRED, legion_mapper_id(), tag_); + for (auto& future_ : futures_) single_task->add_future(future_); req_analyzer_->populate_launcher(single_task); @@ -411,6 +472,7 @@ Legion::IndexTaskLauncher* TaskLauncher::build_index_task(const Legion::Domain& false /*must*/, legion_mapper_id(), tag_); + for (auto& future_ : futures_) index_task->add_future(future_); req_analyzer_->populate_launcher(index_task); diff --git a/src/core/runtime/launcher.h b/src/core/runtime/launcher.h index 8d93b67624..75287f4f9a 100644 --- a/src/core/runtime/launcher.h +++ b/src/core/runtime/launcher.h @@ -164,6 +164,7 @@ class TaskLauncher { std::vector outputs_; std::vector reductions_; std::vector scalars_; + std::vector futures_; private: RequirementAnalyzer* req_analyzer_; diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 9bafe48108..77980bd0da 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -636,6 +636,11 @@ LogicalStore Runtime::create_store(std::vector extents, LegateTypeCode c return LogicalStore(this, code, extents); } +LogicalStore Runtime::create_store(const Scalar& scalar) +{ + return LogicalStore(this, scalar.code(), scalar.ptr()); +} + std::shared_ptr Runtime::create_region_field(const std::vector& extents, LegateTypeCode code) { @@ -733,6 +738,11 @@ Legion::LogicalPartition Runtime::create_logical_partition( return legion_runtime_->get_logical_partition(legion_context_, logical_region, index_partition); } +Legion::Future Runtime::create_future(const void* data, size_t datalen) const +{ + return Legion::Future::from_untyped_pointer(legion_runtime_, data, datalen); +} + FieldID Runtime::allocate_field(const FieldSpace& field_space, size_t field_size) { assert(nullptr != legion_context_); diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index a234e79f95..68541630f1 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -99,6 +99,7 @@ class Runtime { public: LogicalStore create_store(std::vector extents, LegateTypeCode code); + LogicalStore create_store(const Scalar& scalar); std::shared_ptr create_region_field(const std::vector& extents, LegateTypeCode code); RegionField map_region_field(LibraryContext* context, @@ -120,6 +121,7 @@ class Runtime { const Legion::FieldSpace& field_space); Legion::LogicalPartition create_logical_partition(const Legion::LogicalRegion& logical_region, const Legion::IndexPartition& index_partition); + Legion::Future create_future(const void* data, size_t datalen) const; Legion::FieldID allocate_field(const Legion::FieldSpace& field_space, size_t field_size); Legion::Domain get_index_space_domain(const Legion::IndexSpace& index_space) const; From 9e0a41c666545c4a0c746ce520d854e44d7821ef Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 10 Nov 2021 15:52:50 -0800 Subject: [PATCH 0029/1425] Factor out the BufferBuilder class and pack transforms --- src/core.mk | 1 + src/core/data/logical_store.cc | 27 ++++++++++++ src/core/data/logical_store.h | 4 ++ src/core/data/transform.cc | 38 +++++++++++++++++ src/core/data/transform.h | 8 ++++ src/core/runtime/launcher.cc | 61 ++------------------------- src/core/utilities/buffer_builder.cc | 39 +++++++++++++++++ src/core/utilities/buffer_builder.h | 43 +++++++++++++++++++ src/core/utilities/buffer_builder.inl | 33 +++++++++++++++ 9 files changed, 196 insertions(+), 58 deletions(-) create mode 100644 src/core/utilities/buffer_builder.cc create mode 100644 src/core/utilities/buffer_builder.h create mode 100644 src/core/utilities/buffer_builder.inl diff --git a/src/core.mk b/src/core.mk index b88a0d26c2..473ed89ee8 100644 --- a/src/core.mk +++ b/src/core.mk @@ -37,6 +37,7 @@ GEN_CPU_SRC = core/legate_c.cc \ core/runtime/shard.cc \ core/task/return.cc \ core/task/task.cc \ + core/utilities/buffer_builder.cc \ core/utilities/deserializer.cc \ core/utilities/machine.cc \ core/utilities/linearize.cc diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index 68237c19be..543a7d1987 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -19,6 +19,7 @@ #include "core/partitioning/partition.h" #include "core/runtime/launcher.h" #include "core/runtime/runtime.h" +#include "core/utilities/buffer_builder.h" #include "core/utilities/dispatch.h" #include "core/utilities/type_traits.h" #include "legate_defines.h" @@ -92,6 +93,12 @@ class LogicalStore { std::unique_ptr find_or_create_partition(const Partition* partition); std::unique_ptr find_or_create_key_partition(); + public: + void pack(BufferBuilder& buffer) const; + + private: + void pack_transform(BufferBuilder& buffer) const; + private: bool scalar_{false}; Runtime* runtime_{nullptr}; @@ -235,6 +242,24 @@ std::unique_ptr LogicalStore::find_or_create_key_partition() } } +void LogicalStore::pack(BufferBuilder& buffer) const +{ + buffer.pack(scalar_); + buffer.pack(dim()); + buffer.pack(code_); + pack_transform(buffer); +} + +void LogicalStore::pack_transform(BufferBuilder& buffer) const +{ + if (nullptr == parent_) + buffer.pack(-1); + else { + transform_->pack(buffer); + parent_->pack_transform(buffer); + } +} + } // namespace detail LogicalStore::LogicalStore() {} @@ -292,4 +317,6 @@ std::unique_ptr LogicalStore::find_or_create_key_partition() return impl_->find_or_create_key_partition(); } +void LogicalStore::pack(BufferBuilder& buffer) const { impl_->pack(buffer); } + } // namespace legate diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index 7606fefb82..1b123da899 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -31,6 +31,7 @@ class LogicalStore; } // namespace detail +class BufferBuilder; class LibraryContext; class Partition; class Projection; @@ -110,6 +111,9 @@ class LogicalStore { std::unique_ptr find_or_create_partition(const Partition* partition); std::unique_ptr find_or_create_key_partition(); + public: + void pack(BufferBuilder& buffer) const; + private: std::shared_ptr impl_{nullptr}; }; diff --git a/src/core/data/transform.cc b/src/core/data/transform.cc index 0f2fddc66a..fc5f850abd 100644 --- a/src/core/data/transform.cc +++ b/src/core/data/transform.cc @@ -15,6 +15,8 @@ */ #include "core/data/transform.h" +#include "core/legate_c.h" +#include "core/utilities/buffer_builder.h" namespace legate { @@ -80,6 +82,13 @@ DomainAffineTransform Shift::inverse_transform(int32_t in_dim) const return result; } +void Shift::pack(BufferBuilder& buffer) const +{ + buffer.pack(LEGATE_CORE_TRANSFORM_SHIFT); + buffer.pack(dim_); + buffer.pack(offset_); +} + void Shift::print(std::ostream& out) const { out << "Shift("; @@ -148,6 +157,13 @@ DomainAffineTransform Promote::inverse_transform(int32_t in_dim) const return result; } +void Promote::pack(BufferBuilder& buffer) const +{ + buffer.pack(LEGATE_CORE_TRANSFORM_PROMOTE); + buffer.pack(extra_dim_); + buffer.pack(dim_size_); +} + void Promote::print(std::ostream& out) const { out << "Promote("; @@ -216,6 +232,13 @@ DomainAffineTransform Project::inverse_transform(int32_t in_dim) const return result; } +void Project::pack(BufferBuilder& buffer) const +{ + buffer.pack(LEGATE_CORE_TRANSFORM_PROJECT); + buffer.pack(dim_); + buffer.pack(coord_); +} + void Project::print(std::ostream& out) const { out << "Project("; @@ -291,6 +314,13 @@ void print_vector(std::ostream& out, const std::vector& vec) } } // anonymous namespace +void Transpose::pack(BufferBuilder& buffer) const +{ + buffer.pack(LEGATE_CORE_TRANSFORM_TRANSPOSE); + buffer.pack(axes_.size()); + for (auto axis : axes_) buffer.pack(axis); +} + void Transpose::print(std::ostream& out) const { out << "Transpose("; @@ -374,6 +404,14 @@ DomainAffineTransform Delinearize::inverse_transform(int32_t in_dim) const return result; } +void Delinearize::pack(BufferBuilder& buffer) const +{ + buffer.pack(LEGATE_CORE_TRANSFORM_DELINEARIZE); + buffer.pack(dim_); + buffer.pack(sizes_.size()); + for (auto extent : sizes_) buffer.pack(extent); +} + void Delinearize::print(std::ostream& out) const { out << "Delinearize("; diff --git a/src/core/data/transform.h b/src/core/data/transform.h index 16b68618d3..619f4ab292 100644 --- a/src/core/data/transform.h +++ b/src/core/data/transform.h @@ -22,6 +22,8 @@ namespace legate { +class BufferBuilder; + class StoreTransform { public: StoreTransform() {} @@ -31,6 +33,7 @@ class StoreTransform { public: virtual Legion::Domain transform(const Legion::Domain& input) const = 0; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const = 0; + virtual void pack(BufferBuilder& buffer) const = 0; virtual void print(std::ostream& out) const = 0; protected: @@ -47,6 +50,7 @@ class Shift : public StoreTransform { public: virtual Legion::Domain transform(const Legion::Domain& input) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; + virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; private: @@ -62,6 +66,7 @@ class Promote : public StoreTransform { public: virtual Legion::Domain transform(const Legion::Domain& input) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; + virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; private: @@ -77,6 +82,7 @@ class Project : public StoreTransform { public: virtual Legion::Domain transform(const Legion::Domain& domain) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; + virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; private: @@ -92,6 +98,7 @@ class Transpose : public StoreTransform { public: virtual Legion::Domain transform(const Legion::Domain& domain) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; + virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; private: @@ -108,6 +115,7 @@ class Delinearize : public StoreTransform { public: virtual Legion::Domain transform(const Legion::Domain& domain) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; + virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; private: diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/launcher.cc index 8897b7dfc5..6fa1495c78 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -19,28 +19,11 @@ #include "core/data/scalar.h" #include "core/runtime/context.h" #include "core/runtime/runtime.h" +#include "core/utilities/buffer_builder.h" #include "core/utilities/dispatch.h" namespace legate { -class BufferBuilder { - public: - BufferBuilder(); - - public: - template - void pack(const T& value); - template - void pack(const std::vector& values); - void pack_buffer(const void* buffer, size_t size); - - public: - Legion::UntypedBuffer to_legion_buffer() const; - - private: - std::vector buffer_; -}; - class RequirementAnalyzer { public: ~RequirementAnalyzer(); @@ -119,38 +102,6 @@ struct FutureStoreArg : public ArgWrapper { bool read_only_; }; -BufferBuilder::BufferBuilder() -{ - // Reserve 4KB to minimize resizing while packing the arguments. - buffer_.reserve(4096); -} - -template -void BufferBuilder::pack(const T& value) -{ - pack_buffer(reinterpret_cast(&value), sizeof(T)); -} - -template -void BufferBuilder::pack(const std::vector& values) -{ - uint32_t size = values.size(); - pack(size); - pack_buffer(values.data(), size * sizeof(T)); -} - -void BufferBuilder::pack_buffer(const void* src, size_t size) -{ - auto tgt = buffer_.data() + buffer_.size(); - buffer_.resize(buffer_.size() + size); - memcpy(tgt, src, size); -} - -Legion::UntypedBuffer BufferBuilder::to_legion_buffer() const -{ - return Legion::UntypedBuffer(buffer_.data(), buffer_.size()); -} - RequirementAnalyzer::~RequirementAnalyzer() { for (auto& pair : requirements_) delete pair.first; @@ -212,10 +163,7 @@ RegionFieldArg::RegionFieldArg(RequirementAnalyzer* analyzer, void RegionFieldArg::pack(BufferBuilder& buffer) const { - buffer.pack(false); - buffer.pack(store_.dim()); - buffer.pack(store_.code()); - buffer.pack(-1); + store_.pack(buffer); buffer.pack(redop_); buffer.pack(dim_); @@ -238,10 +186,7 @@ struct datalen_fn { void FutureStoreArg::pack(BufferBuilder& buffer) const { - buffer.pack(true); - buffer.pack(store_.dim()); - buffer.pack(store_.code()); - buffer.pack(-1); + store_.pack(buffer); buffer.pack(read_only_); buffer.pack(true); diff --git a/src/core/utilities/buffer_builder.cc b/src/core/utilities/buffer_builder.cc new file mode 100644 index 0000000000..9068921e76 --- /dev/null +++ b/src/core/utilities/buffer_builder.cc @@ -0,0 +1,39 @@ +/* Copyright 2021 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/utilities/buffer_builder.h" + +namespace legate { + +BufferBuilder::BufferBuilder() +{ + // Reserve 4KB to minimize resizing while packing the arguments. + buffer_.reserve(4096); +} + +void BufferBuilder::pack_buffer(const void* src, size_t size) +{ + auto tgt = buffer_.data() + buffer_.size(); + buffer_.resize(buffer_.size() + size); + memcpy(tgt, src, size); +} + +Legion::UntypedBuffer BufferBuilder::to_legion_buffer() const +{ + return Legion::UntypedBuffer(buffer_.data(), buffer_.size()); +} + +} // namespace legate diff --git a/src/core/utilities/buffer_builder.h b/src/core/utilities/buffer_builder.h new file mode 100644 index 0000000000..9d3c149b9a --- /dev/null +++ b/src/core/utilities/buffer_builder.h @@ -0,0 +1,43 @@ +/* Copyright 2021 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include "legion.h" + +namespace legate { + +class BufferBuilder { + public: + BufferBuilder(); + + public: + template + void pack(const T& value); + template + void pack(const std::vector& values); + void pack_buffer(const void* buffer, size_t size); + + public: + Legion::UntypedBuffer to_legion_buffer() const; + + private: + std::vector buffer_; +}; + +} // namespace legate + +#include "core/utilities/buffer_builder.inl" diff --git a/src/core/utilities/buffer_builder.inl b/src/core/utilities/buffer_builder.inl new file mode 100644 index 0000000000..664b719582 --- /dev/null +++ b/src/core/utilities/buffer_builder.inl @@ -0,0 +1,33 @@ +/* Copyright 2021 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +namespace legate { + +template +void BufferBuilder::pack(const T& value) +{ + pack_buffer(reinterpret_cast(&value), sizeof(T)); +} + +template +void BufferBuilder::pack(const std::vector& values) +{ + uint32_t size = values.size(); + pack(size); + pack_buffer(values.data(), size * sizeof(T)); +} + +} // namespace legate From f90e725c05e4e4041eb5ac738464c0a1c8f8c6a4 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Thu, 11 Nov 2021 10:06:15 -0800 Subject: [PATCH 0030/1425] Changes to compute partitions and projections correctly for transformed stores --- src/core/data/logical_store.cc | 64 +++++++++++++++++++++-- src/core/data/transform.cc | 63 +++++++++++++++++++++++ src/core/data/transform.h | 21 ++++++-- src/core/legate_c.h | 2 +- src/core/mapping/base_mapper.cc | 10 ++-- src/core/partitioning/partition.cc | 81 ++++++------------------------ src/core/partitioning/partition.h | 79 +++++++++++++++++++++++++++++ src/core/runtime/projection.cc | 12 ++--- src/core/runtime/runtime.cc | 14 ++++++ src/core/runtime/runtime.h | 10 +++- 10 files changed, 269 insertions(+), 87 deletions(-) diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index 543a7d1987..f441d080bd 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -14,6 +14,8 @@ * */ +#include + #include "core/data/logical_store.h" #include "core/data/store.h" #include "core/partitioning/partition.h" @@ -93,6 +95,11 @@ class LogicalStore { std::unique_ptr find_or_create_partition(const Partition* partition); std::unique_ptr find_or_create_key_partition(); + private: + std::unique_ptr invert_partition(const Partition* partition) const; + void invert_dimensions(std::vector& dims) const; + Legion::ProjectionID compute_projection() const; + public: void pack(BufferBuilder& buffer) const; @@ -216,16 +223,67 @@ std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) return mapped_; } +std::unique_ptr LogicalStore::invert_partition(const Partition* partition) const +{ + if (nullptr == parent_) { + switch (partition->kind()) { + case Partition::Kind::NO_PARTITION: { + return create_no_partition(runtime_); + } + case Partition::Kind::TILING: { + auto tiling = static_cast(partition); + Shape tile_shape = tiling->tile_shape(); + Shape color_shape = tiling->color_shape(); + Shape offsets = tiling->offsets(); + return create_tiling( + runtime_, std::move(tile_shape), std::move(color_shape), std::move(offsets)); + } + } + } else { + auto inverted = transform_->invert_partition(partition); + return parent_->invert_partition(inverted.get()); + } + assert(false); + return nullptr; +} + +void LogicalStore::invert_dimensions(std::vector& dims) const +{ + if (nullptr == parent_) return; + transform_->invert_dimensions(dims); + parent_->invert_dimensions(dims); +} + +Legion::ProjectionID LogicalStore::compute_projection() const +{ + auto ndim = dim(); + std::vector dims(ndim); + std::iota(dims.begin(), dims.end(), 0); + invert_dimensions(dims); + + bool identity_mapping = dims.size() == ndim; + for (int32_t dim = 0; dim < ndim && identity_mapping; ++dim) + if (dims[dim] != dim) identity_mapping = false; + + if (identity_mapping) + return 0; + else + return runtime_->get_projection(ndim, dims); +} + std::unique_ptr LogicalStore::find_or_create_partition(const Partition* partition) { if (scalar_) return std::make_unique(); // We're about to create a legion partition for this store, so the store should have its region // created. + auto proj = compute_projection(); + auto inverted = invert_partition(partition); + auto lr = get_storage()->region(); - auto lp = partition->construct( - lr, partition->is_disjoint_for(nullptr), partition->is_complete_for(nullptr)); - return std::make_unique(lp, 0); + auto lp = + inverted->construct(lr, inverted->is_disjoint_for(nullptr), inverted->is_complete_for(nullptr)); + return std::make_unique(lp, proj); } std::unique_ptr LogicalStore::find_or_create_key_partition() diff --git a/src/core/data/transform.cc b/src/core/data/transform.cc index fc5f850abd..1affcd26a2 100644 --- a/src/core/data/transform.cc +++ b/src/core/data/transform.cc @@ -16,6 +16,7 @@ #include "core/data/transform.h" #include "core/legate_c.h" +#include "core/partitioning/partition.h" #include "core/utilities/buffer_builder.h" namespace legate { @@ -82,6 +83,13 @@ DomainAffineTransform Shift::inverse_transform(int32_t in_dim) const return result; } +std::unique_ptr Shift::invert_partition(const Partition* partition) const +{ + return nullptr; +} + +void Shift::invert_dimensions(std::vector& dims) const {} + void Shift::pack(BufferBuilder& buffer) const { buffer.pack(LEGATE_CORE_TRANSFORM_SHIFT); @@ -157,6 +165,40 @@ DomainAffineTransform Promote::inverse_transform(int32_t in_dim) const return result; } +static Shape remove(const Shape& shape, int32_t to_remove) +{ + Shape new_shape; + + for (int32_t idx = 0; idx < to_remove; ++idx) new_shape.push_back(shape[idx]); + for (int32_t idx = to_remove + 1; idx < static_cast(shape.size()); ++idx) + new_shape.push_back(shape[idx]); + + return std::move(new_shape); +} + +std::unique_ptr Promote::invert_partition(const Partition* partition) const +{ + switch (partition->kind()) { + case Partition::Kind::NO_PARTITION: { + return create_no_partition(partition->runtime()); + } + case Partition::Kind::TILING: { + auto tiling = static_cast(partition); + return create_tiling(tiling->runtime(), + remove(tiling->tile_shape(), extra_dim_), + remove(tiling->color_shape(), extra_dim_), + remove(tiling->offsets(), extra_dim_)); + } + } + assert(false); + return nullptr; +} + +void Promote::invert_dimensions(std::vector& dims) const +{ + dims.erase(dims.begin() + extra_dim_); +} + void Promote::pack(BufferBuilder& buffer) const { buffer.pack(LEGATE_CORE_TRANSFORM_PROMOTE); @@ -232,6 +274,13 @@ DomainAffineTransform Project::inverse_transform(int32_t in_dim) const return result; } +std::unique_ptr Project::invert_partition(const Partition* partition) const +{ + return nullptr; +} + +void Project::invert_dimensions(std::vector& dims) const {} + void Project::pack(BufferBuilder& buffer) const { buffer.pack(LEGATE_CORE_TRANSFORM_PROJECT); @@ -296,6 +345,13 @@ DomainAffineTransform Transpose::inverse_transform(int32_t in_dim) const return result; } +std::unique_ptr Transpose::invert_partition(const Partition* partition) const +{ + return nullptr; +} + +void Transpose::invert_dimensions(std::vector& dims) const {} + namespace { // anonymous template void print_vector(std::ostream& out, const std::vector& vec) @@ -404,6 +460,13 @@ DomainAffineTransform Delinearize::inverse_transform(int32_t in_dim) const return result; } +std::unique_ptr Delinearize::invert_partition(const Partition* partition) const +{ + return nullptr; +} + +void Delinearize::invert_dimensions(std::vector& dims) const {} + void Delinearize::pack(BufferBuilder& buffer) const { buffer.pack(LEGATE_CORE_TRANSFORM_DELINEARIZE); diff --git a/src/core/data/transform.h b/src/core/data/transform.h index 619f4ab292..9e12938757 100644 --- a/src/core/data/transform.h +++ b/src/core/data/transform.h @@ -23,6 +23,7 @@ namespace legate { class BufferBuilder; +class Partition; class StoreTransform { public: @@ -31,10 +32,12 @@ class StoreTransform { virtual ~StoreTransform() {} public: - virtual Legion::Domain transform(const Legion::Domain& input) const = 0; - virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const = 0; - virtual void pack(BufferBuilder& buffer) const = 0; - virtual void print(std::ostream& out) const = 0; + virtual Legion::Domain transform(const Legion::Domain& input) const = 0; + virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const = 0; + virtual std::unique_ptr invert_partition(const Partition* partition) const = 0; + virtual void invert_dimensions(std::vector& dims) const = 0; + virtual void pack(BufferBuilder& buffer) const = 0; + virtual void print(std::ostream& out) const = 0; protected: std::shared_ptr parent_{nullptr}; @@ -50,6 +53,8 @@ class Shift : public StoreTransform { public: virtual Legion::Domain transform(const Legion::Domain& input) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; + virtual std::unique_ptr invert_partition(const Partition* partition) const override; + virtual void invert_dimensions(std::vector& dims) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; @@ -66,6 +71,8 @@ class Promote : public StoreTransform { public: virtual Legion::Domain transform(const Legion::Domain& input) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; + virtual std::unique_ptr invert_partition(const Partition* partition) const override; + virtual void invert_dimensions(std::vector& dims) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; @@ -82,6 +89,8 @@ class Project : public StoreTransform { public: virtual Legion::Domain transform(const Legion::Domain& domain) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; + virtual std::unique_ptr invert_partition(const Partition* partition) const override; + virtual void invert_dimensions(std::vector& dims) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; @@ -98,6 +107,8 @@ class Transpose : public StoreTransform { public: virtual Legion::Domain transform(const Legion::Domain& domain) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; + virtual std::unique_ptr invert_partition(const Partition* partition) const override; + virtual void invert_dimensions(std::vector& dims) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; @@ -115,6 +126,8 @@ class Delinearize : public StoreTransform { public: virtual Legion::Domain transform(const Legion::Domain& domain) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; + virtual std::unique_ptr invert_partition(const Partition* partition) const override; + virtual void invert_dimensions(std::vector& dims) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; diff --git a/src/core/legate_c.h b/src/core/legate_c.h index a7ab191ff5..2413541884 100644 --- a/src/core/legate_c.h +++ b/src/core/legate_c.h @@ -104,7 +104,7 @@ void legate_shutdown(void); void legate_core_perform_registration(void); -void legate_register_projection_functor(int32_t, int32_t, int32_t*, legion_projection_id_t); +void legate_register_projection_functor(int32_t, int32_t, const int32_t*, legion_projection_id_t); void legate_create_sharding_functor_using_projection(legion_sharding_id_t, legion_projection_id_t); diff --git a/src/core/mapping/base_mapper.cc b/src/core/mapping/base_mapper.cc index 9f679de9ec..a526c6cb68 100644 --- a/src/core/mapping/base_mapper.cc +++ b/src/core/mapping/base_mapper.cc @@ -1419,7 +1419,7 @@ void BaseMapper::select_sharding_functor(const MapperContext ctx, } void BaseMapper::select_partition_projection(const MapperContext ctx, - const Partition& partition, + const Legion::Partition& partition, const SelectPartitionProjectionInput& input, SelectPartitionProjectionOutput& output) { @@ -1431,7 +1431,7 @@ void BaseMapper::select_partition_projection(const MapperContext ctx, } void BaseMapper::map_partition(const MapperContext ctx, - const Partition& partition, + const Legion::Partition& partition, const MapPartitionInput& input, MapPartitionOutput& output) { @@ -1487,7 +1487,7 @@ void BaseMapper::map_partition(const MapperContext ctx, } void BaseMapper::select_partition_sources(const MapperContext ctx, - const Partition& partition, + const Legion::Partition& partition, const SelectPartitionSrcInput& input, SelectPartitionSrcOutput& output) { @@ -1495,7 +1495,7 @@ void BaseMapper::select_partition_sources(const MapperContext ctx, } void BaseMapper::report_profiling(const MapperContext ctx, - const Partition& partition, + const Legion::Partition& partition, const PartitionProfilingInfo& input) { // No profiling yet @@ -1503,7 +1503,7 @@ void BaseMapper::report_profiling(const MapperContext ctx, } void BaseMapper::select_sharding_functor(const MapperContext ctx, - const Partition& partition, + const Legion::Partition& partition, const SelectShardingFunctorInput& input, SelectShardingFunctorOutput& output) { diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index a54288c872..b2295c2d06 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -23,8 +23,6 @@ namespace legate { -using Shape = std::vector; - std::string to_string(const Shape& shape) { std::stringstream ss; @@ -34,59 +32,6 @@ std::string to_string(const Shape& shape) return ss.str(); } -class NoPartition : public Partition { - public: - NoPartition(Runtime* runtime); - - public: - virtual bool is_complete_for(const LogicalStore* store) const override; - virtual bool is_disjoint_for(const LogicalStore* store) const override; - - public: - virtual Legion::LogicalPartition construct(Legion::LogicalRegion region, - bool disjoint, - bool complete) const override; - virtual std::unique_ptr get_projection(LogicalStore store) const override; - - public: - virtual bool has_launch_domain() const override; - virtual Legion::Domain launch_domain() const override; - - public: - virtual std::string to_string() const override; - - private: - Runtime* runtime_; -}; - -class Tiling : public Partition { - public: - Tiling(Runtime* runtime, Shape&& tile_shape, Shape&& color_shape, Shape&& offsets); - - public: - virtual bool is_complete_for(const LogicalStore* store) const override; - virtual bool is_disjoint_for(const LogicalStore* store) const override; - - public: - virtual Legion::LogicalPartition construct(Legion::LogicalRegion region, - bool disjoint, - bool complete) const override; - virtual std::unique_ptr get_projection(LogicalStore store) const override; - - public: - virtual bool has_launch_domain() const override; - virtual Legion::Domain launch_domain() const override; - - public: - virtual std::string to_string() const override; - - private: - Runtime* runtime_; - Shape tile_shape_; - Shape color_shape_; - Shape offsets_; -}; - struct PartitionByRestriction : public PartitioningFunctor { public: PartitionByRestriction(Legion::DomainTransform transform, Legion::Domain extent); @@ -103,19 +48,9 @@ struct PartitionByRestriction : public PartitioningFunctor { Legion::Domain extent_; }; -Tiling::Tiling(Runtime* runtime, Shape&& tile_shape, Shape&& color_shape, Shape&& offsets) - : runtime_(runtime), - tile_shape_(std::forward(tile_shape)), - color_shape_(std::forward(color_shape)), - offsets_(std::forward(offsets)) -{ - if (offsets_.empty()) - for (auto _ : tile_shape_) offsets_.push_back(0); - assert(tile_shape_.size() == color_shape_.size()); - assert(tile_shape_.size() == offsets_.size()); -} +Partition::Partition(Runtime* runtime) : runtime_(runtime) {} -NoPartition::NoPartition(Runtime* runtime) : runtime_(runtime) {} +NoPartition::NoPartition(Runtime* runtime) : Partition(runtime) {} bool NoPartition::is_complete_for(const LogicalStore* store) const { return false; } @@ -143,6 +78,18 @@ Legion::Domain NoPartition::launch_domain() const std::string NoPartition::to_string() const { return "NoPartition"; } +Tiling::Tiling(Runtime* runtime, Shape&& tile_shape, Shape&& color_shape, Shape&& offsets) + : Partition(runtime), + tile_shape_(std::forward(tile_shape)), + color_shape_(std::forward(color_shape)), + offsets_(std::forward(offsets)) +{ + if (offsets_.empty()) + for (auto _ : tile_shape_) offsets_.push_back(0); + assert(tile_shape_.size() == color_shape_.size()); + assert(tile_shape_.size() == offsets_.size()); +} + bool Tiling::is_complete_for(const LogicalStore* store) const { return false; } bool Tiling::is_disjoint_for(const LogicalStore* store) const { return true; } diff --git a/src/core/partitioning/partition.h b/src/core/partitioning/partition.h index d9e4b40610..1d6104e4a6 100644 --- a/src/core/partitioning/partition.h +++ b/src/core/partitioning/partition.h @@ -26,10 +26,22 @@ class LogicalStore; class Projection; class Runtime; +using Shape = std::vector; + struct Partition { public: + enum class Kind : int32_t { + NO_PARTITION = 0, + TILING = 1, + }; + + public: + Partition(Runtime* runtime); virtual ~Partition() {} + public: + virtual Kind kind() const = 0; + public: virtual bool is_complete_for(const LogicalStore* store) const = 0; virtual bool is_disjoint_for(const LogicalStore* store) const = 0; @@ -45,7 +57,11 @@ struct Partition { virtual Legion::Domain launch_domain() const = 0; public: + Runtime* runtime() const { return runtime_; } virtual std::string to_string() const = 0; + + protected: + Runtime* runtime_; }; struct PartitioningFunctor { @@ -57,6 +73,69 @@ struct PartitioningFunctor { Legion::PartitionKind kind) const = 0; }; +class NoPartition : public Partition { + public: + NoPartition(Runtime* runtime); + + public: + virtual Kind kind() const override { return Kind::NO_PARTITION; } + + public: + virtual bool is_complete_for(const LogicalStore* store) const override; + virtual bool is_disjoint_for(const LogicalStore* store) const override; + + public: + virtual Legion::LogicalPartition construct(Legion::LogicalRegion region, + bool disjoint, + bool complete) const override; + virtual std::unique_ptr get_projection(LogicalStore store) const override; + + public: + virtual bool has_launch_domain() const override; + virtual Legion::Domain launch_domain() const override; + + public: + virtual std::string to_string() const override; + + private: + Runtime* runtime_; +}; + +class Tiling : public Partition { + public: + Tiling(Runtime* runtime, Shape&& tile_shape, Shape&& color_shape, Shape&& offsets); + + public: + virtual Kind kind() const override { return Kind::TILING; } + + public: + virtual bool is_complete_for(const LogicalStore* store) const override; + virtual bool is_disjoint_for(const LogicalStore* store) const override; + + public: + virtual Legion::LogicalPartition construct(Legion::LogicalRegion region, + bool disjoint, + bool complete) const override; + virtual std::unique_ptr get_projection(LogicalStore store) const override; + + public: + virtual bool has_launch_domain() const override; + virtual Legion::Domain launch_domain() const override; + + public: + virtual std::string to_string() const override; + + public: + const Shape& tile_shape() const { return tile_shape_; } + const Shape& color_shape() const { return color_shape_; } + const Shape& offsets() const { return offsets_; } + + private: + Shape tile_shape_; + Shape color_shape_; + Shape offsets_; +}; + std::unique_ptr create_no_partition(Runtime* runtime); std::unique_ptr create_tiling(Runtime* runtime, diff --git a/src/core/runtime/projection.cc b/src/core/runtime/projection.cc index 88ab4fc809..76063f0880 100644 --- a/src/core/runtime/projection.cc +++ b/src/core/runtime/projection.cc @@ -98,7 +98,7 @@ LogicalRegion LegateProjectionFunctor::project(LogicalPartition upper_bound, template class ReductionFunctor : public LegateProjectionFunctor { public: - ReductionFunctor(Runtime* runtime, int32_t* dims); + ReductionFunctor(Runtime* runtime, const int32_t* dims); public: virtual DomainPoint project_point(const DomainPoint& point, @@ -108,21 +108,21 @@ class ReductionFunctor : public LegateProjectionFunctor { } public: - static Transform create_transform(int32_t* dims); + static Transform create_transform(const int32_t* dims); private: const Transform transform_; }; template -ReductionFunctor::ReductionFunctor(Runtime* runtime, int32_t* dims) +ReductionFunctor::ReductionFunctor(Runtime* runtime, const int32_t* dims) : LegateProjectionFunctor(runtime), transform_(create_transform(dims)) { } template /*static*/ Transform ReductionFunctor::create_transform( - int32_t* dims) + const int32_t* dims) { Transform transform; @@ -142,7 +142,7 @@ static std::mutex functor_table_lock; struct create_reduction_functor_fn { template - void operator()(Runtime* runtime, int32_t* dims, ProjectionID proj_id) + void operator()(Runtime* runtime, const int32_t* dims, ProjectionID proj_id) { auto functor = new ReductionFunctor(runtime, dims); runtime->register_projection_functor(proj_id, functor, true /*silence warnings*/); @@ -165,7 +165,7 @@ extern "C" { void legate_register_projection_functor(int32_t src_ndim, int32_t tgt_ndim, - int32_t* dims, + const int32_t* dims, legion_projection_id_t proj_id) { auto runtime = Runtime::get_runtime(); diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 77980bd0da..c62201d533 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -770,6 +770,20 @@ std::shared_ptr Runtime::dispatch(IndexTaskLauncher* launcher) return nullptr; } +Legion::ProjectionID Runtime::get_projection(int32_t src_ndim, const std::vector& dims) +{ + ProjectionDesc key(src_ndim, dims); + auto finder = registered_projections_.find(key); + if (registered_projections_.end() != finder) return finder->second; + + auto proj_id = core_context_->get_projection_id(next_projection_id_++); + legate_register_projection_functor( + src_ndim, static_cast(dims.size()), dims.data(), proj_id); + registered_projections_[key] = proj_id; + + return proj_id; +} + /*static*/ void Runtime::initialize(int32_t argc, char** argv) { Legion::Runtime::initialize(&argc, &argv, true /*filter legion and realm args*/); diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 68541630f1..503072fbbd 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -21,6 +21,7 @@ #include "legion.h" #include "core/data/store.h" +#include "core/legate_c.h" #include "core/runtime/context.h" #include "core/utilities/typedefs.h" @@ -71,7 +72,6 @@ class PartitionManager { }; class Runtime { - public: public: Runtime(Legion::Runtime* legion_runtime); ~Runtime(); @@ -129,6 +129,9 @@ class Runtime { std::shared_ptr dispatch(Legion::TaskLauncher* launcher); std::shared_ptr dispatch(Legion::IndexTaskLauncher* launcher); + public: + Legion::ProjectionID get_projection(int32_t src_ndim, const std::vector& dims); + private: void schedule(std::vector> operations); @@ -154,6 +157,11 @@ class Runtime { private: std::map index_spaces_; + private: + using ProjectionDesc = std::pair>; + int64_t next_projection_id_{LEGATE_CORE_FIRST_DYNAMIC_FUNCTOR_ID}; + std::map registered_projections_; + private: std::vector> operations_; size_t window_size_{1}; From 99166b01b7ff3fdfd2a172d76c134487f25e382e Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Thu, 11 Nov 2021 12:13:39 -0800 Subject: [PATCH 0031/1425] Start using a custom tuple implementation for shape information --- src/core.mk | 2 + src/core/data/logical_store.cc | 32 ++--- src/core/data/logical_store.h | 2 +- src/core/data/transform.cc | 30 ++--- src/core/data/transform.h | 13 +- src/core/partitioning/partition.cc | 15 +-- src/core/partitioning/partition.h | 9 +- src/core/runtime/runtime.cc | 22 ++-- src/core/runtime/runtime.h | 10 +- src/core/utilities/tuple.h | 91 ++++++++++++++ src/core/utilities/tuple.inl | 191 +++++++++++++++++++++++++++++ 11 files changed, 334 insertions(+), 83 deletions(-) create mode 100644 src/core/utilities/tuple.h create mode 100644 src/core/utilities/tuple.inl diff --git a/src/core.mk b/src/core.mk index 473ed89ee8..f16a174900 100644 --- a/src/core.mk +++ b/src/core.mk @@ -74,5 +74,7 @@ INSTALL_HEADERS = legate.h \ core/utilities/dispatch.h \ core/utilities/machine.h \ core/utilities/span.h \ + core/utilities/tuple.inl \ + core/utilities/tuple.h \ core/utilities/type_traits.h \ core/utilities/typedefs.h diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index f441d080bd..24df8feda4 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -51,7 +51,7 @@ class LogicalStore { LogicalStore(); LogicalStore(Runtime* runtime, LegateTypeCode code, - std::vector extents, + tuple extents, std::shared_ptr parent, std::shared_ptr transform); LogicalStore(Runtime* runtime, LegateTypeCode code, const void* data); @@ -97,7 +97,7 @@ class LogicalStore { private: std::unique_ptr invert_partition(const Partition* partition) const; - void invert_dimensions(std::vector& dims) const; + void invert_dimensions(tuple& dims) const; Legion::ProjectionID compute_projection() const; public: @@ -110,7 +110,7 @@ class LogicalStore { bool scalar_{false}; Runtime* runtime_{nullptr}; LegateTypeCode code_{MAX_TYPE_NUMBER}; - std::vector extents_; + tuple extents_; std::shared_ptr region_field_{nullptr}; Legion::Future future_{}; std::shared_ptr parent_{nullptr}; @@ -120,7 +120,7 @@ class LogicalStore { LogicalStore::LogicalStore(Runtime* runtime, LegateTypeCode code, - std::vector extents, + tuple extents, std::shared_ptr parent, std::shared_ptr transform) : runtime_(runtime), @@ -158,14 +158,9 @@ Domain LogicalStore::domain() const return region_field_->domain(); } -const std::vector& LogicalStore::extents() const { return extents_; } +const std::vector& LogicalStore::extents() const { return extents_.data(); } -size_t LogicalStore::volume() const -{ - size_t vol = 1; - for (auto extent : extents_) vol *= extent; - return vol; -} +size_t LogicalStore::volume() const { return extents_.volume(); } bool LogicalStore::has_storage() const { return nullptr != region_field_; } @@ -202,13 +197,9 @@ std::shared_ptr LogicalStore::promote(int32_t extra_dim, "Invalid promotion on dimension %d for a %zd-D store", extra_dim, extents_.size()); LEGATE_ABORT } - std::vector new_extents; - for (int32_t dim = 0; dim < extra_dim; ++dim) new_extents.push_back(extents_[dim]); - new_extents.push_back(dim_size); - for (int32_t dim = extra_dim; dim < static_cast(extents_.size()); ++dim) - new_extents.push_back(extents_[dim]); - auto transform = std::make_shared(extra_dim, dim_size); + auto new_extents = extents_.insert(extra_dim, dim_size); + auto transform = std::make_shared(extra_dim, dim_size); return std::make_shared( runtime_, code_, std::move(new_extents), std::move(parent), std::move(transform)); } @@ -247,7 +238,7 @@ std::unique_ptr LogicalStore::invert_partition(const Partition* parti return nullptr; } -void LogicalStore::invert_dimensions(std::vector& dims) const +void LogicalStore::invert_dimensions(tuple& dims) const { if (nullptr == parent_) return; transform_->invert_dimensions(dims); @@ -257,8 +248,7 @@ void LogicalStore::invert_dimensions(std::vector& dims) const Legion::ProjectionID LogicalStore::compute_projection() const { auto ndim = dim(); - std::vector dims(ndim); - std::iota(dims.begin(), dims.end(), 0); + auto dims = from_range(ndim); invert_dimensions(dims); bool identity_mapping = dims.size() == ndim; @@ -324,7 +314,7 @@ LogicalStore::LogicalStore() {} LogicalStore::LogicalStore(Runtime* runtime, LegateTypeCode code, - std::vector extents, + tuple extents, LogicalStore parent, /* = LogicalStore() */ std::shared_ptr transform /*= nullptr*/) : impl_(std::make_shared( diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index 1b123da899..079c76e9ea 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -71,7 +71,7 @@ class LogicalStore { LogicalStore(); LogicalStore(Runtime* runtime, LegateTypeCode code, - std::vector extents, + tuple extents, LogicalStore parent = LogicalStore(), std::shared_ptr transform = nullptr); // Creates a read-only store from a scalar diff --git a/src/core/data/transform.cc b/src/core/data/transform.cc index 1affcd26a2..2082a649cb 100644 --- a/src/core/data/transform.cc +++ b/src/core/data/transform.cc @@ -88,7 +88,7 @@ std::unique_ptr Shift::invert_partition(const Partition* partition) c return nullptr; } -void Shift::invert_dimensions(std::vector& dims) const {} +void Shift::invert_dimensions(tuple& dims) const {} void Shift::pack(BufferBuilder& buffer) const { @@ -165,17 +165,6 @@ DomainAffineTransform Promote::inverse_transform(int32_t in_dim) const return result; } -static Shape remove(const Shape& shape, int32_t to_remove) -{ - Shape new_shape; - - for (int32_t idx = 0; idx < to_remove; ++idx) new_shape.push_back(shape[idx]); - for (int32_t idx = to_remove + 1; idx < static_cast(shape.size()); ++idx) - new_shape.push_back(shape[idx]); - - return std::move(new_shape); -} - std::unique_ptr Promote::invert_partition(const Partition* partition) const { switch (partition->kind()) { @@ -185,19 +174,16 @@ std::unique_ptr Promote::invert_partition(const Partition* partition) case Partition::Kind::TILING: { auto tiling = static_cast(partition); return create_tiling(tiling->runtime(), - remove(tiling->tile_shape(), extra_dim_), - remove(tiling->color_shape(), extra_dim_), - remove(tiling->offsets(), extra_dim_)); + tiling->tile_shape().remove(extra_dim_), + tiling->color_shape().remove(extra_dim_), + tiling->offsets().remove(extra_dim_)); } } assert(false); return nullptr; } -void Promote::invert_dimensions(std::vector& dims) const -{ - dims.erase(dims.begin() + extra_dim_); -} +void Promote::invert_dimensions(tuple& dims) const { dims.remove_inplace(extra_dim_); } void Promote::pack(BufferBuilder& buffer) const { @@ -279,7 +265,7 @@ std::unique_ptr Project::invert_partition(const Partition* partition) return nullptr; } -void Project::invert_dimensions(std::vector& dims) const {} +void Project::invert_dimensions(tuple& dims) const {} void Project::pack(BufferBuilder& buffer) const { @@ -350,7 +336,7 @@ std::unique_ptr Transpose::invert_partition(const Partition* partitio return nullptr; } -void Transpose::invert_dimensions(std::vector& dims) const {} +void Transpose::invert_dimensions(tuple& dims) const {} namespace { // anonymous template @@ -465,7 +451,7 @@ std::unique_ptr Delinearize::invert_partition(const Partition* partit return nullptr; } -void Delinearize::invert_dimensions(std::vector& dims) const {} +void Delinearize::invert_dimensions(tuple& dims) const {} void Delinearize::pack(BufferBuilder& buffer) const { diff --git a/src/core/data/transform.h b/src/core/data/transform.h index 9e12938757..f0158bb041 100644 --- a/src/core/data/transform.h +++ b/src/core/data/transform.h @@ -18,6 +18,7 @@ #include +#include "core/utilities/tuple.h" #include "legion.h" namespace legate { @@ -35,7 +36,7 @@ class StoreTransform { virtual Legion::Domain transform(const Legion::Domain& input) const = 0; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const = 0; virtual std::unique_ptr invert_partition(const Partition* partition) const = 0; - virtual void invert_dimensions(std::vector& dims) const = 0; + virtual void invert_dimensions(tuple& dims) const = 0; virtual void pack(BufferBuilder& buffer) const = 0; virtual void print(std::ostream& out) const = 0; @@ -54,7 +55,7 @@ class Shift : public StoreTransform { virtual Legion::Domain transform(const Legion::Domain& input) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; virtual std::unique_ptr invert_partition(const Partition* partition) const override; - virtual void invert_dimensions(std::vector& dims) const override; + virtual void invert_dimensions(tuple& dims) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; @@ -72,7 +73,7 @@ class Promote : public StoreTransform { virtual Legion::Domain transform(const Legion::Domain& input) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; virtual std::unique_ptr invert_partition(const Partition* partition) const override; - virtual void invert_dimensions(std::vector& dims) const override; + virtual void invert_dimensions(tuple& dims) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; @@ -90,7 +91,7 @@ class Project : public StoreTransform { virtual Legion::Domain transform(const Legion::Domain& domain) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; virtual std::unique_ptr invert_partition(const Partition* partition) const override; - virtual void invert_dimensions(std::vector& dims) const override; + virtual void invert_dimensions(tuple& dims) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; @@ -108,7 +109,7 @@ class Transpose : public StoreTransform { virtual Legion::Domain transform(const Legion::Domain& domain) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; virtual std::unique_ptr invert_partition(const Partition* partition) const override; - virtual void invert_dimensions(std::vector& dims) const override; + virtual void invert_dimensions(tuple& dims) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; @@ -127,7 +128,7 @@ class Delinearize : public StoreTransform { virtual Legion::Domain transform(const Legion::Domain& domain) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; virtual std::unique_ptr invert_partition(const Partition* partition) const override; - virtual void invert_dimensions(std::vector& dims) const override; + virtual void invert_dimensions(tuple& dims) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index b2295c2d06..ea3fd830ec 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -23,15 +23,6 @@ namespace legate { -std::string to_string(const Shape& shape) -{ - std::stringstream ss; - ss << "("; - for (auto extent : shape) ss << extent << ","; - ss << ")"; - return ss.str(); -} - struct PartitionByRestriction : public PartitioningFunctor { public: PartitionByRestriction(Legion::DomainTransform transform, Legion::Domain extent); @@ -84,8 +75,7 @@ Tiling::Tiling(Runtime* runtime, Shape&& tile_shape, Shape&& color_shape, Shape& color_shape_(std::forward(color_shape)), offsets_(std::forward(offsets)) { - if (offsets_.empty()) - for (auto _ : tile_shape_) offsets_.push_back(0); + if (offsets_.empty()) offsets_ = tuple(tile_shape_.size(), 0); assert(tile_shape_.size() == color_shape_.size()); assert(tile_shape_.size() == offsets_.size()); } @@ -153,8 +143,7 @@ Legion::Domain Tiling::launch_domain() const std::string Tiling::to_string() const { std::stringstream ss; - ss << "Tiling(tile:" << legate::to_string(tile_shape_) - << ",colors:" << legate::to_string(color_shape_) << ",offset:" << legate::to_string(offsets_) + ss << "Tiling(tile:" << tile_shape_ << ",colors:" << color_shape_ << ",offset:" << offsets_ << ")"; return ss.str(); } diff --git a/src/core/partitioning/partition.h b/src/core/partitioning/partition.h index 1d6104e4a6..061d544ef4 100644 --- a/src/core/partitioning/partition.h +++ b/src/core/partitioning/partition.h @@ -18,6 +18,7 @@ #include +#include "core/utilities/tuple.h" #include "legion.h" namespace legate { @@ -26,7 +27,7 @@ class LogicalStore; class Projection; class Runtime; -using Shape = std::vector; +using Shape = tuple; struct Partition { public: @@ -139,8 +140,8 @@ class Tiling : public Partition { std::unique_ptr create_no_partition(Runtime* runtime); std::unique_ptr create_tiling(Runtime* runtime, - std::vector&& tile_shape, - std::vector&& color_shape, - std::vector&& offsets = {}); + Shape&& tile_shape, + Shape&& color_shape, + Shape&& offsets = {}); } // namespace legate diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index c62201d533..5adc57919f 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -415,13 +415,13 @@ PartitionManager::PartitionManager(Runtime* runtime, const LibraryContext* conte push_factors(2); } -std::vector PartitionManager::compute_launch_shape(const std::vector& shape) +tuple PartitionManager::compute_launch_shape(const tuple& shape) { // Easy case if we only have one piece: no parallel launch space if (num_pieces_ == 1) return {}; // If we only have one point then we never do parallel launches - if (std::all_of(shape.begin(), shape.end(), [](auto extent) { return 1 == extent; })) return {}; + if (shape.all([](auto extent) { return 1 == extent; })) return {}; // Prune out any dimensions that are 1 std::vector temp_shape{}; @@ -537,18 +537,18 @@ std::vector PartitionManager::compute_launch_shape(const std::vector result(shape.size(), 1); for (uint32_t idx = 0; idx < ndim; ++idx) result[temp_dims[idx]] = temp_result[idx]; - return std::move(result); + return tuple(std::move(result)); } -std::vector PartitionManager::compute_tile_shape(const std::vector& extents, - const std::vector& launch_shape) +tuple PartitionManager::compute_tile_shape(const tuple& extents, + const tuple& launch_shape) { assert(extents.size() == launch_shape.size()); - std::vector tile_shape; + tuple tile_shape; for (uint32_t idx = 0; idx < extents.size(); ++idx) { auto x = extents[idx]; auto y = launch_shape[idx]; - tile_shape.push_back((x + y - 1) / y); + tile_shape.append_inplace((x + y - 1) / y); } return std::move(tile_shape); } @@ -641,7 +641,7 @@ LogicalStore Runtime::create_store(const Scalar& scalar) return LogicalStore(this, scalar.code(), scalar.ptr()); } -std::shared_ptr Runtime::create_region_field(const std::vector& extents, +std::shared_ptr Runtime::create_region_field(const tuple& extents, LegateTypeCode code) { DomainPoint lo, hi; @@ -770,15 +770,15 @@ std::shared_ptr Runtime::dispatch(IndexTaskLauncher* launcher) return nullptr; } -Legion::ProjectionID Runtime::get_projection(int32_t src_ndim, const std::vector& dims) +Legion::ProjectionID Runtime::get_projection(int32_t src_ndim, const tuple& dims) { - ProjectionDesc key(src_ndim, dims); + ProjectionDesc key(src_ndim, dims.data()); auto finder = registered_projections_.find(key); if (registered_projections_.end() != finder) return finder->second; auto proj_id = core_context_->get_projection_id(next_projection_id_++); legate_register_projection_functor( - src_ndim, static_cast(dims.size()), dims.data(), proj_id); + src_ndim, static_cast(dims.size()), key.second.data(), proj_id); registered_projections_[key] = proj_id; return proj_id; diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 503072fbbd..85edfa010b 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -23,6 +23,7 @@ #include "core/data/store.h" #include "core/legate_c.h" #include "core/runtime/context.h" +#include "core/utilities/tuple.h" #include "core/utilities/typedefs.h" namespace legate { @@ -61,9 +62,8 @@ class PartitionManager { PartitionManager(Runtime* runtime, const LibraryContext* context); public: - std::vector compute_launch_shape(const std::vector& shape); - std::vector compute_tile_shape(const std::vector& extents, - const std::vector& launch_shape); + tuple compute_launch_shape(const tuple& shape); + tuple compute_tile_shape(const tuple& extents, const tuple& launch_shape); private: int32_t num_pieces_; @@ -100,7 +100,7 @@ class Runtime { public: LogicalStore create_store(std::vector extents, LegateTypeCode code); LogicalStore create_store(const Scalar& scalar); - std::shared_ptr create_region_field(const std::vector& extents, + std::shared_ptr create_region_field(const tuple& extents, LegateTypeCode code); RegionField map_region_field(LibraryContext* context, std::shared_ptr region_field); @@ -130,7 +130,7 @@ class Runtime { std::shared_ptr dispatch(Legion::IndexTaskLauncher* launcher); public: - Legion::ProjectionID get_projection(int32_t src_ndim, const std::vector& dims); + Legion::ProjectionID get_projection(int32_t src_ndim, const tuple& dims); private: void schedule(std::vector> operations); diff --git a/src/core/utilities/tuple.h b/src/core/utilities/tuple.h new file mode 100644 index 0000000000..2e39bd14b1 --- /dev/null +++ b/src/core/utilities/tuple.h @@ -0,0 +1,91 @@ +/* Copyright 2021 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace legate { + +// A simple wrapper around an STL vector to provide common utilities +template +class tuple { + public: + tuple(); + tuple(const std::vector& values); + tuple(std::vector&& values); + tuple(std::initializer_list list); + tuple(size_t size, T init); + + public: + tuple(const tuple&) = default; + tuple(tuple&&) = default; + tuple& operator=(const tuple&) = default; + tuple& operator=(tuple&&) = default; + + public: + const T& operator[](uint32_t idx) const; + T& operator[](uint32_t idx); + + public: + bool empty() const; + size_t size() const; + + public: + tuple insert(int32_t pos, const T& value) const; + tuple append(const T& value) const; + tuple remove(int32_t pos) const; + + void insert_inplace(int32_t pos, const T& value); + void append_inplace(const T& value); + void remove_inplace(int32_t pos); + + public: + template + T reduce(FUNC func, const T& init) const; + T volume() const; + template + bool all(PRED pred) const; + template + bool any(PRED pred) const; + + public: + std::string to_string() const; + template + friend std::ostream& operator<<(std::ostream& out, const tuple<_T>& tpl); + + public: + const std::vector& data() const; + + private: + std::vector data_{}; +}; + +template +tuple from_range(T stop); + +template +tuple from_range(T start, T stop); + +} // namespace legate + +#include "core/utilities/tuple.inl" diff --git a/src/core/utilities/tuple.inl b/src/core/utilities/tuple.inl new file mode 100644 index 0000000000..e1b4411fa2 --- /dev/null +++ b/src/core/utilities/tuple.inl @@ -0,0 +1,191 @@ +/* Copyright 2021 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include +#include + +namespace legate { + +template +tuple::tuple() +{ +} + +template +tuple::tuple(const std::vector& values) : data_(values) +{ +} + +template +tuple::tuple(std::vector&& values) : data_(std::forward>(values)) +{ +} + +template +tuple::tuple(std::initializer_list list) : data_(list) +{ +} + +template +tuple::tuple(size_t size, T init) : data_(size, init) +{ +} + +template +const T& tuple::operator[](uint32_t idx) const +{ + return data_[idx]; +} + +template +T& tuple::operator[](uint32_t idx) +{ + return data_[idx]; +} + +template +bool tuple::empty() const +{ + return data_.empty(); +} + +template +size_t tuple::size() const +{ + return data_.size(); +} + +template +tuple tuple::insert(int32_t pos, const T& value) const +{ + std::vector new_values; + auto len = static_cast(data_.size()); + + for (int32_t idx = 0; idx < pos; ++idx) new_values.push_back(data_[idx]); + new_values.push_back(value); + for (int32_t idx = pos; idx < len; ++idx) new_values.push_back(data_[idx]); + return tuple(std::move(new_values)); +} + +template +tuple tuple::append(const T& value) const +{ + std::vector new_values(data_); + new_values.push_back(value); + return tuple(std::move(new_values)); +} + +template +tuple tuple::remove(int32_t pos) const +{ + std::vector new_values; + auto len = static_cast(data_.size()); + + for (int32_t idx = 0; idx < pos; ++idx) new_values.push_back(data_[idx]); + for (int32_t idx = pos + 1; idx < len; ++idx) new_values.push_back(data_[idx]); + return tuple(std::move(new_values)); +} + +template +void tuple::insert_inplace(int32_t pos, const T& value) +{ + auto oldlen = static_cast(data_.size()); + data_.resize(oldlen + 1); + + for (int32_t idx = oldlen; idx > pos; --idx) data_[idx + 1] = data[idx]; + data_[pos] = value; +} + +template +void tuple::append_inplace(const T& value) +{ + data_.push_back(value); +} + +template +void tuple::remove_inplace(int32_t pos) +{ + data_.erase(data_.begin() + pos); +} + +template +template +T tuple::reduce(FUNC func, const T& init) const +{ + T agg{init}; + for (auto value : data_) func(agg, value); + return agg; +} + +template +T tuple::volume() const +{ + return reduce(std::multiplies{}, T{1}); +} + +template +template +bool tuple::all(PRED pred) const +{ + return std::all_of(data_.begin(), data_.end(), pred); +} + +template +template +bool tuple::any(PRED pred) const +{ + return std::any_of(data_.begin(), data_.end(), pred); +} + +template +std::string tuple::to_string() const +{ + std::stringstream ss; + ss << *this; + return ss.str(); +} + +template +std::ostream& operator<<(std::ostream& out, const tuple<_T>& tpl) +{ + out << "("; + for (auto value : tpl.data_) out << value << ","; + out << ")"; + return out; +} + +template +const std::vector& tuple::data() const +{ + return data_; +} + +template +tuple from_range(T stop) +{ + return from_range(T{0}, stop); +} + +template +tuple from_range(T start, T stop) +{ + std::vector values(stop - start); + std::iota(values.begin(), values.end(), start); + return tuple(std::move(values)); +} + +} // namespace legate From 0466165b22430b4435e6558102451f4e06e9a295 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 12 Nov 2021 11:18:03 -0800 Subject: [PATCH 0032/1425] Add field recycling to the field manager --- src/core/data/logical_store.cc | 8 ++++++++ src/core/data/logical_store.h | 5 ----- src/core/data/store.cc | 9 +++++++++ src/core/data/store.h | 6 ++++++ src/core/runtime/launcher.cc | 13 +++++++++++++ src/core/runtime/runtime.cc | 34 +++++++++++++++++++++++++++++----- src/core/runtime/runtime.h | 1 + 7 files changed, 66 insertions(+), 10 deletions(-) diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index 24df8feda4..072cfbc256 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -56,6 +56,9 @@ class LogicalStore { std::shared_ptr transform); LogicalStore(Runtime* runtime, LegateTypeCode code, const void* data); + public: + ~LogicalStore(); + private: LogicalStore(std::shared_ptr impl); @@ -146,6 +149,11 @@ LogicalStore::LogicalStore(Runtime* runtime, LegateTypeCode code, const void* da future_ = runtime_->create_future(data, datalen); } +LogicalStore::~LogicalStore() +{ + if (mapped_ != nullptr) mapped_->unmap(); +} + bool LogicalStore::scalar() const { return scalar_; } int32_t LogicalStore::dim() const { return static_cast(extents_.size()); } diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index 079c76e9ea..04cc66ffd0 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -52,12 +52,7 @@ class LogicalRegionField { Legion::LogicalRegion region() const { return lr_; } Legion::FieldID field_id() const { return fid_; } - // public: - // RegionField map(); - public: - // template - // Legion::Rect shape() const; Legion::Domain domain() const; private: diff --git a/src/core/data/store.cc b/src/core/data/store.cc index 2fa1cb77e1..fc676eea09 100644 --- a/src/core/data/store.cc +++ b/src/core/data/store.cc @@ -15,6 +15,7 @@ */ #include "core/data/store.h" +#include "core/runtime/runtime.h" #include "core/utilities/dispatch.h" namespace legate { @@ -53,6 +54,8 @@ RegionField& RegionField::operator=(RegionField&& other) noexcept Domain RegionField::domain() const { return dim_dispatch(dim_, get_domain_fn{}, pr_); } +void RegionField::unmap() { Runtime::get_runtime()->unmap_physical_region(pr_); } + OutputRegionField::OutputRegionField(const OutputRegion& out, FieldID fid) : out_(out), fid_(fid) {} OutputRegionField::OutputRegionField(OutputRegionField&& other) noexcept @@ -218,4 +221,10 @@ Domain Store::domain() const return result; } +void Store::unmap() +{ + if (is_future_ || is_output_store_) return; + region_field_.unmap(); +} + } // namespace legate diff --git a/src/core/data/store.h b/src/core/data/store.h index 15471e3894..317d37ab60 100644 --- a/src/core/data/store.h +++ b/src/core/data/store.h @@ -140,6 +140,9 @@ class RegionField { Legion::Rect shape() const; Legion::Domain domain() const; + public: + void unmap(); + public: bool is_readable() const { return readable_; } bool is_writable() const { return writable_; } @@ -293,6 +296,9 @@ class Store { Legion::Rect shape() const; Legion::Domain domain() const; + public: + void unmap(); + public: bool is_readable() const { return readable_; } bool is_writable() const { return writable_; } diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/launcher.cc index 6fa1495c78..51892621d1 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -43,6 +43,7 @@ class RequirementAnalyzer { }; struct ArgWrapper { + virtual ~ArgWrapper() {} virtual void pack(BufferBuilder& buffer) const = 0; }; @@ -51,6 +52,9 @@ struct ScalarArg : public ArgWrapper { public: ScalarArg(const T& value) : value_(value) {} + public: + virtual ~ScalarArg() {} + public: virtual void pack(BufferBuilder& buffer) const override; @@ -62,6 +66,9 @@ struct UntypedScalarArg : public ArgWrapper { public: UntypedScalarArg(const Scalar& scalar) : scalar_(scalar) {} + public: + virtual ~UntypedScalarArg() {} + public: virtual void pack(BufferBuilder& buffer) const override; @@ -81,6 +88,9 @@ struct RegionFieldArg : public ArgWrapper { public: virtual void pack(BufferBuilder& buffer) const override; + public: + virtual ~RegionFieldArg() {} + private: RequirementAnalyzer* analyzer_; LogicalStore store_; @@ -94,6 +104,9 @@ struct FutureStoreArg : public ArgWrapper { public: FutureStoreArg(LogicalStore store, bool read_only); + public: + virtual ~FutureStoreArg() {} + public: virtual void pack(BufferBuilder& buffer) const override; diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 5adc57919f..7b6a56fc57 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -362,6 +362,10 @@ class FieldManager { Legion::Domain shape_; LegateTypeCode code_; size_t field_size_; + + private: + using FreeField = std::pair; + std::deque free_fields_; }; struct field_size_fn { @@ -381,11 +385,26 @@ FieldManager::FieldManager(Runtime* runtime, const Legion::Domain& shape, Legate std::shared_ptr FieldManager::allocate_field() { - auto rgn_mgr = runtime_->find_or_create_region_manager(shape_); - LogicalRegion lr; - FieldID fid; - std::tie(lr, fid) = rgn_mgr->allocate_field(field_size_); - return std::make_shared(runtime_, lr, fid); + LogicalRegionField* rf = nullptr; + if (!free_fields_.empty()) { + auto field = free_fields_.front(); + log_legate.debug("Field %u recycled in field manager %p", field.second, this); + free_fields_.pop_front(); + rf = new LogicalRegionField(runtime_, field.first, field.second); + } else { + auto rgn_mgr = runtime_->find_or_create_region_manager(shape_); + LogicalRegion lr; + FieldID fid; + std::tie(lr, fid) = rgn_mgr->allocate_field(field_size_); + rf = new LogicalRegionField(runtime_, lr, fid); + log_legate.debug("Field %u created in field manager %p", fid, this); + } + assert(rf != nullptr); + return std::shared_ptr(rf, [this](auto* field) { + log_legate.debug("Field %u freed in field manager %p", field->field_id(), this); + this->free_fields_.push_back(FreeField(field->region(), field->field_id())); + delete field; + }); } //////////////////////////////////////////////////// @@ -671,6 +690,11 @@ RegionField Runtime::map_region_field(LibraryContext* context, return RegionField(rf->dim(), pr, field_id); } +void Runtime::unmap_physical_region(Legion::PhysicalRegion pr) +{ + legion_runtime_->unmap_region(legion_context_, pr); +} + RegionManager* Runtime::find_or_create_region_manager(const Domain& shape) { auto finder = region_managers_.find(shape); diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 85edfa010b..c916f8536e 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -104,6 +104,7 @@ class Runtime { LegateTypeCode code); RegionField map_region_field(LibraryContext* context, std::shared_ptr region_field); + void unmap_physical_region(Legion::PhysicalRegion pr); public: RegionManager* find_or_create_region_manager(const Legion::Domain& shape); From 6ce06b71f469f6974ca05cfb373051ff2f0738c5 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 12 Nov 2021 15:51:35 -0800 Subject: [PATCH 0033/1425] Fix a typo --- src/core/utilities/tuple.inl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/utilities/tuple.inl b/src/core/utilities/tuple.inl index e1b4411fa2..7b0ac98052 100644 --- a/src/core/utilities/tuple.inl +++ b/src/core/utilities/tuple.inl @@ -106,7 +106,7 @@ void tuple::insert_inplace(int32_t pos, const T& value) auto oldlen = static_cast(data_.size()); data_.resize(oldlen + 1); - for (int32_t idx = oldlen; idx > pos; --idx) data_[idx + 1] = data[idx]; + for (int32_t idx = oldlen; idx > pos; --idx) data_[idx + 1] = data_[idx]; data_[pos] = value; } From b7245a7943ad361921573ca02f60191c75fa488f Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 12 Nov 2021 15:51:43 -0800 Subject: [PATCH 0034/1425] Add a virtual destructor --- src/core/runtime/operation.cc | 8 ++++++++ src/core/runtime/operation.h | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/src/core/runtime/operation.cc b/src/core/runtime/operation.cc index cec2b4776a..89ce7d978c 100644 --- a/src/core/runtime/operation.cc +++ b/src/core/runtime/operation.cc @@ -41,6 +41,10 @@ Operation::Operation(Runtime* runtime, { } +Operation::~Operation() +{ +} + void Operation::add_input(LogicalStore store, std::shared_ptr partition) { constraints_->add_variable(partition); @@ -92,6 +96,10 @@ Task::Task(Runtime* runtime, { } +Task::~Task() +{ +} + void Task::add_scalar_arg(const Scalar& scalar) { scalars_.push_back(scalar); } void Task::launch(Strategy* p_strategy) diff --git a/src/core/runtime/operation.h b/src/core/runtime/operation.h index d3969f3b3b..f8c7989733 100644 --- a/src/core/runtime/operation.h +++ b/src/core/runtime/operation.h @@ -36,6 +36,9 @@ class Operation { public: Operation(Runtime* runtime, LibraryContext* library, uint64_t unique_id, int64_t mapper_id); + public: + virtual ~Operation(); + public: void add_input(LogicalStore store, std::shared_ptr partition); void add_output(LogicalStore store, std::shared_ptr partition); @@ -86,6 +89,9 @@ class Task : public Operation { uint64_t unique_id, int64_t mapper_id = 0); + public: + ~Task(); + public: void add_scalar_arg(const Scalar& scalar); From 2040284d7c289d77ae5de3982102d067b202705a Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Mon, 7 Feb 2022 15:36:08 -0800 Subject: [PATCH 0035/1425] APIs to query launch shape --- src/core/runtime/context.cc | 6 ++++++ src/core/runtime/context.h | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/src/core/runtime/context.cc b/src/core/runtime/context.cc index 63ce792555..9a47f3e2b0 100644 --- a/src/core/runtime/context.cc +++ b/src/core/runtime/context.cc @@ -152,6 +152,12 @@ TaskContext::TaskContext(const Legion::Task* task, scalars_ = dez.unpack>(); } +bool TaskContext::is_single_task() const { return !task_->is_index_space; } + +Legion::DomainPoint TaskContext::get_task_index() const { return task_->index_point; } + +Legion::Domain TaskContext::get_launch_domain() const { return task_->index_domain; } + ReturnValues TaskContext::pack_return_values() const { std::vector return_values; diff --git a/src/core/runtime/context.h b/src/core/runtime/context.h index 92db6123f2..c46cddaa0c 100644 --- a/src/core/runtime/context.h +++ b/src/core/runtime/context.h @@ -118,6 +118,11 @@ class TaskContext { std::vector& reductions() { return reductions_; } std::vector& scalars() { return scalars_; } + public: + bool is_single_task() const; + Legion::DomainPoint get_task_index() const; + Legion::Domain get_launch_domain() const; + public: ReturnValues pack_return_values() const; From 53d7b6f9e930897d2e97d0579aa134f685f7e4d1 Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Mon, 7 Feb 2022 18:40:00 -0800 Subject: [PATCH 0036/1425] Update version. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b436816050..a19abc861b 100755 --- a/setup.py +++ b/setup.py @@ -70,7 +70,7 @@ def run(self): sys.argv.remove("--recurse") setup( name="legate.core", - version="0.1", + version="22.01", packages=["legate", "legate.core", "legate.timing"], cmdclass={"build_py": my_build_py}, ) From 14796ded2949cfd0bfd02945d788f023077e152e Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Mon, 7 Feb 2022 18:40:00 -0800 Subject: [PATCH 0037/1425] Release changes --- install.py | 20 +++++++++++++++----- setup.py | 6 +++--- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/install.py b/install.py index 58fddaca6e..955ab13b83 100755 --- a/install.py +++ b/install.py @@ -132,6 +132,7 @@ def git_clone(repo_dir, url, branch=None, tag=None, commit=None): verbose_check_call( ["git", "submodule", "update", "--init"], cwd=repo_dir ) + git_reset(repo_dir, commit) else: verbose_check_call( [ @@ -152,10 +153,13 @@ def git_reset(repo_dir, refspec): verbose_check_call(["git", "reset", "--hard", refspec], cwd=repo_dir) -def git_update(repo_dir, branch=None): +def git_update(repo_dir, branch=None, tag=None, commit=None): if branch is not None: verbose_check_call(["git", "checkout", branch], cwd=repo_dir) - verbose_check_call(["git", "pull", "--ff-only"], cwd=repo_dir) + verbose_check_call(["git", "pull", "--ff-only"], cwd=repo_dir) + else: + verbose_check_call(["git", "fetch"], cwd=repo_dir) + verbose_check_call(["git", "checkout", commit or tag], cwd=repo_dir) def load_json_config(filename): @@ -199,13 +203,14 @@ def install_gasnet(gasnet_dir, conduit, thread_count): shutil.rmtree(temp_dir) -def install_legion(legion_src_dir, branch): +def install_legion(legion_src_dir, branch, commit="c7ca2dbc"): print("Legate is installing Legion into a local directory...") # For now all we have to do is clone legion since we build it with Legate git_clone( legion_src_dir, url="https://gitlab.com/StanfordLegion/legion.git", branch=branch, + commit=commit, ) @@ -218,9 +223,9 @@ def install_thrust(thrust_dir): ) -def update_legion(legion_src_dir, branch): +def update_legion(legion_src_dir, branch, commit="c7ca2dbc"): # Make sure we are on the right branch for single/multi-node - git_update(legion_src_dir, branch=branch) + git_update(legion_src_dir, branch=branch, commit=commit) def build_legion( @@ -557,6 +562,11 @@ def install( legate_core_dir = os.path.dirname(os.path.realpath(__file__)) + # For the release, we will use a hardcoded commit unless user asks for + # a branch + # if legion_branch is None: + # legion_branch = find_default_legion_branch(legate_core_dir) + cmake_config = os.path.join(legate_core_dir, ".cmake.json") dump_json_config(cmake_config, cmake) diff --git a/setup.py b/setup.py index a19abc861b..82195265c7 100755 --- a/setup.py +++ b/setup.py @@ -69,9 +69,9 @@ def run(self): # Remove the recurse argument from the list sys.argv.remove("--recurse") setup( - name="legate.core", - version="22.01", - packages=["legate", "legate.core", "legate.timing"], + name="legate-core", + version="22.01.00", + packages=["legate", "legate-core", "legate-timing"], cmdclass={"build_py": my_build_py}, ) else: From e89bb08cc42e1d9bd6084b9eda869fd67edb77a3 Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Tue, 8 Feb 2022 11:07:54 -0800 Subject: [PATCH 0038/1425] Fix setup.py packages --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 82195265c7..10d98c0ea7 100755 --- a/setup.py +++ b/setup.py @@ -71,7 +71,7 @@ def run(self): setup( name="legate-core", version="22.01.00", - packages=["legate", "legate-core", "legate-timing"], + packages=["legate", "legate.core", "legate.timing"], cmdclass={"build_py": my_build_py}, ) else: From 34fd8ca3a53d0c3e3786fff9cad2e00af58fffcc Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Tue, 8 Feb 2022 11:14:45 -0800 Subject: [PATCH 0039/1425] Revert package name change. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 10d98c0ea7..b37dd6941b 100755 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ def run(self): # Remove the recurse argument from the list sys.argv.remove("--recurse") setup( - name="legate-core", + name="legate.core", version="22.01.00", packages=["legate", "legate.core", "legate.timing"], cmdclass={"build_py": my_build_py}, From 8b156959d735a40b20b3c98ff9e637e73101d5b5 Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Tue, 15 Feb 2022 10:44:38 -0800 Subject: [PATCH 0040/1425] Do not set a default branch for release. --- install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.py b/install.py index 955ab13b83..3eba4f1bb9 100755 --- a/install.py +++ b/install.py @@ -953,7 +953,7 @@ def driver(): "--legion-branch", dest="legion_branch", required=False, - default="control_replication", + default=None, help="Legion branch to build Legate with.", ) args, unknown = parser.parse_known_args() From f319ebab141d4a881b026b68be9aeb00854a96fd Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Tue, 15 Feb 2022 11:10:46 -0800 Subject: [PATCH 0041/1425] Set legion commit to the latest commit. --- install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install.py b/install.py index 3eba4f1bb9..5e80b04783 100755 --- a/install.py +++ b/install.py @@ -203,7 +203,7 @@ def install_gasnet(gasnet_dir, conduit, thread_count): shutil.rmtree(temp_dir) -def install_legion(legion_src_dir, branch, commit="c7ca2dbc"): +def install_legion(legion_src_dir, branch, commit="5b5f7b50"): print("Legate is installing Legion into a local directory...") # For now all we have to do is clone legion since we build it with Legate git_clone( @@ -223,7 +223,7 @@ def install_thrust(thrust_dir): ) -def update_legion(legion_src_dir, branch, commit="c7ca2dbc"): +def update_legion(legion_src_dir, branch, commit="5b5f7b50"): # Make sure we are on the right branch for single/multi-node git_update(legion_src_dir, branch=branch, commit=commit) From 6446612a3ca4431f225e40ea2c17a9799844c76b Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Tue, 15 Feb 2022 12:52:03 -0800 Subject: [PATCH 0042/1425] Set legion commit to the latest commit. --- install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install.py b/install.py index 5e80b04783..4e5b261abb 100755 --- a/install.py +++ b/install.py @@ -203,7 +203,7 @@ def install_gasnet(gasnet_dir, conduit, thread_count): shutil.rmtree(temp_dir) -def install_legion(legion_src_dir, branch, commit="5b5f7b50"): +def install_legion(legion_src_dir, branch, commit="ae594733"): print("Legate is installing Legion into a local directory...") # For now all we have to do is clone legion since we build it with Legate git_clone( @@ -223,7 +223,7 @@ def install_thrust(thrust_dir): ) -def update_legion(legion_src_dir, branch, commit="5b5f7b50"): +def update_legion(legion_src_dir, branch, commit="ae594733"): # Make sure we are on the right branch for single/multi-node git_update(legion_src_dir, branch=branch, commit=commit) From aee68c37138766fb9cd6a857774794e585601421 Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Fri, 25 Mar 2022 03:16:52 -0700 Subject: [PATCH 0043/1425] Add support for checking out a commit --- install.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/install.py b/install.py index 43322de1a8..45f0fd78a5 100755 --- a/install.py +++ b/install.py @@ -133,6 +133,7 @@ def git_clone(repo_dir, url, branch=None, tag=None, commit=None): verbose_check_call( ["git", "submodule", "update", "--init"], cwd=repo_dir ) + git_reset(repo_dir, commit) else: verbose_check_call( [ @@ -153,10 +154,14 @@ def git_reset(repo_dir, refspec): verbose_check_call(["git", "reset", "--hard", refspec], cwd=repo_dir) -def git_update(repo_dir, branch=None): +def git_update(repo_dir, branch=None, tag=None, commit=None): if branch is not None: + verbose_check_call(["git", "fetch"], cwd=repo_dir) verbose_check_call(["git", "checkout", branch], cwd=repo_dir) - verbose_check_call(["git", "pull", "--ff-only"], cwd=repo_dir) + verbose_check_call(["git", "pull", "--ff-only"], cwd=repo_dir) + else: + verbose_check_call(["git", "fetch"], cwd=repo_dir) + verbose_check_call(["git", "checkout", commit or tag], cwd=repo_dir) def load_json_config(filename): @@ -200,13 +205,14 @@ def install_gasnet(gasnet_dir, conduit, thread_count): shutil.rmtree(temp_dir) -def install_legion(legion_src_dir, branch): +def install_legion(legion_src_dir, branch, commit=None): print("Legate is installing Legion into a local directory...") # For now all we have to do is clone legion since we build it with Legate git_clone( legion_src_dir, url="https://gitlab.com/StanfordLegion/legion.git", branch=branch, + commit=commit, ) @@ -219,9 +225,9 @@ def install_thrust(thrust_dir): ) -def update_legion(legion_src_dir, branch): +def update_legion(legion_src_dir, branch, commit=None): # Make sure we are on the right branch for single/multi-node - git_update(legion_src_dir, branch=branch) + git_update(legion_src_dir, branch=branch, commit=commit) def build_legion( From 587c5dd62328be8cbe7b1dfe1dfeee9563823764 Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Mon, 28 Mar 2022 18:30:52 -0700 Subject: [PATCH 0044/1425] Fix the version of Legion to use in release --- install.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install.py b/install.py index 45f0fd78a5..f74c6ed84a 100755 --- a/install.py +++ b/install.py @@ -205,7 +205,7 @@ def install_gasnet(gasnet_dir, conduit, thread_count): shutil.rmtree(temp_dir) -def install_legion(legion_src_dir, branch, commit=None): +def install_legion(legion_src_dir, branch, commit="24437c29"): print("Legate is installing Legion into a local directory...") # For now all we have to do is clone legion since we build it with Legate git_clone( @@ -225,7 +225,7 @@ def install_thrust(thrust_dir): ) -def update_legion(legion_src_dir, branch, commit=None): +def update_legion(legion_src_dir, branch, commit="24437c29"): # Make sure we are on the right branch for single/multi-node git_update(legion_src_dir, branch=branch, commit=commit) @@ -976,7 +976,7 @@ def driver(): ) parser.add_argument( "--legion-branch", - dest="legion_branch", + dest=None, required=False, default="control_replication", help="Legion branch to build Legate with.", From 7a856146abc32d345bb0309f2a38b5018ff94c1b Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Thu, 31 Mar 2022 22:26:00 -0700 Subject: [PATCH 0045/1425] Fix merge errors. --- README.md | 3 --- legate/core/__init__.py | 4 ---- src/core/runtime/context.h | 5 ----- 3 files changed, 12 deletions(-) diff --git a/README.md b/README.md index 16912ac4ea..564210b93b 100644 --- a/README.md +++ b/README.md @@ -230,9 +230,6 @@ Windows. Legate Core requires the following: - Python >= 3.7 - - `pyarrow=5.0.0` - - `numpy` - - `cffi` - [CUDA](https://developer.nvidia.com/cuda-downloads) >= 8.0 - GNU Make - C++14 compatible compiler (g++, clang, or nvc++) diff --git a/legate/core/__init__.py b/legate/core/__init__.py index 65590b10e0..88511337e5 100644 --- a/legate/core/__init__.py +++ b/legate/core/__init__.py @@ -120,10 +120,6 @@ # are overriding that module's name. from legion_cffi import ffi, lib as legion -# NOTE: This needs to come after the imports from legate.core.legion, as we -# are overriding that module's name. -from legion_cffi import ffi, lib as legion - # Import the PyArrow type system from pyarrow import ( DataType, diff --git a/src/core/runtime/context.h b/src/core/runtime/context.h index 3bd246d8ee..e449ba86c2 100644 --- a/src/core/runtime/context.h +++ b/src/core/runtime/context.h @@ -125,11 +125,6 @@ class TaskContext { Legion::DomainPoint get_task_index() const; Legion::Domain get_launch_domain() const; - public: - bool is_single_task() const; - Legion::DomainPoint get_task_index() const; - Legion::Domain get_launch_domain() const; - public: ReturnValues pack_return_values() const; From 0d2bca783ac64812fa116846146a9d8914fea63a Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Wed, 25 May 2022 14:21:59 -0700 Subject: [PATCH 0046/1425] Fix version in setup.py. (#244) Co-authored-by: Marcin Zalewski --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5759076033..2f00ec303c 100755 --- a/setup.py +++ b/setup.py @@ -72,7 +72,7 @@ def run(self): sys.argv.remove("--recurse") setup( name="legate.core", - version="22.03", + version="22.05.00", packages=find_packages( where=".", include=["legate*"], From 060e85acff00ac86f014e42527cc203153d79d1f Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Thu, 26 May 2022 02:19:42 -0700 Subject: [PATCH 0047/1425] Use phase barriers to work around the CUDA driver bug (#246) (#247) * Utility methods to construct futures from buffers or cdata * Use NVML to check if the CUDA driver has a fix for GH #374 * Use barriers at the beginning of NCCL tasks to work around a driver bug * Install Legion C util as it's now used by the core * Rename the parameter to match the definition * Use a CUDA driver call for the version check * Add some note on what the barrier is intended to do Co-authored-by: Wonchan Lee --- install.py | 8 +++++++ legate/core/_legion/future.py | 38 +++++++++++++++++++++++++++++- legate/core/communicator.py | 11 ++++++++- legate/core/io.py | 15 ++++++++---- legate/core/launcher.py | 10 ++++++++ legate/core/operation.py | 2 ++ legate/core/runtime.py | 27 +++++++++++++++++++++ src/Makefile | 2 +- src/core/comm/comm_nccl.cu | 22 +++++++++++++++++ src/core/comm/comm_nccl.h | 2 ++ src/core/legate_c.h | 1 + src/core/mapping/core_mapper.cc | 11 +++++++++ src/core/runtime/context.cc | 22 ++++++++++++++++- src/core/utilities/deserializer.cc | 11 +++++++++ src/core/utilities/deserializer.h | 1 + typings/legion_cffi/lib.pyi | 5 ++++ 16 files changed, 179 insertions(+), 9 deletions(-) diff --git a/install.py b/install.py index f7066cee68..2856ff9afc 100755 --- a/install.py +++ b/install.py @@ -421,6 +421,14 @@ def build_legion( + setup_py_flags, cwd=legion_python_dir, ) + verbose_check_call( + [ + "cp", + "legion_c_util.h", + os.path.join(install_dir, "include", "legion", "legion_c_util.h"), + ], + cwd=os.path.join(legion_src_dir, "runtime", "legion"), + ) verbose_check_call( [ "cp", diff --git a/legate/core/_legion/future.py b/legate/core/_legion/future.py index caa6b21af3..f4d98c8822 100644 --- a/legate/core/_legion/future.py +++ b/legate/core/_legion/future.py @@ -80,7 +80,7 @@ def set_value( runtime: legion.legion_runtime_t, data: Any, size: int, - type: object = None, + type: Optional[Any] = None, ) -> None: """ Parameters @@ -101,6 +101,42 @@ def set_value( ) self._type = type + @classmethod + def from_buffer( + cls, + runtime: legion.legion_runtime_t, + buf: Any, + type: Optional[Any] = None, + ) -> Future: + """ + Construct a future from a buffer storing data + + Parameters + ---------- + buf : buffer + Buffer to create a future from + Returns + ------- + Future + """ + return cls( + legion.legion_future_from_untyped_pointer( + runtime, ffi.from_buffer(buf), len(buf) + ), + type=type, + ) + + @classmethod + def from_cdata( + cls, + runtime: legion.legion_runtime_t, + cdata: Any, + type: Optional[Any] = None, + ) -> Future: + return cls.from_buffer( + runtime, ffi.buffer(ffi.addressof(cdata)), type=type + ) + def get_buffer(self, size: Optional[int] = None) -> Any: """ Return a buffer storing the data for this Future. diff --git a/legate/core/communicator.py b/legate/core/communicator.py index b1f9f5bb75..2f550da13b 100644 --- a/legate/core/communicator.py +++ b/legate/core/communicator.py @@ -14,7 +14,7 @@ # from __future__ import annotations -from abc import ABC, abstractmethod +from abc import ABC, abstractmethod, abstractproperty from typing import TYPE_CHECKING from . import FutureMap, Rect @@ -59,6 +59,10 @@ def destroy(self) -> None: for volume, handle in self._handles.items(): self._finalize(volume, handle) + @abstractproperty + def needs_barrier(self) -> bool: + ... + @abstractmethod def _initialize(self, volume: int) -> FutureMap: ... @@ -77,6 +81,11 @@ def __init__(self, runtime: Runtime) -> None: self._init_nccl = library.LEGATE_CORE_INIT_NCCL_TASK_ID self._finalize_nccl = library.LEGATE_CORE_FINALIZE_NCCL_TASK_ID self._tag = library.LEGATE_GPU_VARIANT + self._needs_barrier = runtime.nccl_needs_barrier + + @property + def needs_barrier(self) -> bool: + return self._needs_barrier def _initialize(self, volume: int) -> FutureMap: # This doesn't need to run on a GPU, but will use it anyway diff --git a/legate/core/io.py b/legate/core/io.py index be3d3526d3..4f5a176552 100644 --- a/legate/core/io.py +++ b/legate/core/io.py @@ -17,7 +17,15 @@ from typing import TYPE_CHECKING, Callable, Iterable, Optional, Union from . import ffi # Make sure we only have one ffi instance -from . import FutureMap, IndexPartition, PartitionByDomain, Point, Rect, legion +from . import ( + Future, + FutureMap, + IndexPartition, + PartitionByDomain, + Point, + Rect, + legion, +) from .legate import Array, Table from .partition import Tiling from .runtime import _runtime @@ -69,13 +77,10 @@ def make_partition( colors: tuple[int, ...], local_colors: Iterable[Point], ) -> Partition: - fut_size = ffi.sizeof("legion_domain_t") futures = {} for c in local_colors: rect = self.get_subdomain(c) - futures[c] = _runtime.create_future( - ffi.buffer(ffi.addressof(rect.raw())), fut_size - ) + futures[c] = Future.from_cdata(_runtime.legion_runtime, rect.raw()) domains = FutureMap.from_dict( _runtime.legion_context, _runtime.legion_runtime, diff --git a/legate/core/launcher.py b/legate/core/launcher.py index 6745067d3b..a47607c303 100644 --- a/legate/core/launcher.py +++ b/legate/core/launcher.py @@ -722,6 +722,7 @@ def __init__( self._output_regions: list[OutputRegion] = [] self._error_on_interference = error_on_interference self._has_side_effect = side_effect + self._insert_barrier = False @property def library_task_id(self) -> int: @@ -858,6 +859,9 @@ def add_future_map(self, future_map: FutureMap) -> None: def add_communicator(self, handle: FutureMap) -> None: self._comms.append(handle) + def insert_barrier(self) -> None: + self._insert_barrier = True + def set_sharding_space(self, space: IndexSpace) -> None: self._sharding_space = space @@ -883,6 +887,7 @@ def build_task( self.pack_args(argbuf, self._outputs) self.pack_args(argbuf, self._reductions) self.pack_args(argbuf, self._scalars) + argbuf.pack_bool(self._insert_barrier) argbuf.pack_32bit_uint(len(self._comms)) task = IndexTask( @@ -901,6 +906,11 @@ def build_task( req.proj.add(task, req, fields, _index_task_calls) for future in self._future_args: task.add_future(future) + if self._insert_barrier: + volume = launch_domain.get_volume() + arrival, wait = self._runtime.get_barriers(volume) + task.add_future(arrival) + task.add_future(wait) for (out_req, fields) in self._out_analyzer.requirements: out_req.add(task, fields) for comm in self._comms: diff --git a/legate/core/operation.py b/legate/core/operation.py index 81f0963617..bf5882066c 100644 --- a/legate/core/operation.py +++ b/legate/core/operation.py @@ -342,6 +342,8 @@ def _add_communicators( for comm in self._comm_args: handle = comm.get_handle(launch_domain) launcher.add_communicator(handle) + if any(comm.needs_barrier for comm in self._comm_args): + launcher.insert_barrier() class AutoOperation(Operation): diff --git a/legate/core/runtime.py b/legate/core/runtime.py index d799a7528b..134ae89772 100644 --- a/legate/core/runtime.py +++ b/legate/core/runtime.py @@ -839,6 +839,14 @@ def __init__(self, core_library: CoreLib) -> None: ty.uint32, ) + self._barriers: List[legion.legion_phase_barrier_t] = [] + self.nccl_needs_barrier = bool( + self._core_context.get_tunable( + legion.LEGATE_CORE_TUNABLE_NCCL_NEEDS_BARRIER, + ty.bool_, + ) + ) + # Now we initialize managers self._attachment_manager = AttachmentManager(self) self._partition_manager = PartitionManager(self) @@ -943,6 +951,10 @@ def destroy(self) -> None: self.flush_scheduling_window() self._comm_manager.destroy() + for barrier in self._barriers: + legion.legion_phase_barrier_destroy( + self.legion_runtime, self.legion_context, barrier + ) # Destroy all libraries. Note that we should do this # from the lastly added one to the first one @@ -1307,6 +1319,21 @@ def delinearize_future_map( ) return FutureMap(handle) + def get_barriers(self, count: int) -> tuple[Future, Future]: + arrival_barrier = legion.legion_phase_barrier_create( + self.legion_runtime, self.legion_context, count + ) + wait_barrier = legion.legion_phase_barrier_advance( + self.legion_runtime, self.legion_context, arrival_barrier + ) + # TODO: For now we destroy these barriers during shutdown + self._barriers.append(arrival_barrier) + self._barriers.append(wait_barrier) + return ( + Future.from_cdata(self.legion_runtime, arrival_barrier), + Future.from_cdata(self.legion_runtime, wait_barrier), + ) + _runtime = Runtime(core_library) diff --git a/src/Makefile b/src/Makefile index ff050c0f4d..5eb69c4d0b 100644 --- a/src/Makefile +++ b/src/Makefile @@ -48,7 +48,7 @@ ifeq ($(strip $(USE_CUDA)),1) NVCC_FLAGS += -I$(NCCL_DIR)/include -LD_FLAGS += -L$(NCCL_DIR)/lib -lnccl +LD_FLAGS += -L$(NCCL_DIR)/lib -lnccl -lcuda endif diff --git a/src/core/comm/comm_nccl.cu b/src/core/comm/comm_nccl.cu index dbb3fc2e12..2e22b6e922 100644 --- a/src/core/comm/comm_nccl.cu +++ b/src/core/comm/comm_nccl.cu @@ -20,6 +20,7 @@ #include "core/utilities/nvtx_help.h" #include "legate.h" +#include #include using namespace Legion; @@ -170,6 +171,27 @@ void register_tasks(Legion::Machine machine, } } +bool needs_barrier() +{ + int32_t ver; + auto status = cuDriverGetVersion(&ver); + if (status != CUDA_SUCCESS) { + const char* error_string; + cuGetErrorString(status, &error_string); + fprintf(stderr, + "Internal CUDA failure with error %s in file %s at line %d\n", + error_string, + __FILE__, + __LINE__); + exit(status); + } + + int32_t major = ver / 1000; + int32_t minor = (ver - major * 1000) / 10; + + return major < 11 || major == 11 && minor < 8; +} + } // namespace nccl } // namespace comm } // namespace legate diff --git a/src/core/comm/comm_nccl.h b/src/core/comm/comm_nccl.h index 36e2d3c67e..82f4f5f7ce 100644 --- a/src/core/comm/comm_nccl.h +++ b/src/core/comm/comm_nccl.h @@ -27,6 +27,8 @@ void register_tasks(Legion::Machine machine, Legion::Runtime* runtime, const LibraryContext& context); +bool needs_barrier(); + } // namespace nccl } // namespace comm } // namespace legate diff --git a/src/core/legate_c.h b/src/core/legate_c.h index ac71c40eee..2852f3bf4e 100644 --- a/src/core/legate_c.h +++ b/src/core/legate_c.h @@ -48,6 +48,7 @@ typedef enum legate_core_tunable_t { LEGATE_CORE_TUNABLE_WINDOW_SIZE, LEGATE_CORE_TUNABLE_FIELD_REUSE_SIZE, LEGATE_CORE_TUNABLE_FIELD_REUSE_FREQUENCY, + LEGATE_CORE_TUNABLE_NCCL_NEEDS_BARRIER, } legate_core_tunable_t; typedef enum legate_core_variant_t { diff --git a/src/core/mapping/core_mapper.cc b/src/core/mapping/core_mapper.cc index 990810fe00..f359b50bfc 100644 --- a/src/core/mapping/core_mapper.cc +++ b/src/core/mapping/core_mapper.cc @@ -19,6 +19,9 @@ #include "legate.h" #include "core/mapping/core_mapper.h" +#ifdef LEGATE_USE_CUDA +#include "core/comm/comm_nccl.h" +#endif namespace legate { @@ -407,6 +410,14 @@ void CoreMapper::select_tunable_value(const MapperContext ctx, pack_tunable(field_reuse_freq, output); return; } + case LEGATE_CORE_TUNABLE_NCCL_NEEDS_BARRIER: { +#ifdef LEGATE_USE_CUDA + pack_tunable(local_gpus.empty() ? false : comm::nccl::needs_barrier(), output); +#else + pack_tunable(false, output); +#endif + return; + } } // Illegal tunable variable LEGATE_ABORT; diff --git a/src/core/runtime/context.cc b/src/core/runtime/context.cc index 46f4271b45..0a7f1dad0c 100644 --- a/src/core/runtime/context.cc +++ b/src/core/runtime/context.cc @@ -150,7 +150,17 @@ TaskContext::TaskContext(const Legion::Task* task, outputs_ = dez.unpack>(); reductions_ = dez.unpack>(); scalars_ = dez.unpack>(); - if (task->is_index_space) comms_ = dez.unpack>(); + + bool insert_barrier = false; + Legion::PhaseBarrier arrival, wait; + if (task->is_index_space) { + insert_barrier = dez.unpack(); + if (insert_barrier) { + arrival = dez.unpack(); + wait = dez.unpack(); + } + comms_ = dez.unpack>(); + } // For reduction tree cases, some input stores may be mapped to NO_REGION // when the number of subregions isn't a multiple of the chosen radix. // To simplify the programming mode, we filter out those "invalid" stores out. @@ -160,6 +170,16 @@ TaskContext::TaskContext(const Legion::Task* task, if (input.valid()) inputs.push_back(std::move(input)); inputs_.swap(inputs); } + + // CUDA drivers < 520 have a bug that causes deadlock under certain circumstances + // if the application has multiple threads that launch blocking kernels, such as + // NCCL all-reduce kernels. This barrier prevents such deadlock by making sure + // all CUDA driver calls from Realm are done before any of the GPU tasks starts + // making progress. + if (insert_barrier) { + arrival.arrive(); + wait.wait(); + } } bool TaskContext::is_single_task() const { return !task_->is_index_space; } diff --git a/src/core/utilities/deserializer.cc b/src/core/utilities/deserializer.cc index 0b3e373ff3..785d3272f4 100644 --- a/src/core/utilities/deserializer.cc +++ b/src/core/utilities/deserializer.cc @@ -20,6 +20,9 @@ #include "core/mapping/task.h" #include "core/utilities/machine.h" +#include "legion/legion_c.h" +#include "legion/legion_c_util.h" + using LegionTask = Legion::Task; using namespace Legion; @@ -115,6 +118,14 @@ void TaskDeserializer::_unpack(comm::Communicator& value) value = comm::Communicator(future); } +void TaskDeserializer::_unpack(Legion::PhaseBarrier& barrier) +{ + auto future = futures_[0]; + futures_ = futures_.subspan(1); + auto barrier_ = future.get_result(); + barrier = CObjectWrapper::unwrap(barrier_); +} + namespace mapping { MapperDeserializer::MapperDeserializer(const LegionTask* task, diff --git a/src/core/utilities/deserializer.h b/src/core/utilities/deserializer.h index 4df945e1b3..de3881267e 100644 --- a/src/core/utilities/deserializer.h +++ b/src/core/utilities/deserializer.h @@ -89,6 +89,7 @@ class TaskDeserializer : public BaseDeserializer { void _unpack(RegionField& value); void _unpack(OutputRegionField& value); void _unpack(comm::Communicator& value); + void _unpack(Legion::PhaseBarrier& barrier); private: Span futures_; diff --git a/typings/legion_cffi/lib.pyi b/typings/legion_cffi/lib.pyi index b5e4453cca..349296843e 100644 --- a/typings/legion_cffi/lib.pyi +++ b/typings/legion_cffi/lib.pyi @@ -17,6 +17,7 @@ from typing import Any class legion_runtime_t: ... class legion_context_t: ... +class legion_phase_barrier_t: ... LEGION_DISJOINT_COMPLETE_KIND: int LEGION_DISJOINT_INCOMPLETE_KIND: int @@ -65,6 +66,7 @@ LEGATE_CORE_TUNABLE_FIELD_REUSE_SIZE: int LEGATE_CORE_TUNABLE_FIELD_REUSE_FREQUENCY: int LEGATE_CORE_TUNABLE_NUM_PIECES: int LEGATE_CORE_TUNABLE_MIN_SHARD_VOLUME: int +LEGATE_CORE_TUNABLE_NCCL_NEEDS_BARRIER: int def legion_acquire_launcher_add_field(*args: Any) -> Any: ... def legion_acquire_launcher_create(*args: Any) -> Any: ... @@ -275,6 +277,9 @@ def legion_output_requirement_create_region_requirement(*args: Any) -> Any: ... def legion_output_requirement_destroy(*args: Any) -> Any: ... def legion_output_requirement_get_parent(*args: Any) -> Any: ... def legion_output_requirement_get_partition(*args: Any) -> Any: ... +def legion_phase_barrier_create(*args: Any) -> Any: ... +def legion_phase_barrier_advance(*args: Any) -> Any: ... +def legion_phase_barrier_destroy(*args: Any) -> Any: ... def legion_physical_region_destroy(*args: Any) -> Any: ... def legion_physical_region_is_mapped(*args: Any) -> Any: ... def legion_physical_region_wait_until_valid(*args: Any) -> Any: ... From b6406f69d1e7f4b1786952936c698bb82714aa52 Mon Sep 17 00:00:00 2001 From: Manolis Papadakis Date: Thu, 26 May 2022 13:47:21 -0700 Subject: [PATCH 0048/1425] Update for changes to Legion C API --- legate/core/_legion/partition.py | 1 - 1 file changed, 1 deletion(-) diff --git a/legate/core/_legion/partition.py b/legate/core/_legion/partition.py index 2619419253..c0d4c5c24d 100644 --- a/legate/core/_legion/partition.py +++ b/legate/core/_legion/partition.py @@ -63,7 +63,6 @@ def __init__( if handle is None: handle = legion.legion_logical_partition_create( runtime, - context, parent.handle, index_partition.handle, ) From 266a94965a5f9ec2bb76dcaf49cbde3fc4bee452 Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Fri, 27 May 2022 11:27:56 -0700 Subject: [PATCH 0049/1425] Fix Legion commit (#248) * Fix Legion commit * Update Legion version * Update Legion version Co-authored-by: Marcin Zalewski --- install.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install.py b/install.py index 2856ff9afc..0d30b4ace2 100755 --- a/install.py +++ b/install.py @@ -205,7 +205,7 @@ def install_gasnet(gasnet_dir, conduit, thread_count): shutil.rmtree(temp_dir) -def install_legion(legion_src_dir, branch, commit=None): +def install_legion(legion_src_dir, branch, commit="d517c5de"): print("Legate is installing Legion into a local directory...") # For now all we have to do is clone legion since we build it with Legate git_clone( @@ -225,7 +225,7 @@ def install_thrust(thrust_dir): ) -def update_legion(legion_src_dir, branch, commit=None): +def update_legion(legion_src_dir, branch, commit="d517c5de"): # Make sure we are on the right branch for single/multi-node git_update(legion_src_dir, branch=branch, commit=commit) @@ -1034,7 +1034,7 @@ def driver(): "--legion-branch", dest="legion_branch", required=False, - default="control_replication", + default=None, help="Legion branch to build Legate with.", ) args, unknown = parser.parse_known_args() From 8913e832013a509bf37d8a01bef8523b0473b5f9 Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Fri, 3 Jun 2022 00:27:43 -0700 Subject: [PATCH 0050/1425] Update Legion commit (#257) Co-authored-by: Marcin Zalewski --- install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install.py b/install.py index 0d30b4ace2..f83da223b6 100755 --- a/install.py +++ b/install.py @@ -205,7 +205,7 @@ def install_gasnet(gasnet_dir, conduit, thread_count): shutil.rmtree(temp_dir) -def install_legion(legion_src_dir, branch, commit="d517c5de"): +def install_legion(legion_src_dir, branch, commit="0e2e31bbd"): print("Legate is installing Legion into a local directory...") # For now all we have to do is clone legion since we build it with Legate git_clone( @@ -225,7 +225,7 @@ def install_thrust(thrust_dir): ) -def update_legion(legion_src_dir, branch, commit="d517c5de"): +def update_legion(legion_src_dir, branch, commit="0e2e31bbd"): # Make sure we are on the right branch for single/multi-node git_update(legion_src_dir, branch=branch, commit=commit) From 5c88b8578f29941ab4aaaf8646deda6aaf0dddee Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Wed, 15 Jun 2022 18:12:34 -0700 Subject: [PATCH 0051/1425] Freeze Conda Compiler Versions (#261) (#276) Co-authored-by: Mark Vaz --- conda/conda-build/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/conda-build/meta.yaml b/conda/conda-build/meta.yaml index 0e7c0073cf..906339c8e5 100644 --- a/conda/conda-build/meta.yaml +++ b/conda/conda-build/meta.yaml @@ -69,8 +69,8 @@ build: requirements: build: - - {{ compiler('c') }} - - {{ compiler('cxx') }} + - {{ compiler('c') }} =11.2 + - {{ compiler('cxx') }} =11.2 - make {% if gpu_enabled_bool %} - cuda-nvcc ={{ cuda_version }} From fa6975f21d9bbd70e383860cfb3eae2e3670489d Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Wed, 15 Jun 2022 18:12:58 -0700 Subject: [PATCH 0052/1425] Set cuda virtual package as hard run requirement for conda gpu package (#266) (#277) Co-authored-by: Mark Vaz --- conda/conda-build/meta.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/conda/conda-build/meta.yaml b/conda/conda-build/meta.yaml index 906339c8e5..0c4d04a1cb 100644 --- a/conda/conda-build/meta.yaml +++ b/conda/conda-build/meta.yaml @@ -90,12 +90,10 @@ requirements: {% if gpu_enabled_bool %} - cuda-cudart - nccl + - __cuda >= 11.4 {% endif %} run_constrained: -{% if gpu_enabled_bool %} - - __cuda >= 11.4 -{% endif %} - __glibc >=2.17 # [linux] test: From b61ecb33c7a2c04f9eab0b16ccf7515b737c6851 Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Thu, 16 Jun 2022 13:38:06 -0700 Subject: [PATCH 0053/1425] Release 22.05.01 * Freeze Conda Compiler Versions (#261) (#276) Co-authored-by: Mark Vaz * Set cuda virtual package as hard run requirement for conda gpu package (#266) (#277) Co-authored-by: Mark Vaz Co-authored-by: Mark Vaz --- conda/conda-build/meta.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/conda/conda-build/meta.yaml b/conda/conda-build/meta.yaml index 0e7c0073cf..0c4d04a1cb 100644 --- a/conda/conda-build/meta.yaml +++ b/conda/conda-build/meta.yaml @@ -69,8 +69,8 @@ build: requirements: build: - - {{ compiler('c') }} - - {{ compiler('cxx') }} + - {{ compiler('c') }} =11.2 + - {{ compiler('cxx') }} =11.2 - make {% if gpu_enabled_bool %} - cuda-nvcc ={{ cuda_version }} @@ -90,12 +90,10 @@ requirements: {% if gpu_enabled_bool %} - cuda-cudart - nccl + - __cuda >= 11.4 {% endif %} run_constrained: -{% if gpu_enabled_bool %} - - __cuda >= 11.4 -{% endif %} - __glibc >=2.17 # [linux] test: From 04c1869a59268ff9092f80ef290cbe89cf929b25 Mon Sep 17 00:00:00 2001 From: Mark Vaz Date: Tue, 21 Jun 2022 03:39:31 -0700 Subject: [PATCH 0054/1425] Fix typo in conda run requirements (#281) --- conda/conda-build/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/conda-build/meta.yaml b/conda/conda-build/meta.yaml index 0c4d04a1cb..cf6bfb52c0 100644 --- a/conda/conda-build/meta.yaml +++ b/conda/conda-build/meta.yaml @@ -90,7 +90,7 @@ requirements: {% if gpu_enabled_bool %} - cuda-cudart - nccl - - __cuda >= 11.4 + - __cuda >=11.4 {% endif %} run_constrained: From 2c3872d8f87550ecd1f8ee22a2762c556b3ba3af Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Tue, 21 Jun 2022 23:35:04 -0700 Subject: [PATCH 0055/1425] Catch up the ABI change --- src/core/data/logical_store.cc | 1 + src/core/runtime/launcher.cc | 32 +++++++++++++++++++++++--------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index cd3fd5aa46..277a5ff725 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -301,6 +301,7 @@ std::unique_ptr LogicalStore::find_or_create_key_partition() void LogicalStore::pack(BufferBuilder& buffer) const { buffer.pack(scalar_); + buffer.pack(false); buffer.pack(dim()); buffer.pack(code_); pack_transform(buffer); diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/launcher.cc index 51892621d1..84f1ff10c4 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -102,7 +102,7 @@ struct RegionFieldArg : public ArgWrapper { struct FutureStoreArg : public ArgWrapper { public: - FutureStoreArg(LogicalStore store, bool read_only); + FutureStoreArg(LogicalStore store, bool read_only, bool has_storage, Legion::ReductionOpID redop); public: virtual ~FutureStoreArg() {} @@ -113,6 +113,8 @@ struct FutureStoreArg : public ArgWrapper { private: LogicalStore store_; bool read_only_; + bool has_storage_; + Legion::ReductionOpID redop_; }; RequirementAnalyzer::~RequirementAnalyzer() @@ -184,8 +186,11 @@ void RegionFieldArg::pack(BufferBuilder& buffer) const buffer.pack(field_id_); } -FutureStoreArg::FutureStoreArg(LogicalStore store, bool read_only) - : store_(std::move(store)), read_only_(read_only) +FutureStoreArg::FutureStoreArg(LogicalStore store, + bool read_only, + bool has_storage, + Legion::ReductionOpID redop) + : store_(std::move(store)), read_only_(read_only), has_storage_(has_storage), redop_(redop) { } @@ -201,8 +206,9 @@ void FutureStoreArg::pack(BufferBuilder& buffer) const { store_.pack(buffer); + buffer.pack(redop_); buffer.pack(read_only_); - buffer.pack(true); + buffer.pack(has_storage_); buffer.pack(type_dispatch(store_.code(), datalen_fn{})); buffer.pack(store_.extents()); } @@ -372,17 +378,19 @@ void TaskLauncher::add_store(std::vector& args, Legion::PrivilegeMode privilege, uint64_t tag) { + auto redop = nullptr != proj->redop ? *proj->redop : -1; + if (store.scalar()) { - futures_.push_back(store.get_future()); - auto read_only = privilege == READ_ONLY; - args.push_back(new FutureStoreArg(std::move(store), read_only)); + auto has_storage = privilege != WRITE_ONLY; + auto read_only = privilege == READ_ONLY; + if (has_storage) futures_.push_back(store.get_future()); + args.push_back(new FutureStoreArg(std::move(store), read_only, has_storage, redop)); } else { auto storage = store.get_storage(); auto region = storage->region(); auto field_id = storage->field_id(); - auto redop = nullptr != proj->redop ? *proj->redop : -1; - auto req = new RegionReq(region, privilege, std::move(proj), tag); + auto req = new RegionReq(region, privilege, std::move(proj), tag); req_analyzer_->insert(req, field_id); args.push_back( @@ -402,6 +410,9 @@ Legion::TaskLauncher* TaskLauncher::build_single_task() pack_args(outputs_); pack_args(reductions_); pack_args(scalars_); + buffer_->pack(false); + buffer_->pack(false); + buffer_->pack(0); auto single_task = new Legion::TaskLauncher(legion_task_id(), buffer_->to_legion_buffer(), @@ -421,6 +432,9 @@ Legion::IndexTaskLauncher* TaskLauncher::build_index_task(const Legion::Domain& pack_args(outputs_); pack_args(reductions_); pack_args(scalars_); + buffer_->pack(false); + buffer_->pack(false); + buffer_->pack(0); auto index_task = new Legion::IndexTaskLauncher(legion_task_id(), launch_domain, From 52d3e68ec676e60ba6ef9eec2a6c70fdb4b5b6ea Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Thu, 23 Jun 2022 00:17:13 -0700 Subject: [PATCH 0056/1425] Port over symbolic points and use them to generate projection functors --- src/core.mk | 1 + src/core/data/logical_store.cc | 22 ++++++------ src/core/data/transform.cc | 37 ++++++++++++++++++--- src/core/data/transform.h | 15 +++++---- src/core/runtime/projection.cc | 61 ++++++++++++++++++++++++++++++++++ src/core/runtime/projection.h | 36 ++++++++++++++++++++ src/core/runtime/runtime.cc | 16 +++++++-- src/core/runtime/runtime.h | 4 +-- src/core/utilities/tuple.h | 10 ++++-- src/core/utilities/tuple.inl | 12 +++++++ 10 files changed, 183 insertions(+), 31 deletions(-) diff --git a/src/core.mk b/src/core.mk index 985a072167..ee6ec186fb 100644 --- a/src/core.mk +++ b/src/core.mk @@ -87,6 +87,7 @@ INSTALL_HEADERS = legate.h \ core/partitioning/constraint.h \ core/runtime/context.h \ core/runtime/operation.h \ + core/runtime/projection.h \ core/runtime/runtime.h \ core/runtime/runtime.inl \ core/task/exception.h \ diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index 277a5ff725..007695847b 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -100,7 +100,7 @@ class LogicalStore { private: std::unique_ptr invert_partition(const Partition* partition) const; - void invert_dimensions(tuple& dims) const; + proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const; Legion::ProjectionID compute_projection() const; public: @@ -246,27 +246,27 @@ std::unique_ptr LogicalStore::invert_partition(const Partition* parti return nullptr; } -void LogicalStore::invert_dimensions(tuple& dims) const +proj::SymbolicPoint LogicalStore::invert(const proj::SymbolicPoint& point) const { - if (nullptr == parent_) return; - transform_->invert_dimensions(dims); - parent_->invert_dimensions(dims); + if (nullptr == parent_) return point; + return parent_->invert(transform_->invert(point)); } Legion::ProjectionID LogicalStore::compute_projection() const { auto ndim = dim(); - auto dims = from_range(ndim); - invert_dimensions(dims); + std::vector exprs; + for (int32_t dim = 0; dim < ndim; ++dim) exprs.push_back(proj::SymbolicExpr(dim)); + auto point = invert(proj::SymbolicPoint(std::move(exprs))); - bool identity_mapping = dims.size() == ndim; - for (int32_t dim = 0; dim < ndim && identity_mapping; ++dim) - if (dims[dim] != dim) identity_mapping = false; + bool identity_mapping = true; + for (int32_t dim = 0; dim < ndim; ++dim) + identity_mapping = identity_mapping && point[dim].is_identity(dim); if (identity_mapping) return 0; else - return runtime_->get_projection(ndim, dims); + return runtime_->get_projection(ndim, point); } std::unique_ptr LogicalStore::find_or_create_partition(const Partition* partition) diff --git a/src/core/data/transform.cc b/src/core/data/transform.cc index 259669e361..e5907497f2 100644 --- a/src/core/data/transform.cc +++ b/src/core/data/transform.cc @@ -88,7 +88,8 @@ std::unique_ptr Shift::invert_partition(const Partition* partition) c return nullptr; } -void Shift::invert_dimensions(tuple& dims) const {} +// the shift transform makes no change on the store's dimensions +proj::SymbolicPoint Shift::invert(const proj::SymbolicPoint& point) const { return point; } void Shift::pack(BufferBuilder& buffer) const { @@ -184,7 +185,10 @@ std::unique_ptr Promote::invert_partition(const Partition* partition) return nullptr; } -void Promote::invert_dimensions(tuple& dims) const { dims.remove_inplace(extra_dim_); } +proj::SymbolicPoint Promote::invert(const proj::SymbolicPoint& point) const +{ + return point.remove(extra_dim_); +} void Promote::pack(BufferBuilder& buffer) const { @@ -266,7 +270,10 @@ std::unique_ptr Project::invert_partition(const Partition* partition) return nullptr; } -void Project::invert_dimensions(tuple& dims) const {} +proj::SymbolicPoint Project::invert(const proj::SymbolicPoint& point) const +{ + return point.insert(dim_, proj::SymbolicExpr()); +} void Project::pack(BufferBuilder& buffer) const { @@ -289,6 +296,11 @@ void Project::print(std::ostream& out) const Transpose::Transpose(std::vector&& axes, StoreTransformP parent) : StoreTransform(std::forward(parent)), axes_(std::move(axes)) { + inverse_.resize(axes_.size()); + std::iota(inverse_.begin(), inverse_.end(), 0); + std::sort(inverse_.begin(), inverse_.end(), [&](const int32_t& idx1, const int32_t& idx2) { + return axes_[idx1] < axes_[idx2]; + }); } Domain Transpose::transform(const Domain& input) const @@ -337,7 +349,16 @@ std::unique_ptr Transpose::invert_partition(const Partition* partitio return nullptr; } -void Transpose::invert_dimensions(tuple& dims) const {} +proj::SymbolicPoint Transpose::invert(const proj::SymbolicPoint& point) const +{ + std::vector exprs; + std::transform( + point.data().begin(), point.data().end(), exprs.end(), [&](const proj::SymbolicExpr& expr) { + auto dim = inverse_[expr.dim()]; + return proj::SymbolicExpr(dim, expr.weight(), expr.offset()); + }); + return proj::SymbolicPoint(std::move(exprs)); +} namespace { // anonymous template @@ -452,7 +473,13 @@ std::unique_ptr Delinearize::invert_partition(const Partition* partit return nullptr; } -void Delinearize::invert_dimensions(tuple& dims) const {} +proj::SymbolicPoint Delinearize::invert(const proj::SymbolicPoint& point) const +{ + std::vector exprs; + for (int32_t dim = 0; dim < dim_ + 1; ++dim) exprs.push_back(point[dim]); + for (int32_t dim = dim_ + sizes_.size(); dim < point.size(); ++dim) exprs.push_back(point[dim]); + return proj::SymbolicPoint(std::move(exprs)); +} void Delinearize::pack(BufferBuilder& buffer) const { diff --git a/src/core/data/transform.h b/src/core/data/transform.h index fe9237dea5..884668b1d9 100644 --- a/src/core/data/transform.h +++ b/src/core/data/transform.h @@ -18,7 +18,7 @@ #include -#include "core/utilities/tuple.h" +#include "core/runtime/projection.h" #include "legion.h" namespace legate { @@ -36,7 +36,7 @@ class StoreTransform { virtual Legion::Domain transform(const Legion::Domain& input) const = 0; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const = 0; virtual std::unique_ptr invert_partition(const Partition* partition) const = 0; - virtual void invert_dimensions(tuple& dims) const = 0; + virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const = 0; virtual void pack(BufferBuilder& buffer) const = 0; virtual void print(std::ostream& out) const = 0; @@ -55,7 +55,7 @@ class Shift : public StoreTransform { virtual Legion::Domain transform(const Legion::Domain& input) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; virtual std::unique_ptr invert_partition(const Partition* partition) const override; - virtual void invert_dimensions(tuple& dims) const override; + virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; @@ -73,7 +73,7 @@ class Promote : public StoreTransform { virtual Legion::Domain transform(const Legion::Domain& input) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; virtual std::unique_ptr invert_partition(const Partition* partition) const override; - virtual void invert_dimensions(tuple& dims) const override; + virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; @@ -91,7 +91,7 @@ class Project : public StoreTransform { virtual Legion::Domain transform(const Legion::Domain& domain) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; virtual std::unique_ptr invert_partition(const Partition* partition) const override; - virtual void invert_dimensions(tuple& dims) const override; + virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; @@ -109,12 +109,13 @@ class Transpose : public StoreTransform { virtual Legion::Domain transform(const Legion::Domain& domain) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; virtual std::unique_ptr invert_partition(const Partition* partition) const override; - virtual void invert_dimensions(tuple& dims) const override; + virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; private: std::vector axes_; + std::vector inverse_; }; class Delinearize : public StoreTransform { @@ -128,7 +129,7 @@ class Delinearize : public StoreTransform { virtual Legion::Domain transform(const Legion::Domain& domain) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; virtual std::unique_ptr invert_partition(const Partition* partition) const override; - virtual void invert_dimensions(tuple& dims) const override; + virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; diff --git a/src/core/runtime/projection.cc b/src/core/runtime/projection.cc index 1fde46ea05..71d3c011ff 100644 --- a/src/core/runtime/projection.cc +++ b/src/core/runtime/projection.cc @@ -27,6 +27,67 @@ using namespace Legion; namespace legate { +namespace proj { + +SymbolicExpr::SymbolicExpr(int32_t dim, int32_t weight, int32_t offset) + : dim_(dim), weight_(weight), offset_(offset) +{ +} + +bool SymbolicExpr::is_identity(int32_t dim) const +{ + return dim_ == dim && weight_ == 1 && offset_ == 0; +} + +bool SymbolicExpr::operator==(const SymbolicExpr& other) const +{ + return dim_ == other.dim_ && weight_ == other.weight_ && offset_ == other.offset_; +} + +bool SymbolicExpr::operator<(const SymbolicExpr& other) const +{ + if (dim_ < other.dim_) + return true; + else if (dim_ > other.dim_) + return false; + if (weight_ < other.weight_) + return true; + else if (weight_ > other.weight_) + return false; + if (offset_ < other.offset_) return true; + return false; +} + +SymbolicExpr SymbolicExpr::operator*(int32_t other) const +{ + return SymbolicExpr(dim_, weight_ * other, offset_ * other); +} + +SymbolicExpr SymbolicExpr::operator+(int32_t other) const +{ + return SymbolicExpr(dim_, weight_, offset_ + other); +} + +std::ostream& operator<<(std::ostream& out, const SymbolicExpr& expr) +{ + auto weight = expr.weight(); + auto offset = expr.offset(); + + if (weight != 0) { + if (weight != 1) out << weight << "*"; + out << "COORD" << expr.dim(); + } + if (offset != 0) { + if (offset > 0) + out << "+" << offset; + else + out << "-" << -offset; + } + return out; +} + +} // namespace proj + class DelinearizationFunctor : public ProjectionFunctor { public: DelinearizationFunctor(Runtime* runtime); diff --git a/src/core/runtime/projection.h b/src/core/runtime/projection.h index 740c4bef5a..59d03ad66d 100644 --- a/src/core/runtime/projection.h +++ b/src/core/runtime/projection.h @@ -19,9 +19,45 @@ #include "legion.h" #include "core/runtime/context.h" +#include "core/utilities/tuple.h" namespace legate { +namespace proj { + +class SymbolicExpr { + public: + SymbolicExpr(int32_t dim = -1, int32_t weight = 1, int32_t offset = 0); + + public: + int32_t dim() const { return dim_; } + int32_t weight() const { return weight_; } + int32_t offset() const { return offset_; } + + public: + bool is_identity(int32_t dim) const; + + public: + bool operator==(const SymbolicExpr& other) const; + bool operator<(const SymbolicExpr& other) const; + + public: + SymbolicExpr operator*(int32_t other) const; + SymbolicExpr operator+(int32_t other) const; + + private: + int32_t dim_{-1}; + int32_t weight_{1}; + int32_t offset_{0}; +}; + +std::ostream& operator<<(std::ostream& out, const SymbolicExpr& expr); + +using SymbolicPoint = tuple; +using SymbolicFunctor = SymbolicPoint (*)(const SymbolicPoint&); + +} // namespace proj + // Interface for Legate projection functors class LegateProjectionFunctor : public Legion::ProjectionFunctor { public: diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index ee57627cef..7102db2e95 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -738,15 +738,25 @@ std::shared_ptr Runtime::dispatch(IndexTaskLauncher* launcher) return nullptr; } -Legion::ProjectionID Runtime::get_projection(int32_t src_ndim, const tuple& dims) +Legion::ProjectionID Runtime::get_projection(int32_t src_ndim, const proj::SymbolicPoint& point) { - ProjectionDesc key(src_ndim, dims.data()); + ProjectionDesc key(src_ndim, point); auto finder = registered_projections_.find(key); if (registered_projections_.end() != finder) return finder->second; auto proj_id = core_context_->get_projection_id(next_projection_id_++); + + auto ndim = point.size(); + std::vector dims; + std::vector weights; + std::vector offsets; + for (auto& expr : point.data()) { + dims.push_back(expr.dim()); + weights.push_back(expr.weight()); + offsets.push_back(expr.offset()); + } legate_register_affine_projection_functor( - src_ndim, static_cast(dims.size()), key.second.data(), nullptr, nullptr, proj_id); + src_ndim, ndim, dims.data(), weights.data(), offsets.data(), proj_id); registered_projections_[key] = proj_id; return proj_id; diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 2ed6c39ff4..fdcfcba158 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -135,7 +135,7 @@ class Runtime { std::shared_ptr dispatch(Legion::IndexTaskLauncher* launcher); public: - Legion::ProjectionID get_projection(int32_t src_ndim, const tuple& dims); + Legion::ProjectionID get_projection(int32_t src_ndim, const proj::SymbolicPoint& point); private: void schedule(std::vector> operations); @@ -163,7 +163,7 @@ class Runtime { std::map index_spaces_; private: - using ProjectionDesc = std::pair>; + using ProjectionDesc = std::pair; int64_t next_projection_id_{LEGATE_CORE_FIRST_DYNAMIC_FUNCTOR_ID}; std::map registered_projections_; diff --git a/src/core/utilities/tuple.h b/src/core/utilities/tuple.h index 2e39bd14b1..c54dc39aee 100644 --- a/src/core/utilities/tuple.h +++ b/src/core/utilities/tuple.h @@ -37,15 +37,19 @@ class tuple { tuple(size_t size, T init); public: - tuple(const tuple&) = default; - tuple(tuple&&) = default; + tuple(const tuple&) = default; + tuple(tuple&&) = default; tuple& operator=(const tuple&) = default; - tuple& operator=(tuple&&) = default; + tuple& operator=(tuple&&) = default; public: const T& operator[](uint32_t idx) const; T& operator[](uint32_t idx); + public: + bool operator==(const tuple& other) const; + bool operator<(const tuple& other) const; + public: bool empty() const; size_t size() const; diff --git a/src/core/utilities/tuple.inl b/src/core/utilities/tuple.inl index 7b0ac98052..059c817961 100644 --- a/src/core/utilities/tuple.inl +++ b/src/core/utilities/tuple.inl @@ -57,6 +57,18 @@ T& tuple::operator[](uint32_t idx) return data_[idx]; } +template +bool tuple::operator==(const tuple& other) const +{ + return data_ == other.data_; +} + +template +bool tuple::operator<(const tuple& other) const +{ + return data_ < other.data_; +} + template bool tuple::empty() const { From 1df274e6ea57942c92eea53fe505ca9bcff453de Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Thu, 23 Jun 2022 23:26:30 -0700 Subject: [PATCH 0057/1425] Use integers for reduction op ids --- src/core/runtime/launcher.cc | 31 +++++++++++++++++++------------ src/core/runtime/launcher.h | 2 +- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/launcher.cc index 84f1ff10c4..787f97d468 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -213,14 +213,9 @@ void FutureStoreArg::pack(BufferBuilder& buffer) const buffer.pack(store_.extents()); } -Projection::Projection(Legion::ReductionOpID r) : redop(std::make_unique(r)) -{ -} +Projection::Projection(Legion::ReductionOpID r) : redop(r) {} -void Projection::set_reduction_op(Legion::ReductionOpID r) -{ - redop = std::make_unique(r); -} +void Projection::set_reduction_op(Legion::ReductionOpID r) { redop = r; } Replicate::Replicate() : Projection() {} @@ -231,7 +226,10 @@ void Replicate::populate_launcher(Legion::TaskLauncher* task, const std::vector& fields) const { if (req.priv == REDUCE) { - Legion::RegionRequirement legion_req(req.region, *redop, EXCLUSIVE, req.region, req.tag); +#ifdef DEBUG_LEGATE + assert(redop != -1); +#endif + Legion::RegionRequirement legion_req(req.region, redop, EXCLUSIVE, req.region, req.tag); legion_req.add_fields(fields); task->add_region_requirement(legion_req); } else { @@ -246,7 +244,10 @@ void Replicate::populate_launcher(Legion::IndexTaskLauncher* task, const std::vector& fields) const { if (req.priv == REDUCE) { - Legion::RegionRequirement legion_req(req.region, *redop, EXCLUSIVE, req.region, req.tag); +#ifdef DEBUG_LEGATE + assert(redop != -1); +#endif + Legion::RegionRequirement legion_req(req.region, redop, EXCLUSIVE, req.region, req.tag); legion_req.add_fields(fields); task->add_region_requirement(legion_req); } else { @@ -273,7 +274,10 @@ void MapPartition::populate_launcher(Legion::TaskLauncher* task, const std::vector& fields) const { if (req.priv == REDUCE) { - Legion::RegionRequirement legion_req(req.region, *redop, EXCLUSIVE, req.region, req.tag); +#ifdef DEBUG_LEGATE + assert(redop != -1); +#endif + Legion::RegionRequirement legion_req(req.region, redop, EXCLUSIVE, req.region, req.tag); legion_req.add_fields(fields); task->add_region_requirement(legion_req); } else { @@ -288,8 +292,11 @@ void MapPartition::populate_launcher(Legion::IndexTaskLauncher* task, const std::vector& fields) const { if (req.priv == REDUCE) { +#ifdef DEBUG_LEGATE + assert(redop != -1); +#endif Legion::RegionRequirement legion_req( - partition_, proj_id_, *redop, EXCLUSIVE, req.region, req.tag); + partition_, proj_id_, redop, EXCLUSIVE, req.region, req.tag); legion_req.add_fields(fields); task->add_region_requirement(legion_req); } else { @@ -378,7 +385,7 @@ void TaskLauncher::add_store(std::vector& args, Legion::PrivilegeMode privilege, uint64_t tag) { - auto redop = nullptr != proj->redop ? *proj->redop : -1; + auto redop = proj->redop; if (store.scalar()) { auto has_storage = privilege != WRITE_ONLY; diff --git a/src/core/runtime/launcher.h b/src/core/runtime/launcher.h index 75287f4f9a..9c119a3284 100644 --- a/src/core/runtime/launcher.h +++ b/src/core/runtime/launcher.h @@ -51,7 +51,7 @@ class Projection { void set_reduction_op(Legion::ReductionOpID redop); public: - std::unique_ptr redop{nullptr}; + Legion::ReductionOpID redop{-1}; }; class Replicate : public Projection { From a281039dc1ce3d044141953bbb0998253aadce02 Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Tue, 12 Jul 2022 19:49:46 -0700 Subject: [PATCH 0058/1425] Update legion commit to the latest (#297) Co-authored-by: Marcin Zalewski --- install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install.py b/install.py index 10ca1d0dd5..88e8595588 100755 --- a/install.py +++ b/install.py @@ -205,7 +205,7 @@ def install_gasnet(gasnet_dir, conduit, thread_count): shutil.rmtree(temp_dir) -def install_legion(legion_src_dir, branch, commit="0e2e31bbd"): +def install_legion(legion_src_dir, branch, commit="36851ec7"): print("Legate is installing Legion into a local directory...") # For now all we have to do is clone legion since we build it with Legate git_clone( @@ -225,7 +225,7 @@ def install_thrust(thrust_dir): ) -def update_legion(legion_src_dir, branch, commit="0e2e31bbd"): +def update_legion(legion_src_dir, branch, commit="36851ec7"): # Make sure we are on the right branch for single/multi-node git_update(legion_src_dir, branch=branch, commit=commit) From 69fcb0d670dbe6bebfee746f0459ee02aceca846 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 27 Jul 2022 22:30:25 -0700 Subject: [PATCH 0059/1425] Post merge cleanup --- src/core/data/logical_store.cc | 20 ++++++++++++++------ src/core/data/logical_store.h | 8 ++++---- src/core/data/store.cc | 18 ++++++++++++++++++ src/core/data/store.h | 5 +++++ src/core/data/transform.cc | 24 ++++++++++++++++++++++++ src/core/data/transform.h | 3 +++ 6 files changed, 68 insertions(+), 10 deletions(-) diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index 007695847b..9f8ca25869 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -53,7 +53,7 @@ class LogicalStore { LegateTypeCode code, tuple extents, std::shared_ptr parent, - std::shared_ptr transform); + std::shared_ptr transform); LogicalStore(Runtime* runtime, LegateTypeCode code, const void* data); public: @@ -117,7 +117,7 @@ class LogicalStore { std::shared_ptr region_field_{nullptr}; Legion::Future future_{}; std::shared_ptr parent_{nullptr}; - std::shared_ptr transform_{nullptr}; + std::shared_ptr transform_{nullptr}; std::shared_ptr mapped_{nullptr}; }; @@ -125,7 +125,7 @@ LogicalStore::LogicalStore(Runtime* runtime, LegateTypeCode code, tuple extents, std::shared_ptr parent, - std::shared_ptr transform) + std::shared_ptr transform) : runtime_(runtime), code_(code), extents_(std::move(extents)), @@ -207,7 +207,14 @@ std::shared_ptr LogicalStore::promote(int32_t extra_dim, } auto new_extents = extents_.insert(extra_dim, dim_size); - auto transform = std::make_shared(extra_dim, dim_size); + // TODO: Move this push operation to TransformStack. + // Two prerequisites: + // 1) make members of TransformStack read only (i.e., by adding const) + // 2) make TransformStack inherit std::enable_shared_from_this. + // Then we can add a const push method to TransformStack that returns + // a fresh shared_ptr of TransformStack with a transform put on top. + auto transform = + std::make_shared(std::make_unique(extra_dim, dim_size), transform_); return std::make_shared( runtime_, code_, std::move(new_extents), std::move(parent), std::move(transform)); } @@ -304,7 +311,8 @@ void LogicalStore::pack(BufferBuilder& buffer) const buffer.pack(false); buffer.pack(dim()); buffer.pack(code_); - pack_transform(buffer); + if (transform_ != nullptr) transform_->pack(buffer); + buffer.pack(-1); } void LogicalStore::pack_transform(BufferBuilder& buffer) const @@ -325,7 +333,7 @@ LogicalStore::LogicalStore(Runtime* runtime, LegateTypeCode code, tuple extents, LogicalStore parent, /* = LogicalStore() */ - std::shared_ptr transform /*= nullptr*/) + std::shared_ptr transform /*= nullptr*/) : impl_(std::make_shared( runtime, code, std::move(extents), parent.impl_, std::move(transform))) { diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index 04cc66ffd0..0644bf1f8c 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -44,7 +44,7 @@ class LogicalRegionField { LogicalRegionField(Runtime* runtime, const Legion::LogicalRegion& lr, Legion::FieldID fid); public: - LogicalRegionField(const LogicalRegionField& other) = default; + LogicalRegionField(const LogicalRegionField& other) = default; LogicalRegionField& operator=(const LogicalRegionField& other) = default; public: @@ -68,7 +68,7 @@ class LogicalStore { LegateTypeCode code, tuple extents, LogicalStore parent = LogicalStore(), - std::shared_ptr transform = nullptr); + std::shared_ptr transform = nullptr); // Creates a read-only store from a scalar LogicalStore(Runtime* runtime, LegateTypeCode code, const void* data); @@ -76,11 +76,11 @@ class LogicalStore { LogicalStore(std::shared_ptr impl); public: - LogicalStore(const LogicalStore& other) = default; + LogicalStore(const LogicalStore& other) = default; LogicalStore& operator=(const LogicalStore& other) = default; public: - LogicalStore(LogicalStore&& other) = default; + LogicalStore(LogicalStore&& other) = default; LogicalStore& operator=(LogicalStore&& other) = default; public: diff --git a/src/core/data/store.cc b/src/core/data/store.cc index 0eaf08eb16..baa65ecdd3 100644 --- a/src/core/data/store.cc +++ b/src/core/data/store.cc @@ -205,6 +205,24 @@ Store::Store(int32_t dim, { } +Store::Store(int32_t dim, + int32_t code, + int32_t redop_id, + RegionField&& region_field, + const std::shared_ptr& transform) + : is_future_(false), + is_output_store_(false), + dim_(dim), + code_(code), + redop_id_(redop_id), + region_field_(std::forward(region_field)), + transform_(transform) +{ + readable_ = region_field_.is_readable(); + writable_ = region_field_.is_writable(); + reducible_ = region_field_.is_reducible(); +} + Store::Store(Store&& other) noexcept : is_future_(other.is_future_), is_output_store_(other.is_output_store_), diff --git a/src/core/data/store.h b/src/core/data/store.h index 83ac5fd880..837e932077 100644 --- a/src/core/data/store.h +++ b/src/core/data/store.h @@ -271,6 +271,11 @@ class Store { int32_t code, OutputRegionField&& output, std::shared_ptr&& transform = nullptr); + Store(int32_t dim, + int32_t code, + int32_t redop_id, + RegionField&& region_field, + const std::shared_ptr& transform); public: Store(Store&& other) noexcept; diff --git a/src/core/data/transform.cc b/src/core/data/transform.cc index ed69ae810e..8b490f1f93 100644 --- a/src/core/data/transform.cc +++ b/src/core/data/transform.cc @@ -33,6 +33,12 @@ DomainAffineTransform combine(const DomainAffineTransform& lhs, const DomainAffi return result; } +TransformStack::TransformStack(std::unique_ptr&& transform, + const std::shared_ptr& parent) + : transform_(std::forward(transform)), parent_(parent) +{ +} + TransformStack::TransformStack(std::unique_ptr&& transform, std::shared_ptr&& parent) : transform_(std::forward(transform)), @@ -40,6 +46,24 @@ TransformStack::TransformStack(std::unique_ptr&& transform, { } +std::unique_ptr TransformStack::invert_partition(const Partition* partition) const +{ + auto result = transform_->invert_partition(partition); + return parent_ != nullptr ? parent_->invert_partition(result.get()) : std::move(result); +} + +proj::SymbolicPoint TransformStack::invert(const proj::SymbolicPoint& point) const +{ + auto result = transform_->invert(point); + return parent_ != nullptr ? parent_->invert(result) : result; +} + +void TransformStack::pack(BufferBuilder& buffer) const +{ + transform_->pack(buffer); + if (parent_ != nullptr) parent_->pack(buffer); +} + Legion::Domain TransformStack::transform(const Legion::Domain& input) const { #ifdef DEBUG_LEGATE diff --git a/src/core/data/transform.h b/src/core/data/transform.h index e0b2ddd0a4..8468023c19 100644 --- a/src/core/data/transform.h +++ b/src/core/data/transform.h @@ -42,6 +42,9 @@ struct StoreTransform : public Transform { struct TransformStack : public Transform { public: + // TODO: this constructor will be gone once we move the push method to this class + TransformStack(std::unique_ptr&& transform, + const std::shared_ptr& parent); TransformStack(std::unique_ptr&& transform, std::shared_ptr&& parent); From 583e09b13f2237e0b9bc4e5649a64f68afbb165d Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 27 Jul 2022 23:09:11 -0700 Subject: [PATCH 0060/1425] Minor refactoring before filling in RequirementAnalyzer --- src/core.mk | 2 + src/core/runtime/launcher.cc | 191 +------------------------------ src/core/runtime/launcher_arg.cc | 82 +++++++++++++ src/core/runtime/launcher_arg.h | 103 +++++++++++++++++ src/core/runtime/req_analyzer.cc | 61 ++++++++++ src/core/runtime/req_analyzer.h | 43 +++++++ 6 files changed, 293 insertions(+), 189 deletions(-) create mode 100644 src/core/runtime/launcher_arg.cc create mode 100644 src/core/runtime/launcher_arg.h create mode 100644 src/core/runtime/req_analyzer.cc create mode 100644 src/core/runtime/req_analyzer.h diff --git a/src/core.mk b/src/core.mk index ee6ec186fb..0e22e228b3 100644 --- a/src/core.mk +++ b/src/core.mk @@ -35,8 +35,10 @@ GEN_CPU_SRC = core/legate_c.cc \ core/partitioning/partitioner.cc \ core/runtime/context.cc \ core/runtime/launcher.cc \ + core/runtime/launcher_arg.cc \ core/runtime/operation.cc \ core/runtime/projection.cc \ + core/runtime/req_analyzer.cc \ core/runtime/runtime.cc \ core/runtime/shard.cc \ core/task/return.cc \ diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/launcher.cc index 787f97d468..7f5dc6207f 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -18,201 +18,14 @@ #include "core/data/logical_store.h" #include "core/data/scalar.h" #include "core/runtime/context.h" +#include "core/runtime/launcher_arg.h" +#include "core/runtime/req_analyzer.h" #include "core/runtime/runtime.h" #include "core/utilities/buffer_builder.h" #include "core/utilities/dispatch.h" namespace legate { -class RequirementAnalyzer { - public: - ~RequirementAnalyzer(); - - public: - void insert(RegionReq* req, Legion::FieldID field_id); - uint32_t get_requirement_index(RegionReq* req, Legion::FieldID field_id) const; - - public: - void analyze_requirements(); - template - void populate_launcher(Task* task) const; - - private: - std::map req_indices_; - std::vector>> requirements_; -}; - -struct ArgWrapper { - virtual ~ArgWrapper() {} - virtual void pack(BufferBuilder& buffer) const = 0; -}; - -template -struct ScalarArg : public ArgWrapper { - public: - ScalarArg(const T& value) : value_(value) {} - - public: - virtual ~ScalarArg() {} - - public: - virtual void pack(BufferBuilder& buffer) const override; - - private: - T value_; -}; - -struct UntypedScalarArg : public ArgWrapper { - public: - UntypedScalarArg(const Scalar& scalar) : scalar_(scalar) {} - - public: - virtual ~UntypedScalarArg() {} - - public: - virtual void pack(BufferBuilder& buffer) const override; - - private: - Scalar scalar_; -}; - -struct RegionFieldArg : public ArgWrapper { - public: - RegionFieldArg(RequirementAnalyzer* analyzer, - LogicalStore store, - int32_t dim, - RegionReq* req, - Legion::FieldID field_id, - Legion::ReductionOpID redop); - - public: - virtual void pack(BufferBuilder& buffer) const override; - - public: - virtual ~RegionFieldArg() {} - - private: - RequirementAnalyzer* analyzer_; - LogicalStore store_; - int32_t dim_; - RegionReq* req_; - Legion::FieldID field_id_; - Legion::ReductionOpID redop_; -}; - -struct FutureStoreArg : public ArgWrapper { - public: - FutureStoreArg(LogicalStore store, bool read_only, bool has_storage, Legion::ReductionOpID redop); - - public: - virtual ~FutureStoreArg() {} - - public: - virtual void pack(BufferBuilder& buffer) const override; - - private: - LogicalStore store_; - bool read_only_; - bool has_storage_; - Legion::ReductionOpID redop_; -}; - -RequirementAnalyzer::~RequirementAnalyzer() -{ - for (auto& pair : requirements_) delete pair.first; -} - -void RequirementAnalyzer::insert(RegionReq* req, Legion::FieldID field_id) -{ - uint32_t req_idx = static_cast(requirements_.size()); - requirements_.push_back(std::make_pair(req, std::vector({field_id}))); - req_indices_[req] = req_idx; -} - -uint32_t RequirementAnalyzer::get_requirement_index(RegionReq* req, Legion::FieldID field_id) const -{ - auto finder = req_indices_.find(req); - assert(finder != req_indices_.end()); - return finder->second; -} - -void RequirementAnalyzer::analyze_requirements() {} - -template -void RequirementAnalyzer::populate_launcher(Task* task) const -{ - for (auto& pair : requirements_) { - auto& req = pair.first; - auto& fields = pair.second; - req->proj->populate_launcher(task, *req, fields); - } -} - -template -void ScalarArg::pack(BufferBuilder& buffer) const -{ - buffer.pack(value_); -} - -void UntypedScalarArg::pack(BufferBuilder& buffer) const -{ - buffer.pack(scalar_.is_tuple()); - buffer.pack(scalar_.code()); - buffer.pack_buffer(scalar_.ptr(), scalar_.size()); -} - -RegionFieldArg::RegionFieldArg(RequirementAnalyzer* analyzer, - LogicalStore store, - int32_t dim, - RegionReq* req, - Legion::FieldID field_id, - Legion::ReductionOpID redop) - : analyzer_(analyzer), - store_(std::move(store)), - dim_(dim), - req_(req), - field_id_(field_id), - redop_(redop) -{ -} - -void RegionFieldArg::pack(BufferBuilder& buffer) const -{ - store_.pack(buffer); - - buffer.pack(redop_); - buffer.pack(dim_); - buffer.pack(analyzer_->get_requirement_index(req_, field_id_)); - buffer.pack(field_id_); -} - -FutureStoreArg::FutureStoreArg(LogicalStore store, - bool read_only, - bool has_storage, - Legion::ReductionOpID redop) - : store_(std::move(store)), read_only_(read_only), has_storage_(has_storage), redop_(redop) -{ -} - -struct datalen_fn { - template - size_t operator()() - { - return sizeof(legate_type_of); - } -}; - -void FutureStoreArg::pack(BufferBuilder& buffer) const -{ - store_.pack(buffer); - - buffer.pack(redop_); - buffer.pack(read_only_); - buffer.pack(has_storage_); - buffer.pack(type_dispatch(store_.code(), datalen_fn{})); - buffer.pack(store_.extents()); -} - Projection::Projection(Legion::ReductionOpID r) : redop(r) {} void Projection::set_reduction_op(Legion::ReductionOpID r) { redop = r; } diff --git a/src/core/runtime/launcher_arg.cc b/src/core/runtime/launcher_arg.cc new file mode 100644 index 0000000000..85d943642e --- /dev/null +++ b/src/core/runtime/launcher_arg.cc @@ -0,0 +1,82 @@ +/* Copyright 2022 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/runtime/launcher_arg.h" +#include "core/runtime/launcher.h" +#include "core/runtime/req_analyzer.h" + +namespace legate { + +void UntypedScalarArg::pack(BufferBuilder& buffer) const +{ + buffer.pack(scalar_.is_tuple()); + buffer.pack(scalar_.code()); + buffer.pack_buffer(scalar_.ptr(), scalar_.size()); +} + +RegionFieldArg::RegionFieldArg(RequirementAnalyzer* analyzer, + LogicalStore store, + int32_t dim, + RegionReq* req, + Legion::FieldID field_id, + Legion::ReductionOpID redop) + : analyzer_(analyzer), + store_(std::move(store)), + dim_(dim), + req_(req), + field_id_(field_id), + redop_(redop) +{ +} + +void RegionFieldArg::pack(BufferBuilder& buffer) const +{ + store_.pack(buffer); + + buffer.pack(redop_); + buffer.pack(dim_); + buffer.pack(analyzer_->get_requirement_index(req_, field_id_)); + buffer.pack(field_id_); +} + +FutureStoreArg::FutureStoreArg(LogicalStore store, + bool read_only, + bool has_storage, + Legion::ReductionOpID redop) + : store_(std::move(store)), read_only_(read_only), has_storage_(has_storage), redop_(redop) +{ +} + +struct datalen_fn { + template + size_t operator()() + { + return sizeof(legate_type_of); + } +}; + +void FutureStoreArg::pack(BufferBuilder& buffer) const +{ + store_.pack(buffer); + + buffer.pack(redop_); + buffer.pack(read_only_); + buffer.pack(has_storage_); + buffer.pack(type_dispatch(store_.code(), datalen_fn{})); + buffer.pack(store_.extents()); +} + +} // namespace legate diff --git a/src/core/runtime/launcher_arg.h b/src/core/runtime/launcher_arg.h new file mode 100644 index 0000000000..a018f69686 --- /dev/null +++ b/src/core/runtime/launcher_arg.h @@ -0,0 +1,103 @@ +/* Copyright 2022 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include "core/data/logical_store.h" +#include "core/data/scalar.h" +#include "core/utilities/buffer_builder.h" + +namespace legate { + +class RegionReq; +class RequirementAnalyzer; + +struct ArgWrapper { + virtual ~ArgWrapper() {} + virtual void pack(BufferBuilder& buffer) const = 0; +}; + +template +struct ScalarArg : public ArgWrapper { + public: + ScalarArg(const T& value) : value_(value) {} + + public: + virtual ~ScalarArg() {} + + public: + virtual void pack(BufferBuilder& buffer) const override { buffer.pack(value_); } + + private: + T value_; +}; + +struct UntypedScalarArg : public ArgWrapper { + public: + UntypedScalarArg(const Scalar& scalar) : scalar_(scalar) {} + + public: + virtual ~UntypedScalarArg() {} + + public: + virtual void pack(BufferBuilder& buffer) const override; + + private: + Scalar scalar_; +}; + +struct RegionFieldArg : public ArgWrapper { + public: + RegionFieldArg(RequirementAnalyzer* analyzer, + LogicalStore store, + int32_t dim, + RegionReq* req, + Legion::FieldID field_id, + Legion::ReductionOpID redop); + + public: + virtual void pack(BufferBuilder& buffer) const override; + + public: + virtual ~RegionFieldArg() {} + + private: + RequirementAnalyzer* analyzer_; + LogicalStore store_; + int32_t dim_; + RegionReq* req_; + Legion::FieldID field_id_; + Legion::ReductionOpID redop_; +}; + +struct FutureStoreArg : public ArgWrapper { + public: + FutureStoreArg(LogicalStore store, bool read_only, bool has_storage, Legion::ReductionOpID redop); + + public: + virtual ~FutureStoreArg() {} + + public: + virtual void pack(BufferBuilder& buffer) const override; + + private: + LogicalStore store_; + bool read_only_; + bool has_storage_; + Legion::ReductionOpID redop_; +}; + +} // namespace legate diff --git a/src/core/runtime/req_analyzer.cc b/src/core/runtime/req_analyzer.cc new file mode 100644 index 0000000000..1ac511e11f --- /dev/null +++ b/src/core/runtime/req_analyzer.cc @@ -0,0 +1,61 @@ +/* Copyright 2022 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/runtime/req_analyzer.h" +#include "core/runtime/launcher.h" + +namespace legate { + +RequirementAnalyzer::~RequirementAnalyzer() +{ + for (auto& pair : requirements_) delete pair.first; +} + +void RequirementAnalyzer::insert(RegionReq* req, Legion::FieldID field_id) +{ + uint32_t req_idx = static_cast(requirements_.size()); + requirements_.push_back(std::make_pair(req, std::vector({field_id}))); + req_indices_[req] = req_idx; +} + +uint32_t RequirementAnalyzer::get_requirement_index(RegionReq* req, Legion::FieldID field_id) const +{ + auto finder = req_indices_.find(req); + assert(finder != req_indices_.end()); + return finder->second; +} + +void RequirementAnalyzer::analyze_requirements() {} + +void RequirementAnalyzer::populate_launcher(Legion::IndexTaskLauncher* task) const +{ + for (auto& pair : requirements_) { + auto& req = pair.first; + auto& fields = pair.second; + req->proj->populate_launcher(task, *req, fields); + } +} + +void RequirementAnalyzer::populate_launcher(Legion::TaskLauncher* task) const +{ + for (auto& pair : requirements_) { + auto& req = pair.first; + auto& fields = pair.second; + req->proj->populate_launcher(task, *req, fields); + } +} + +} // namespace legate diff --git a/src/core/runtime/req_analyzer.h b/src/core/runtime/req_analyzer.h new file mode 100644 index 0000000000..817428bb4d --- /dev/null +++ b/src/core/runtime/req_analyzer.h @@ -0,0 +1,43 @@ +/* Copyright 2022 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include "legate.h" + +namespace legate { + +class RegionReq; + +class RequirementAnalyzer { + public: + ~RequirementAnalyzer(); + + public: + void insert(RegionReq* req, Legion::FieldID field_id); + uint32_t get_requirement_index(RegionReq* req, Legion::FieldID field_id) const; + + public: + void analyze_requirements(); + void populate_launcher(Legion::IndexTaskLauncher* task) const; + void populate_launcher(Legion::TaskLauncher* task) const; + + private: + std::map req_indices_; + std::vector>> requirements_; +}; + +} // namespace legate From 5d462bfcc39ceb0b76197e92ca2f3fc1b97e1ffa Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 29 Jul 2022 11:20:03 -0700 Subject: [PATCH 0061/1425] Requirement coalescing in requirement analyzer --- src/core/data/logical_store.cc | 6 +- src/core/data/logical_store.h | 2 +- src/core/partitioning/partition.cc | 3 +- src/core/runtime/launcher.cc | 142 ++++-------------- src/core/runtime/launcher.h | 107 ++------------ src/core/runtime/launcher_arg.cc | 18 +-- src/core/runtime/launcher_arg.h | 13 +- src/core/runtime/operation.cc | 9 +- src/core/runtime/req_analyzer.cc | 227 ++++++++++++++++++++++++++--- src/core/runtime/req_analyzer.h | 93 +++++++++++- 10 files changed, 365 insertions(+), 255 deletions(-) diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index 9f8ca25869..8fddd02291 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -19,7 +19,7 @@ #include "core/data/logical_store.h" #include "core/data/store.h" #include "core/partitioning/partition.h" -#include "core/runtime/launcher.h" +#include "core/runtime/req_analyzer.h" #include "core/runtime/runtime.h" #include "core/utilities/buffer_builder.h" #include "core/utilities/dispatch.h" @@ -278,7 +278,7 @@ Legion::ProjectionID LogicalStore::compute_projection() const std::unique_ptr LogicalStore::find_or_create_partition(const Partition* partition) { - if (scalar_) return std::make_unique(); + if (scalar_) return std::make_unique(); // We're about to create a legion partition for this store, so the store should have its region // created. @@ -288,7 +288,7 @@ std::unique_ptr LogicalStore::find_or_create_partition(const Partiti auto lr = get_storage()->region(); auto lp = inverted->construct(lr, inverted->is_disjoint_for(nullptr), inverted->is_complete_for(nullptr)); - return std::make_unique(lp, proj); + return std::make_unique(lp, proj); } std::unique_ptr LogicalStore::find_or_create_key_partition() diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index 0644bf1f8c..b25d89683f 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -49,7 +49,7 @@ class LogicalRegionField { public: int32_t dim() const; - Legion::LogicalRegion region() const { return lr_; } + const Legion::LogicalRegion& region() const { return lr_; } Legion::FieldID field_id() const { return fid_; } public: diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index ea3fd830ec..4cd9dc8ce3 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -19,6 +19,7 @@ #include "core/data/logical_store.h" #include "core/partitioning/partition.h" #include "core/runtime/launcher.h" +#include "core/runtime/req_analyzer.h" #include "core/runtime/runtime.h" namespace legate { @@ -56,7 +57,7 @@ Legion::LogicalPartition NoPartition::construct(Legion::LogicalRegion region, std::unique_ptr NoPartition::get_projection(LogicalStore store) const { - return std::make_unique(); + return std::make_unique(); } bool NoPartition::has_launch_domain() const { return false; } diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/launcher.cc index 7f5dc6207f..a0e50a9c45 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -26,108 +26,6 @@ namespace legate { -Projection::Projection(Legion::ReductionOpID r) : redop(r) {} - -void Projection::set_reduction_op(Legion::ReductionOpID r) { redop = r; } - -Replicate::Replicate() : Projection() {} - -Replicate::Replicate(Legion::ReductionOpID redop) : Projection(redop) {} - -void Replicate::populate_launcher(Legion::TaskLauncher* task, - const RegionReq& req, - const std::vector& fields) const -{ - if (req.priv == REDUCE) { -#ifdef DEBUG_LEGATE - assert(redop != -1); -#endif - Legion::RegionRequirement legion_req(req.region, redop, EXCLUSIVE, req.region, req.tag); - legion_req.add_fields(fields); - task->add_region_requirement(legion_req); - } else { - Legion::RegionRequirement legion_req(req.region, req.priv, EXCLUSIVE, req.region, req.tag); - legion_req.add_fields(fields); - task->add_region_requirement(legion_req); - } -} - -void Replicate::populate_launcher(Legion::IndexTaskLauncher* task, - const RegionReq& req, - const std::vector& fields) const -{ - if (req.priv == REDUCE) { -#ifdef DEBUG_LEGATE - assert(redop != -1); -#endif - Legion::RegionRequirement legion_req(req.region, redop, EXCLUSIVE, req.region, req.tag); - legion_req.add_fields(fields); - task->add_region_requirement(legion_req); - } else { - Legion::RegionRequirement legion_req(req.region, req.priv, EXCLUSIVE, req.region, req.tag); - legion_req.add_fields(fields); - task->add_region_requirement(legion_req); - } -} - -MapPartition::MapPartition(Legion::LogicalPartition partition, Legion::ProjectionID proj_id) - : Projection(), partition_(partition), proj_id_(proj_id) -{ -} - -MapPartition::MapPartition(Legion::LogicalPartition partition, - Legion::ProjectionID proj_id, - Legion::ReductionOpID redop) - : Projection(redop), partition_(partition), proj_id_(proj_id) -{ -} - -void MapPartition::populate_launcher(Legion::TaskLauncher* task, - const RegionReq& req, - const std::vector& fields) const -{ - if (req.priv == REDUCE) { -#ifdef DEBUG_LEGATE - assert(redop != -1); -#endif - Legion::RegionRequirement legion_req(req.region, redop, EXCLUSIVE, req.region, req.tag); - legion_req.add_fields(fields); - task->add_region_requirement(legion_req); - } else { - Legion::RegionRequirement legion_req(req.region, req.priv, EXCLUSIVE, req.region, req.tag); - legion_req.add_fields(fields); - task->add_region_requirement(legion_req); - } -} - -void MapPartition::populate_launcher(Legion::IndexTaskLauncher* task, - const RegionReq& req, - const std::vector& fields) const -{ - if (req.priv == REDUCE) { -#ifdef DEBUG_LEGATE - assert(redop != -1); -#endif - Legion::RegionRequirement legion_req( - partition_, proj_id_, redop, EXCLUSIVE, req.region, req.tag); - legion_req.add_fields(fields); - task->add_region_requirement(legion_req); - } else { - Legion::RegionRequirement legion_req( - partition_, proj_id_, req.priv, EXCLUSIVE, req.region, req.tag); - legion_req.add_fields(fields); - task->add_region_requirement(legion_req); - } -} - -RegionReq::RegionReq(Legion::LogicalRegion _region, - Legion::PrivilegeMode _priv, - ProjectionP _proj, - int64_t _tag) - : region(_region), priv(_priv), proj(std::move(_proj)), tag(_tag) -{ -} - TaskLauncher::TaskLauncher(Runtime* runtime, LibraryContext* library, int64_t task_id, @@ -159,23 +57,30 @@ void TaskLauncher::add_scalar(const Scalar& scalar) scalars_.push_back(new UntypedScalarArg(scalar)); } -void TaskLauncher::add_input(LogicalStore store, ProjectionP proj, uint64_t tag /*= 0*/) +void TaskLauncher::add_input(LogicalStore store, + std::unique_ptr proj, + Legion::MappingTagID tag, + Legion::RegionFlags flags) { - add_store(inputs_, std::move(store), std::move(proj), READ_ONLY, tag); + add_store(inputs_, std::move(store), std::move(proj), READ_ONLY, tag, flags); } -void TaskLauncher::add_output(LogicalStore store, ProjectionP proj, uint64_t tag /*= 0*/) +void TaskLauncher::add_output(LogicalStore store, + std::unique_ptr proj, + Legion::MappingTagID tag, + Legion::RegionFlags flags) { - add_store(outputs_, std::move(store), std::move(proj), WRITE_ONLY, tag); + add_store(outputs_, std::move(store), std::move(proj), WRITE_ONLY, tag, flags); } void TaskLauncher::add_reduction(LogicalStore store, - ProjectionP proj, - uint64_t tag /*= 0*/, + std::unique_ptr proj, + Legion::MappingTagID tag, + Legion::RegionFlags flags, bool read_write /*= false*/) { assert(!read_write); - add_store(reductions_, std::move(store), std::move(proj), REDUCE, tag); + add_store(reductions_, std::move(store), std::move(proj), REDUCE, tag, flags); } void TaskLauncher::execute(const Legion::Domain& launch_domain) @@ -194,9 +99,10 @@ void TaskLauncher::execute_single() void TaskLauncher::add_store(std::vector& args, LogicalStore store, - ProjectionP proj, + std::unique_ptr proj, Legion::PrivilegeMode privilege, - uint64_t tag) + Legion::MappingTagID tag, + Legion::RegionFlags flags) { auto redop = proj->redop; @@ -210,11 +116,11 @@ void TaskLauncher::add_store(std::vector& args, auto region = storage->region(); auto field_id = storage->field_id(); - auto req = new RegionReq(region, privilege, std::move(proj), tag); + auto proj_info = new ProjectionInfo(proj.get(), tag, flags); - req_analyzer_->insert(req, field_id); + req_analyzer_->insert(region, field_id, privilege, proj_info); args.push_back( - new RegionFieldArg(req_analyzer_, std::move(store), region.get_dim(), req, field_id, redop)); + new RegionFieldArg(req_analyzer_, std::move(store), field_id, privilege, proj_info)); } } @@ -226,6 +132,10 @@ void TaskLauncher::pack_args(const std::vector& args) Legion::TaskLauncher* TaskLauncher::build_single_task() { + // Coalesce region requirements before packing task arguments + // as the latter requires requirement indices to be finalized + req_analyzer_->analyze_requirements(); + pack_args(inputs_); pack_args(outputs_); pack_args(reductions_); @@ -248,6 +158,10 @@ Legion::TaskLauncher* TaskLauncher::build_single_task() Legion::IndexTaskLauncher* TaskLauncher::build_index_task(const Legion::Domain& launch_domain) { + // Coalesce region requirements before packing task arguments + // as the latter requires requirement indices to be finalized + req_analyzer_->analyze_requirements(); + pack_args(inputs_); pack_args(outputs_); pack_args(reductions_); diff --git a/src/core/runtime/launcher.h b/src/core/runtime/launcher.h index 9c119a3284..b6d30073cb 100644 --- a/src/core/runtime/launcher.h +++ b/src/core/runtime/launcher.h @@ -26,95 +26,12 @@ class ArgWrapper; class BufferBuilder; class LibraryContext; class LogicalStore; -class RegionReq; +class Projection; class RequirementAnalyzer; class Runtime; class Scalar; -class Projection { - protected: - Projection() {} - Projection(Legion::ReductionOpID redop); - - public: - virtual ~Projection() {} - - public: - virtual void populate_launcher(Legion::TaskLauncher* task, - const RegionReq& req, - const std::vector& fields) const = 0; - virtual void populate_launcher(Legion::IndexTaskLauncher* task, - const RegionReq& req, - const std::vector& fields) const = 0; - - public: - void set_reduction_op(Legion::ReductionOpID redop); - - public: - Legion::ReductionOpID redop{-1}; -}; - -class Replicate : public Projection { - public: - Replicate(); - Replicate(Legion::ReductionOpID redop); - - public: - virtual ~Replicate() {} - - public: - virtual void populate_launcher(Legion::TaskLauncher* task, - const RegionReq& req, - const std::vector& fields) const override; - virtual void populate_launcher(Legion::IndexTaskLauncher* task, - const RegionReq& req, - const std::vector& fields) const override; -}; - -class MapPartition : public Projection { - public: - MapPartition(Legion::LogicalPartition partition, Legion::ProjectionID proj_id); - MapPartition(Legion::LogicalPartition partition, - Legion::ProjectionID proj_id, - Legion::ReductionOpID redop); - - public: - virtual ~MapPartition() {} - - public: - virtual void populate_launcher(Legion::TaskLauncher* task, - const RegionReq& req, - const std::vector& fields) const override; - virtual void populate_launcher(Legion::IndexTaskLauncher* task, - const RegionReq& req, - const std::vector& fields) const override; - - private: - Legion::LogicalPartition partition_; - Legion::ProjectionID proj_id_; -}; - -class RegionReq { - private: - using ProjectionP = std::unique_ptr; - - public: - RegionReq(Legion::LogicalRegion region, - Legion::PrivilegeMode priv, - ProjectionP proj, - int64_t tag); - - public: - Legion::LogicalRegion region; - Legion::PrivilegeMode priv; - ProjectionP proj; - int64_t tag; -}; - class TaskLauncher { - private: - using ProjectionP = std::unique_ptr; - public: TaskLauncher(Runtime* runtime, LibraryContext* library, @@ -129,19 +46,27 @@ class TaskLauncher { public: void add_scalar(const Scalar& scalar); - void add_input(LogicalStore store, ProjectionP proj, uint64_t tag = 0); - void add_output(LogicalStore store, ProjectionP proj, uint64_t tag = 0); + void add_input(LogicalStore store, + std::unique_ptr proj, + Legion::MappingTagID tag = 0, + Legion::RegionFlags flags = LEGION_NO_FLAG); + void add_output(LogicalStore store, + std::unique_ptr proj, + Legion::MappingTagID tag = 0, + Legion::RegionFlags flags = LEGION_NO_FLAG); void add_reduction(LogicalStore store, - ProjectionP proj, - uint64_t tag = 0, - bool read_write = false); + std::unique_ptr proj, + Legion::MappingTagID tag = 0, + Legion::RegionFlags flags = LEGION_NO_FLAG, + bool read_write = false); private: void add_store(std::vector& args, LogicalStore store, - ProjectionP proj, + std::unique_ptr proj, Legion::PrivilegeMode privilege, - uint64_t tag); + Legion::MappingTagID tag, + Legion::RegionFlags flags); public: void execute(const Legion::Domain& launch_domain); diff --git a/src/core/runtime/launcher_arg.cc b/src/core/runtime/launcher_arg.cc index 85d943642e..006bc5a6a9 100644 --- a/src/core/runtime/launcher_arg.cc +++ b/src/core/runtime/launcher_arg.cc @@ -29,26 +29,24 @@ void UntypedScalarArg::pack(BufferBuilder& buffer) const RegionFieldArg::RegionFieldArg(RequirementAnalyzer* analyzer, LogicalStore store, - int32_t dim, - RegionReq* req, Legion::FieldID field_id, - Legion::ReductionOpID redop) + Legion::PrivilegeMode privilege, + const ProjectionInfo* proj_info) : analyzer_(analyzer), store_(std::move(store)), - dim_(dim), - req_(req), + region_(store_.get_storage()->region()), field_id_(field_id), - redop_(redop) + privilege_(privilege), + proj_info_(proj_info) { } void RegionFieldArg::pack(BufferBuilder& buffer) const { store_.pack(buffer); - - buffer.pack(redop_); - buffer.pack(dim_); - buffer.pack(analyzer_->get_requirement_index(req_, field_id_)); + buffer.pack(proj_info_->redop); + buffer.pack(region_.get_dim()); + buffer.pack(analyzer_->get_requirement_index(region_, privilege_, proj_info_)); buffer.pack(field_id_); } diff --git a/src/core/runtime/launcher_arg.h b/src/core/runtime/launcher_arg.h index a018f69686..4efc114af3 100644 --- a/src/core/runtime/launcher_arg.h +++ b/src/core/runtime/launcher_arg.h @@ -22,7 +22,7 @@ namespace legate { -class RegionReq; +class ProjectionInfo; class RequirementAnalyzer; struct ArgWrapper { @@ -63,10 +63,9 @@ struct RegionFieldArg : public ArgWrapper { public: RegionFieldArg(RequirementAnalyzer* analyzer, LogicalStore store, - int32_t dim, - RegionReq* req, Legion::FieldID field_id, - Legion::ReductionOpID redop); + Legion::PrivilegeMode privilege, + const ProjectionInfo* proj_info); public: virtual void pack(BufferBuilder& buffer) const override; @@ -77,10 +76,10 @@ struct RegionFieldArg : public ArgWrapper { private: RequirementAnalyzer* analyzer_; LogicalStore store_; - int32_t dim_; - RegionReq* req_; + Legion::LogicalRegion region_; Legion::FieldID field_id_; - Legion::ReductionOpID redop_; + Legion::PrivilegeMode privilege_; + const ProjectionInfo* proj_info_; }; struct FutureStoreArg : public ArgWrapper { diff --git a/src/core/runtime/operation.cc b/src/core/runtime/operation.cc index 89ce7d978c..80d4316064 100644 --- a/src/core/runtime/operation.cc +++ b/src/core/runtime/operation.cc @@ -25,6 +25,7 @@ #include "core/runtime/context.h" #include "core/runtime/launcher.h" #include "core/runtime/operation.h" +#include "core/runtime/req_analyzer.h" #include "core/runtime/runtime.h" namespace legate { @@ -41,9 +42,7 @@ Operation::Operation(Runtime* runtime, { } -Operation::~Operation() -{ -} +Operation::~Operation() {} void Operation::add_input(LogicalStore store, std::shared_ptr partition) { @@ -96,9 +95,7 @@ Task::Task(Runtime* runtime, { } -Task::~Task() -{ -} +Task::~Task() {} void Task::add_scalar_arg(const Scalar& scalar) { scalars_.push_back(scalar); } diff --git a/src/core/runtime/req_analyzer.cc b/src/core/runtime/req_analyzer.cc index 1ac511e11f..f329e29657 100644 --- a/src/core/runtime/req_analyzer.cc +++ b/src/core/runtime/req_analyzer.cc @@ -19,42 +19,235 @@ namespace legate { -RequirementAnalyzer::~RequirementAnalyzer() +///////////// +// Projection +///////////// + +Projection::Projection(Legion::LogicalPartition p, Legion::ProjectionID pr) + : partition(p), proj_id(pr) +{ +} + +void Projection::set_reduction_op(Legion::ReductionOpID r) { redop = r; } + +///////////////// +// ProjectionInfo +///////////////// + +ProjectionInfo::ProjectionInfo(const Projection* proj, + Legion::MappingTagID _tag, + Legion::RegionFlags _flags) + : partition(proj->partition), proj_id(proj->proj_id), redop(proj->redop), tag(_tag), flags(_flags) +{ +} + +bool ProjectionInfo::operator<(const ProjectionInfo& other) const +{ + if (partition < other.partition) + return true; + else if (other.partition < partition) + return false; + if (proj_id < other.proj_id) + return true; + else if (proj_id > other.proj_id) + return false; + if (redop < other.redop) + return true; + else if (redop > other.redop) + return false; + if (tag < other.tag) + return true; + else + return false; +} + +bool ProjectionInfo::operator==(const ProjectionInfo& other) const +{ + return partition == other.partition && proj_id == other.proj_id && redop == other.redop && + tag == other.tag && flags == other.flags; +} + +void ProjectionInfo::populate_launcher(Legion::TaskLauncher* task, + const Legion::LogicalRegion& region, + const std::vector& fields, + Legion::PrivilegeMode privilege) const +{ + Legion::RegionRequirement legion_req; + + if (LEGION_REDUCE == privilege) { +#ifdef DEBUG_LEGATE + assert(redop != -1); +#endif + new (&legion_req) Legion::RegionRequirement(region, redop, LEGION_EXCLUSIVE, region, tag); + } else { + new (&legion_req) Legion::RegionRequirement(region, privilege, LEGION_EXCLUSIVE, region, tag); + } + + legion_req.add_fields(fields).add_flags(flags); + task->add_region_requirement(legion_req); +} + +void ProjectionInfo::populate_launcher(Legion::IndexTaskLauncher* task, + const Legion::LogicalRegion& region, + const std::vector& fields, + Legion::PrivilegeMode privilege) const +{ + Legion::RegionRequirement legion_req; + + // Broadcast + if (Legion::LogicalPartition::NO_PART == partition) { + if (LEGION_REDUCE == privilege) { +#ifdef DEBUG_LEGATE + assert(redop != -1); +#endif + new (&legion_req) Legion::RegionRequirement(region, redop, LEGION_EXCLUSIVE, region, tag); + } else { + new (&legion_req) Legion::RegionRequirement(region, privilege, LEGION_EXCLUSIVE, region, tag); + } + } else { + if (LEGION_REDUCE == privilege) { +#ifdef DEBUG_LEGATE + assert(redop != -1); +#endif + new (&legion_req) + Legion::RegionRequirement(partition, proj_id, redop, LEGION_EXCLUSIVE, region, tag); + } else { + new (&legion_req) + Legion::RegionRequirement(partition, proj_id, privilege, LEGION_EXCLUSIVE, region, tag); + } + } + + legion_req.add_fields(fields).add_flags(flags); + task->add_region_requirement(legion_req); +} + +//////////////// +// ProjectionSet +//////////////// + +void ProjectionSet::insert(Legion::PrivilegeMode new_privilege, const ProjectionInfo* proj_info) { - for (auto& pair : requirements_) delete pair.first; + if (proj_infos.empty()) privilege = new_privilege; + // conflicting privileges are promoted to a single read-write privilege + else if (privilege != new_privilege) + privilege = LEGION_READ_WRITE; + proj_infos.emplace(*proj_info); + + if (privilege != LEGION_READ_ONLY && proj_infos.size() > 1) { + log_legate.error("Interfering requirements are found"); + LEGATE_ABORT; + } } -void RequirementAnalyzer::insert(RegionReq* req, Legion::FieldID field_id) +/////////// +// FieldSet +/////////// + +void FieldSet::insert(Legion::FieldID field_id, + Legion::PrivilegeMode privilege, + const ProjectionInfo* proj_info) { - uint32_t req_idx = static_cast(requirements_.size()); - requirements_.push_back(std::make_pair(req, std::vector({field_id}))); - req_indices_[req] = req_idx; + field_projs_[field_id].insert(privilege, proj_info); } -uint32_t RequirementAnalyzer::get_requirement_index(RegionReq* req, Legion::FieldID field_id) const +uint32_t FieldSet::num_requirements() const { return static_cast(coalesced_.size()); } + +uint32_t FieldSet::get_requirement_index(Legion::PrivilegeMode privilege, + const ProjectionInfo* proj_info) const { - auto finder = req_indices_.find(req); + auto finder = req_indices_.find(Key(privilege, *proj_info)); +#ifdef DEBUG_LEGATE assert(finder != req_indices_.end()); +#endif return finder->second; } -void RequirementAnalyzer::analyze_requirements() {} +void FieldSet::coalesce() +{ + for (const auto& entry : field_projs_) { + const auto& proj_set = entry.second; + for (const auto& proj_info : proj_set.proj_infos) + coalesced_[Key(proj_set.privilege, proj_info)].push_back(entry.first); + } + uint32_t idx = 0; + for (const auto& entry : coalesced_) req_indices_[entry.first] = idx++; +} + +void FieldSet::populate_launcher(Legion::IndexTaskLauncher* task, + const Legion::LogicalRegion& region) const +{ + for (auto& entry : coalesced_) { + auto privilege = entry.first.first; + const auto& proj_info = entry.first.second; + const auto& fields = entry.second; + proj_info.populate_launcher(task, region, fields, privilege); + } +} + +void FieldSet::populate_launcher(Legion::TaskLauncher* task, + const Legion::LogicalRegion& region) const +{ + for (auto& entry : coalesced_) { + auto privilege = entry.first.first; + const auto& proj_info = entry.first.second; + const auto& fields = entry.second; + proj_info.populate_launcher(task, region, fields, privilege); + } +} + +////////////////////// +// RequirementAnalyzer +////////////////////// + +RequirementAnalyzer::~RequirementAnalyzer() {} + +void RequirementAnalyzer::insert(const Legion::LogicalRegion& region, + Legion::FieldID field_id, + Legion::PrivilegeMode privilege, + const ProjectionInfo* proj_info) +{ + field_sets_[region].first.insert(field_id, privilege, proj_info); +} + +uint32_t RequirementAnalyzer::get_requirement_index(const Legion::LogicalRegion& region, + Legion::PrivilegeMode privilege, + const ProjectionInfo* proj_info) const +{ + auto finder = field_sets_.find(region); +#ifdef DEBUG_LEGATE + assert(finder != field_sets_.end()); +#endif + auto& field_set = finder->second.first; + auto& req_offset = finder->second.second; + return req_offset + field_set.get_requirement_index(privilege, proj_info); +} + +void RequirementAnalyzer::analyze_requirements() +{ + uint32_t num_reqs = 0; + for (auto& entry : field_sets_) { + auto& field_set = entry.second.first; + auto& req_offset = entry.second.second; + + field_set.coalesce(); + req_offset = num_reqs; + num_reqs += field_set.num_requirements(); + } +} void RequirementAnalyzer::populate_launcher(Legion::IndexTaskLauncher* task) const { - for (auto& pair : requirements_) { - auto& req = pair.first; - auto& fields = pair.second; - req->proj->populate_launcher(task, *req, fields); + for (auto& entry : field_sets_) { + const auto& field_set = entry.second.first; + field_set.populate_launcher(task, entry.first); } } void RequirementAnalyzer::populate_launcher(Legion::TaskLauncher* task) const { - for (auto& pair : requirements_) { - auto& req = pair.first; - auto& fields = pair.second; - req->proj->populate_launcher(task, *req, fields); + for (auto& entry : field_sets_) { + const auto& field_set = entry.second.first; + field_set.populate_launcher(task, entry.first); } } diff --git a/src/core/runtime/req_analyzer.h b/src/core/runtime/req_analyzer.h index 817428bb4d..45c6fb0a6b 100644 --- a/src/core/runtime/req_analyzer.h +++ b/src/core/runtime/req_analyzer.h @@ -16,19 +16,103 @@ #pragma once +#include #include "legate.h" namespace legate { -class RegionReq; +class Projection { + public: + Projection() {} + Projection(Legion::LogicalPartition partition, Legion::ProjectionID proj_id); + + public: + void set_reduction_op(Legion::ReductionOpID redop); + + public: + const Legion::LogicalPartition partition{Legion::LogicalPartition::NO_PART}; + const Legion::ProjectionID proj_id{0}; + // TODO: Make this const as well + Legion::ReductionOpID redop{-1}; +}; + +class ProjectionInfo { + public: + ProjectionInfo(const Projection* proj, Legion::MappingTagID tag, Legion::RegionFlags flags); + + public: + ProjectionInfo(const ProjectionInfo&) = default; + ProjectionInfo& operator=(const ProjectionInfo&) = default; + + public: + bool operator<(const ProjectionInfo& other) const; + bool operator==(const ProjectionInfo& other) const; + + public: + void populate_launcher(Legion::TaskLauncher* task, + const Legion::LogicalRegion& region, + const std::vector& fields, + Legion::PrivilegeMode privilege) const; + void populate_launcher(Legion::IndexTaskLauncher* task, + const Legion::LogicalRegion& region, + const std::vector& fields, + Legion::PrivilegeMode privilege) const; + + public: + Legion::LogicalPartition partition; + Legion::ProjectionID proj_id; + Legion::ReductionOpID redop; + Legion::MappingTagID tag; + Legion::RegionFlags flags; +}; + +class ProjectionSet { + public: + void insert(Legion::PrivilegeMode new_privilege, const ProjectionInfo* proj_info); + + public: + Legion::PrivilegeMode privilege; + std::set proj_infos; +}; + +class FieldSet { + public: + using Key = std::pair; + + public: + void insert(Legion::FieldID field_id, + Legion::PrivilegeMode privilege, + const ProjectionInfo* proj_info); + uint32_t num_requirements() const; + uint32_t get_requirement_index(Legion::PrivilegeMode privilege, + const ProjectionInfo* proj_info) const; + + public: + void coalesce(); + void populate_launcher(Legion::IndexTaskLauncher* task, + const Legion::LogicalRegion& region) const; + void populate_launcher(Legion::TaskLauncher* task, const Legion::LogicalRegion& region) const; + + private: + std::map> coalesced_; + std::map req_indices_; + + private: + std::map field_projs_; +}; class RequirementAnalyzer { public: ~RequirementAnalyzer(); public: - void insert(RegionReq* req, Legion::FieldID field_id); - uint32_t get_requirement_index(RegionReq* req, Legion::FieldID field_id) const; + void insert(const Legion::LogicalRegion& region, + Legion::FieldID field_id, + Legion::PrivilegeMode privilege, + const ProjectionInfo* proj_info); + uint32_t get_requirement_index(const Legion::LogicalRegion& region, + Legion::PrivilegeMode privilege, + const ProjectionInfo* proj_info) const; public: void analyze_requirements(); @@ -36,8 +120,7 @@ class RequirementAnalyzer { void populate_launcher(Legion::TaskLauncher* task) const; private: - std::map req_indices_; - std::vector>> requirements_; + std::map> field_sets_; }; } // namespace legate From 4c07c9eca56e57c824ac69c53d41e81b5ada7ca0 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Sat, 30 Jul 2022 00:45:02 -0700 Subject: [PATCH 0062/1425] Add a cache for tiling partitions to the partition manager --- src/core/data/logical_store.cc | 2 +- src/core/partitioning/partition.cc | 65 ++++++++++++++---------------- src/core/partitioning/partition.h | 13 ++---- src/core/runtime/runtime.cc | 32 ++++++++++++--- src/core/runtime/runtime.h | 23 ++++++++--- 5 files changed, 79 insertions(+), 56 deletions(-) diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index 8fddd02291..c6b76bfbad 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -295,7 +295,7 @@ std::unique_ptr LogicalStore::find_or_create_key_partition() { if (scalar_) return create_no_partition(runtime_); - auto part_mgr = runtime_->get_partition_manager(); + auto part_mgr = runtime_->partition_manager(); auto launch_shape = part_mgr->compute_launch_shape(extents()); if (launch_shape.empty()) return create_no_partition(runtime_); diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index 4cd9dc8ce3..a91000b531 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -24,22 +24,6 @@ namespace legate { -struct PartitionByRestriction : public PartitioningFunctor { - public: - PartitionByRestriction(Legion::DomainTransform transform, Legion::Domain extent); - - public: - virtual Legion::IndexPartition construct(Legion::Runtime* legion_runtime, - Legion::Context legion_context, - const Legion::IndexSpace& parent, - const Legion::IndexSpace& color_space, - Legion::PartitionKind kind) const override; - - private: - Legion::DomainTransform transform_; - Legion::Domain extent_; -}; - Partition::Partition(Runtime* runtime) : runtime_(runtime) {} NoPartition::NoPartition(Runtime* runtime) : Partition(runtime) {} @@ -81,6 +65,28 @@ Tiling::Tiling(Runtime* runtime, Shape&& tile_shape, Shape&& color_shape, Shape& assert(tile_shape_.size() == offsets_.size()); } +bool Tiling::operator==(const Tiling& other) const +{ + return tile_shape_ == other.tile_shape_ && color_shape_ == other.color_shape_ && + offsets_ == other.offsets_; +} + +bool Tiling::operator<(const Tiling& other) const +{ + if (tile_shape_ < other.tile_shape_) + return true; + else if (other.tile_shape_ < tile_shape_) + return false; + if (color_shape_ < other.color_shape_) + return true; + else if (other.color_shape_ < color_shape_) + return false; + if (offsets_ < other.offsets_) + return true; + else + return false; +} + bool Tiling::is_complete_for(const LogicalStore* store) const { return false; } bool Tiling::is_disjoint_for(const LogicalStore* store) const { return true; } @@ -89,6 +95,11 @@ Legion::LogicalPartition Tiling::construct(Legion::LogicalRegion region, bool disjoint, bool complete) const { + auto index_space = region.get_index_space(); + auto index_partition = runtime_->partition_manager()->find_index_partition(index_space, *this); + if (index_partition != Legion::IndexPartition::NO_PART) + return runtime_->create_logical_partition(region, index_partition); + auto ndim = static_cast(tile_shape_.size()); Legion::DomainTransform transform; @@ -112,13 +123,13 @@ Legion::LogicalPartition Tiling::construct(Legion::LogicalRegion region, } auto color_space = runtime_->find_or_create_index_space(color_domain); - auto index_space = region.get_index_space(); auto kind = complete ? (disjoint ? LEGION_DISJOINT_COMPLETE_KIND : LEGION_ALIASED_COMPLETE_KIND) : (disjoint ? LEGION_DISJOINT_KIND : LEGION_ALIASED_KIND); - PartitionByRestriction functor(transform, extent); - auto index_partition = runtime_->create_index_partition(index_space, color_space, kind, &functor); + index_partition = + runtime_->create_restricted_partition(index_space, color_space, kind, transform, extent); + runtime_->partition_manager()->record_index_partition(index_space, *this, index_partition); return runtime_->create_logical_partition(region, index_partition); } @@ -149,22 +160,6 @@ std::string Tiling::to_string() const return ss.str(); } -PartitionByRestriction::PartitionByRestriction(Legion::DomainTransform transform, - Legion::Domain extent) - : transform_(transform), extent_(extent) -{ -} - -Legion::IndexPartition PartitionByRestriction::construct(Legion::Runtime* legion_runtime, - Legion::Context legion_context, - const Legion::IndexSpace& parent, - const Legion::IndexSpace& color_space, - Legion::PartitionKind kind) const -{ - return legion_runtime->create_partition_by_restriction( - legion_context, parent, color_space, transform_, extent_, kind); -} - std::unique_ptr create_tiling(Runtime* runtime, Shape&& tile_shape, Shape&& color_shape, diff --git a/src/core/partitioning/partition.h b/src/core/partitioning/partition.h index 061d544ef4..8eb5244716 100644 --- a/src/core/partitioning/partition.h +++ b/src/core/partitioning/partition.h @@ -65,15 +65,6 @@ struct Partition { Runtime* runtime_; }; -struct PartitioningFunctor { - public: - virtual Legion::IndexPartition construct(Legion::Runtime* legion_runtime, - Legion::Context legion_context, - const Legion::IndexSpace& parent, - const Legion::IndexSpace& color_space, - Legion::PartitionKind kind) const = 0; -}; - class NoPartition : public Partition { public: NoPartition(Runtime* runtime); @@ -106,6 +97,10 @@ class Tiling : public Partition { public: Tiling(Runtime* runtime, Shape&& tile_shape, Shape&& color_shape, Shape&& offsets); + public: + bool operator==(const Tiling& other) const; + bool operator<(const Tiling& other) const; + public: virtual Kind kind() const override { return Kind::TILING; } diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 7102db2e95..3f51b98818 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -516,6 +516,23 @@ tuple PartitionManager::compute_tile_shape(const tuple& extents, return std::move(tile_shape); } +Legion::IndexPartition PartitionManager::find_index_partition(const Legion::IndexSpace& index_space, + const Tiling& tiling) const +{ + auto finder = tiling_cache_.find(std::make_pair(index_space, tiling)); + if (finder != tiling_cache_.end()) + return finder->second; + else + return Legion::IndexPartition::NO_PART; +} + +void PartitionManager::record_index_partition(const Legion::IndexSpace& index_space, + const Tiling& tiling, + const Legion::IndexPartition& index_partition) +{ + tiling_cache_[std::make_pair(index_space, tiling)] = index_partition; +} + //////////////////////////////////////////////////// // legate::Runtime //////////////////////////////////////////////////// @@ -664,7 +681,7 @@ FieldManager* Runtime::find_or_create_field_manager(const Domain& shape, LegateT } } -PartitionManager* Runtime::get_partition_manager() { return partition_manager_; } +PartitionManager* Runtime::partition_manager() const { return partition_manager_; } IndexSpace Runtime::find_or_create_index_space(const Domain& shape) { @@ -679,12 +696,15 @@ IndexSpace Runtime::find_or_create_index_space(const Domain& shape) } } -Legion::IndexPartition Runtime::create_index_partition(const Legion::IndexSpace& index_space, - const Legion::IndexSpace& color_space, - Legion::PartitionKind kind, - const PartitioningFunctor* functor) +Legion::IndexPartition Runtime::create_restricted_partition( + const Legion::IndexSpace& index_space, + const Legion::IndexSpace& color_space, + Legion::PartitionKind kind, + const Legion::DomainTransform& transform, + const Legion::Domain& extent) { - return functor->construct(legion_runtime_, legion_context_, index_space, color_space, kind); + return legion_runtime_->create_partition_by_restriction( + legion_context_, index_space, color_space, transform, extent, kind); } FieldSpace Runtime::create_field_space() diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index fdcfcba158..ebd295e950 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -60,6 +60,7 @@ class RegionManager; class ResourceConfig; class Runtime; class Task; +class Tiling; class PartitionManager { public: @@ -69,10 +70,21 @@ class PartitionManager { tuple compute_launch_shape(const tuple& shape); tuple compute_tile_shape(const tuple& extents, const tuple& launch_shape); + public: + Legion::IndexPartition find_index_partition(const Legion::IndexSpace& index_space, + const Tiling& tiling) const; + void record_index_partition(const Legion::IndexSpace& index_space, + const Tiling& tiling, + const Legion::IndexPartition& index_partition); + private: int32_t num_pieces_; int64_t min_shard_volume_; std::vector piece_factors_; + + private: + using TilingCacheKey = std::pair; + std::map tiling_cache_; }; class Runtime { @@ -113,14 +125,15 @@ class Runtime { public: RegionManager* find_or_create_region_manager(const Legion::Domain& shape); FieldManager* find_or_create_field_manager(const Legion::Domain& shape, LegateTypeCode code); - PartitionManager* get_partition_manager(); + PartitionManager* partition_manager() const; public: Legion::IndexSpace find_or_create_index_space(const Legion::Domain& shape); - Legion::IndexPartition create_index_partition(const Legion::IndexSpace& index_space, - const Legion::IndexSpace& color_space, - Legion::PartitionKind kind, - const PartitioningFunctor* functor); + Legion::IndexPartition create_restricted_partition(const Legion::IndexSpace& index_space, + const Legion::IndexSpace& color_space, + Legion::PartitionKind kind, + const Legion::DomainTransform& transform, + const Legion::Domain& extent); Legion::FieldSpace create_field_space(); Legion::LogicalRegion create_region(const Legion::IndexSpace& index_space, const Legion::FieldSpace& field_space); From ec154ba8ff7160a201e4fc69c76de1bb852f5ca3 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Thu, 4 Aug 2022 23:10:06 -0700 Subject: [PATCH 0063/1425] Handle the bottom of transform stacks correctly in the remaining methods --- src/core/data/logical_store.cc | 19 ++++++++++++++----- src/core/data/logical_store.h | 5 +++-- src/core/data/transform.cc | 12 ++++++++---- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index c6b76bfbad..fc343265d8 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -143,7 +143,11 @@ struct datalen_fn { }; LogicalStore::LogicalStore(Runtime* runtime, LegateTypeCode code, const void* data) - : scalar_(true), runtime_(runtime), code_(code), extents_({1}) + : scalar_(true), + runtime_(runtime), + code_(code), + extents_({1}), + transform_(std::make_shared()) { auto datalen = type_dispatch(code, datalen_fn{}); future_ = runtime_->create_future(data, datalen); @@ -311,8 +315,7 @@ void LogicalStore::pack(BufferBuilder& buffer) const buffer.pack(false); buffer.pack(dim()); buffer.pack(code_); - if (transform_ != nullptr) transform_->pack(buffer); - buffer.pack(-1); + transform_->pack(buffer); } void LogicalStore::pack_transform(BufferBuilder& buffer) const @@ -329,11 +332,17 @@ void LogicalStore::pack_transform(BufferBuilder& buffer) const LogicalStore::LogicalStore() {} +LogicalStore::LogicalStore(Runtime* runtime, LegateTypeCode code, tuple extents) + : impl_(std::make_shared( + runtime, code, std::move(extents), nullptr, std::make_shared())) +{ +} + LogicalStore::LogicalStore(Runtime* runtime, LegateTypeCode code, tuple extents, - LogicalStore parent, /* = LogicalStore() */ - std::shared_ptr transform /*= nullptr*/) + LogicalStore parent, + std::shared_ptr transform) : impl_(std::make_shared( runtime, code, std::move(extents), parent.impl_, std::move(transform))) { diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index b25d89683f..43f3517469 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -64,11 +64,12 @@ class LogicalRegionField { class LogicalStore { public: LogicalStore(); + LogicalStore(Runtime* runtime, LegateTypeCode code, tuple extents); LogicalStore(Runtime* runtime, LegateTypeCode code, tuple extents, - LogicalStore parent = LogicalStore(), - std::shared_ptr transform = nullptr); + LogicalStore parent, + std::shared_ptr transform); // Creates a read-only store from a scalar LogicalStore(Runtime* runtime, LegateTypeCode code, const void* data); diff --git a/src/core/data/transform.cc b/src/core/data/transform.cc index 9403a740f9..552ad6572b 100644 --- a/src/core/data/transform.cc +++ b/src/core/data/transform.cc @@ -49,19 +49,23 @@ TransformStack::TransformStack(std::unique_ptr&& transform, std::unique_ptr TransformStack::invert_partition(const Partition* partition) const { auto result = transform_->invert_partition(partition); - return parent_ != nullptr ? parent_->invert_partition(result.get()) : std::move(result); + return parent_->identity() ? std::move(result) : parent_->invert_partition(result.get()); } proj::SymbolicPoint TransformStack::invert(const proj::SymbolicPoint& point) const { auto result = transform_->invert(point); - return parent_ != nullptr ? parent_->invert(result) : result; + return parent_->identity() ? result : parent_->invert(result); } void TransformStack::pack(BufferBuilder& buffer) const { - transform_->pack(buffer); - if (parent_ != nullptr) parent_->pack(buffer); + if (identity()) + buffer.pack(-1); + else { + transform_->pack(buffer); + parent_->pack(buffer); + } } Legion::Domain TransformStack::transform(const Legion::Domain& input) const From 6fca007c635ec7dc0baf7fc0c9b85374f6d8dd10 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Thu, 4 Aug 2022 23:20:34 -0700 Subject: [PATCH 0064/1425] Move the logic for pushing transforms to stacks to TransformStack --- src/core/data/logical_store.cc | 9 +-------- src/core/data/transform.cc | 6 ++++++ src/core/data/transform.h | 4 ++-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index fc343265d8..6e87da8714 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -211,14 +211,7 @@ std::shared_ptr LogicalStore::promote(int32_t extra_dim, } auto new_extents = extents_.insert(extra_dim, dim_size); - // TODO: Move this push operation to TransformStack. - // Two prerequisites: - // 1) make members of TransformStack read only (i.e., by adding const) - // 2) make TransformStack inherit std::enable_shared_from_this. - // Then we can add a const push method to TransformStack that returns - // a fresh shared_ptr of TransformStack with a transform put on top. - auto transform = - std::make_shared(std::make_unique(extra_dim, dim_size), transform_); + auto transform = transform_->push(std::make_unique(extra_dim, dim_size)); return std::make_shared( runtime_, code_, std::move(new_extents), std::move(parent), std::move(transform)); } diff --git a/src/core/data/transform.cc b/src/core/data/transform.cc index 552ad6572b..4d288ebe10 100644 --- a/src/core/data/transform.cc +++ b/src/core/data/transform.cc @@ -117,6 +117,12 @@ std::unique_ptr TransformStack::pop() return std::move(result); } +std::shared_ptr TransformStack::push(std::unique_ptr&& transform) +{ + return std::make_shared(std::forward(transform), + shared_from_this()); +} + void TransformStack::dump() const { std::cerr << *this << std::endl; } Shift::Shift(int32_t dim, int64_t offset) : dim_(dim), offset_(offset) {} diff --git a/src/core/data/transform.h b/src/core/data/transform.h index 26126df09b..d420ddff45 100644 --- a/src/core/data/transform.h +++ b/src/core/data/transform.h @@ -40,10 +40,9 @@ struct StoreTransform : public Transform { virtual int32_t target_ndim(int32_t source_ndim) const = 0; }; -struct TransformStack : public Transform { +struct TransformStack : public Transform, std::enable_shared_from_this { public: TransformStack() {} - // TODO: this constructor will be gone once we move the push method to this class TransformStack(std::unique_ptr&& transform, const std::shared_ptr& parent); TransformStack(std::unique_ptr&& transform, @@ -59,6 +58,7 @@ struct TransformStack : public Transform { public: std::unique_ptr pop(); + std::shared_ptr push(std::unique_ptr&& transform); bool identity() const { return nullptr == transform_; } public: From a5115c00b8e616726d9dd36e43d2aad7bcde1e80 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 5 Aug 2022 14:03:04 -0700 Subject: [PATCH 0065/1425] Rename invert_partition to invert, as the signature describes its semantics --- src/core/data/logical_store.cc | 2 +- src/core/data/transform.cc | 28 ++++++++-------------------- src/core/data/transform.h | 24 ++++++++++++------------ 3 files changed, 21 insertions(+), 33 deletions(-) diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index 6e87da8714..4954f82a92 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -243,7 +243,7 @@ std::unique_ptr LogicalStore::invert_partition(const Partition* parti } } } else { - auto inverted = transform_->invert_partition(partition); + auto inverted = transform_->invert(partition); return parent_->invert_partition(inverted.get()); } assert(false); diff --git a/src/core/data/transform.cc b/src/core/data/transform.cc index 4d288ebe10..6de781b815 100644 --- a/src/core/data/transform.cc +++ b/src/core/data/transform.cc @@ -46,10 +46,10 @@ TransformStack::TransformStack(std::unique_ptr&& transform, { } -std::unique_ptr TransformStack::invert_partition(const Partition* partition) const +std::unique_ptr TransformStack::invert(const Partition* partition) const { - auto result = transform_->invert_partition(partition); - return parent_->identity() ? std::move(result) : parent_->invert_partition(result.get()); + auto result = transform_->invert(partition); + return parent_->identity() ? std::move(result) : parent_->invert(result.get()); } proj::SymbolicPoint TransformStack::invert(const proj::SymbolicPoint& point) const @@ -157,10 +157,7 @@ DomainAffineTransform Shift::inverse_transform(int32_t in_dim) const return result; } -std::unique_ptr Shift::invert_partition(const Partition* partition) const -{ - return nullptr; -} +std::unique_ptr Shift::invert(const Partition* partition) const { return nullptr; } // the shift transform makes no change on the store's dimensions proj::SymbolicPoint Shift::invert(const proj::SymbolicPoint& point) const { return point; } @@ -227,7 +224,7 @@ DomainAffineTransform Promote::inverse_transform(int32_t in_dim) const return result; } -std::unique_ptr Promote::invert_partition(const Partition* partition) const +std::unique_ptr Promote::invert(const Partition* partition) const { switch (partition->kind()) { case Partition::Kind::NO_PARTITION: { @@ -311,10 +308,7 @@ DomainAffineTransform Project::inverse_transform(int32_t in_dim) const return result; } -std::unique_ptr Project::invert_partition(const Partition* partition) const -{ - return nullptr; -} +std::unique_ptr Project::invert(const Partition* partition) const { return nullptr; } proj::SymbolicPoint Project::invert(const proj::SymbolicPoint& point) const { @@ -378,10 +372,7 @@ DomainAffineTransform Transpose::inverse_transform(int32_t in_dim) const return result; } -std::unique_ptr Transpose::invert_partition(const Partition* partition) const -{ - return nullptr; -} +std::unique_ptr Transpose::invert(const Partition* partition) const { return nullptr; } proj::SymbolicPoint Transpose::invert(const proj::SymbolicPoint& point) const { @@ -490,10 +481,7 @@ DomainAffineTransform Delinearize::inverse_transform(int32_t in_dim) const return result; } -std::unique_ptr Delinearize::invert_partition(const Partition* partition) const -{ - return nullptr; -} +std::unique_ptr Delinearize::invert(const Partition* partition) const { return nullptr; } proj::SymbolicPoint Delinearize::invert(const proj::SymbolicPoint& point) const { diff --git a/src/core/data/transform.h b/src/core/data/transform.h index d420ddff45..f88e86cbc6 100644 --- a/src/core/data/transform.h +++ b/src/core/data/transform.h @@ -27,12 +27,12 @@ class BufferBuilder; class Partition; struct Transform { - virtual Legion::Domain transform(const Legion::Domain& input) const = 0; - virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const = 0; - virtual std::unique_ptr invert_partition(const Partition* partition) const = 0; - virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const = 0; - virtual void pack(BufferBuilder& buffer) const = 0; - virtual void print(std::ostream& out) const = 0; + virtual Legion::Domain transform(const Legion::Domain& input) const = 0; + virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const = 0; + virtual std::unique_ptr invert(const Partition* partition) const = 0; + virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const = 0; + virtual void pack(BufferBuilder& buffer) const = 0; + virtual void print(std::ostream& out) const = 0; }; struct StoreTransform : public Transform { @@ -51,7 +51,7 @@ struct TransformStack : public Transform, std::enable_shared_from_this invert_partition(const Partition* partition) const override; + virtual std::unique_ptr invert(const Partition* partition) const override; virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; @@ -76,7 +76,7 @@ class Shift : public StoreTransform { public: virtual Legion::Domain transform(const Legion::Domain& input) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; - virtual std::unique_ptr invert_partition(const Partition* partition) const override; + virtual std::unique_ptr invert(const Partition* partition) const override; virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; @@ -96,7 +96,7 @@ class Promote : public StoreTransform { public: virtual Legion::Domain transform(const Legion::Domain& input) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; - virtual std::unique_ptr invert_partition(const Partition* partition) const override; + virtual std::unique_ptr invert(const Partition* partition) const override; virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; @@ -117,7 +117,7 @@ class Project : public StoreTransform { public: virtual Legion::Domain transform(const Legion::Domain& domain) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; - virtual std::unique_ptr invert_partition(const Partition* partition) const override; + virtual std::unique_ptr invert(const Partition* partition) const override; virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; @@ -137,7 +137,7 @@ class Transpose : public StoreTransform { public: virtual Legion::Domain transform(const Legion::Domain& domain) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; - virtual std::unique_ptr invert_partition(const Partition* partition) const override; + virtual std::unique_ptr invert(const Partition* partition) const override; virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; @@ -157,7 +157,7 @@ class Delinearize : public StoreTransform { public: virtual Legion::Domain transform(const Legion::Domain& domain) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; - virtual std::unique_ptr invert_partition(const Partition* partition) const override; + virtual std::unique_ptr invert(const Partition* partition) const override; virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; From 5ea3325de7bd6dfb3a900690eb34b8b9951daeae Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 5 Aug 2022 15:53:38 -0700 Subject: [PATCH 0066/1425] Make LogicalRegionField an implementation detail --- src/core/data/logical_region_field.h | 48 ++++++++++++++++++++++++++++ src/core/data/logical_store.cc | 1 + src/core/data/logical_store.h | 24 +------------- src/core/runtime/launcher.cc | 1 + src/core/runtime/launcher_arg.cc | 1 + src/core/runtime/runtime.cc | 1 + 6 files changed, 53 insertions(+), 23 deletions(-) create mode 100644 src/core/data/logical_region_field.h diff --git a/src/core/data/logical_region_field.h b/src/core/data/logical_region_field.h new file mode 100644 index 0000000000..1520cb91c4 --- /dev/null +++ b/src/core/data/logical_region_field.h @@ -0,0 +1,48 @@ +/* Copyright 2022 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include "legion.h" + +#include "core/runtime/runtime.h" + +namespace legate { + +class LogicalRegionField { + public: + LogicalRegionField() {} + LogicalRegionField(Runtime* runtime, const Legion::LogicalRegion& lr, Legion::FieldID fid); + + public: + LogicalRegionField(const LogicalRegionField& other) = default; + LogicalRegionField& operator=(const LogicalRegionField& other) = default; + + public: + int32_t dim() const; + const Legion::LogicalRegion& region() const { return lr_; } + Legion::FieldID field_id() const { return fid_; } + + public: + Legion::Domain domain() const; + + private: + Runtime* runtime_{nullptr}; + Legion::LogicalRegion lr_{}; + Legion::FieldID fid_{-1U}; +}; + +} // namespace legate diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index 4954f82a92..580b2d2d1a 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -16,6 +16,7 @@ #include +#include "core/data/logical_region_field.h" #include "core/data/logical_store.h" #include "core/data/store.h" #include "core/partitioning/partition.h" diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index 43f3517469..236637a234 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -33,34 +33,12 @@ class LogicalStore; class BufferBuilder; class LibraryContext; +class LogicalRegionField; class Partition; class Projection; class Runtime; class Store; -class LogicalRegionField { - public: - LogicalRegionField() {} - LogicalRegionField(Runtime* runtime, const Legion::LogicalRegion& lr, Legion::FieldID fid); - - public: - LogicalRegionField(const LogicalRegionField& other) = default; - LogicalRegionField& operator=(const LogicalRegionField& other) = default; - - public: - int32_t dim() const; - const Legion::LogicalRegion& region() const { return lr_; } - Legion::FieldID field_id() const { return fid_; } - - public: - Legion::Domain domain() const; - - private: - Runtime* runtime_{nullptr}; - Legion::LogicalRegion lr_{}; - Legion::FieldID fid_{-1U}; -}; - class LogicalStore { public: LogicalStore(); diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/launcher.cc index a0e50a9c45..351183804c 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -15,6 +15,7 @@ */ #include "core/runtime/launcher.h" +#include "core/data/logical_region_field.h" #include "core/data/logical_store.h" #include "core/data/scalar.h" #include "core/runtime/context.h" diff --git a/src/core/runtime/launcher_arg.cc b/src/core/runtime/launcher_arg.cc index 006bc5a6a9..158719db70 100644 --- a/src/core/runtime/launcher_arg.cc +++ b/src/core/runtime/launcher_arg.cc @@ -15,6 +15,7 @@ */ #include "core/runtime/launcher_arg.h" +#include "core/data/logical_region_field.h" #include "core/runtime/launcher.h" #include "core/runtime/req_analyzer.h" diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 3f51b98818..e2cc0f04ce 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -18,6 +18,7 @@ #include #include "core/comm/comm.h" +#include "core/data/logical_region_field.h" #include "core/data/logical_store.h" #include "core/mapping/core_mapper.h" #include "core/partitioning/partition.h" From 8c4b8adae54126cf04cfd68e438e24f4ddaa0530 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 5 Aug 2022 16:21:49 -0700 Subject: [PATCH 0067/1425] Turn all logical store constructors into internal methods --- src/core/data/logical_store.cc | 107 +++------------------------ src/core/data/logical_store.h | 14 +--- src/core/data/logical_store_detail.h | 103 ++++++++++++++++++++++++++ src/core/runtime/operation.cc | 4 +- src/core/runtime/runtime.cc | 5 +- 5 files changed, 121 insertions(+), 112 deletions(-) create mode 100644 src/core/data/logical_store_detail.h diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index 580b2d2d1a..ba52da2918 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -18,6 +18,7 @@ #include "core/data/logical_region_field.h" #include "core/data/logical_store.h" +#include "core/data/logical_store_detail.h" #include "core/data/store.h" #include "core/partitioning/partition.h" #include "core/runtime/req_analyzer.h" @@ -47,80 +48,14 @@ Domain LogicalRegionField::domain() const namespace detail { -class LogicalStore { - public: - LogicalStore(); - LogicalStore(Runtime* runtime, - LegateTypeCode code, - tuple extents, - std::shared_ptr parent, - std::shared_ptr transform); - LogicalStore(Runtime* runtime, LegateTypeCode code, const void* data); - - public: - ~LogicalStore(); - - private: - LogicalStore(std::shared_ptr impl); - - public: - LogicalStore(const LogicalStore& other) = default; - LogicalStore& operator=(const LogicalStore& other) = default; - - public: - LogicalStore(LogicalStore&& other) = default; - LogicalStore& operator=(LogicalStore&& other) = default; - - public: - bool scalar() const; - int32_t dim() const; - LegateTypeCode code() const; - Legion::Domain domain() const; - const std::vector& extents() const; - size_t volume() const; - - public: - bool has_storage() const; - std::shared_ptr get_storage(); - Legion::Future get_future(); - - private: - void create_storage(); - - public: - std::shared_ptr promote(int32_t extra_dim, - size_t dim_size, - std::shared_ptr parent) const; - - public: - std::shared_ptr get_physical_store(LibraryContext* context); - - public: - std::unique_ptr find_or_create_partition(const Partition* partition); - std::unique_ptr find_or_create_key_partition(); - - private: - std::unique_ptr invert_partition(const Partition* partition) const; - proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const; - Legion::ProjectionID compute_projection() const; - - public: - void pack(BufferBuilder& buffer) const; - - private: - void pack_transform(BufferBuilder& buffer) const; - - private: - bool scalar_{false}; - Runtime* runtime_{nullptr}; - LegateTypeCode code_{MAX_TYPE_NUMBER}; - tuple extents_; - std::shared_ptr region_field_{nullptr}; - Legion::Future future_{}; - std::shared_ptr parent_{nullptr}; - std::shared_ptr transform_{nullptr}; - std::shared_ptr mapped_{nullptr}; -}; +LogicalStore::LogicalStore(Runtime* runtime, LegateTypeCode code, tuple extents) + : runtime_(runtime), + code_(code), + extents_(std::move(extents)), + parent_(nullptr), + transform_(std::make_shared()) +{ +} LogicalStore::LogicalStore(Runtime* runtime, LegateTypeCode code, @@ -324,31 +259,11 @@ void LogicalStore::pack_transform(BufferBuilder& buffer) const } // namespace detail -LogicalStore::LogicalStore() {} - -LogicalStore::LogicalStore(Runtime* runtime, LegateTypeCode code, tuple extents) - : impl_(std::make_shared( - runtime, code, std::move(extents), nullptr, std::make_shared())) -{ -} - -LogicalStore::LogicalStore(Runtime* runtime, - LegateTypeCode code, - tuple extents, - LogicalStore parent, - std::shared_ptr transform) - : impl_(std::make_shared( - runtime, code, std::move(extents), parent.impl_, std::move(transform))) +LogicalStore::LogicalStore(std::shared_ptr&& impl) + : impl_(std::forward(impl)) { } -LogicalStore::LogicalStore(Runtime* runtime, LegateTypeCode code, const void* data) - : impl_(std::make_shared(runtime, code, data)) -{ -} - -LogicalStore::LogicalStore(std::shared_ptr impl) : impl_(std::move(impl)) {} - bool LogicalStore::scalar() const { return impl_->scalar(); } int32_t LogicalStore::dim() const { return impl_->dim(); } diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index 236637a234..d95dc8aae1 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -40,19 +40,9 @@ class Runtime; class Store; class LogicalStore { - public: - LogicalStore(); - LogicalStore(Runtime* runtime, LegateTypeCode code, tuple extents); - LogicalStore(Runtime* runtime, - LegateTypeCode code, - tuple extents, - LogicalStore parent, - std::shared_ptr transform); - // Creates a read-only store from a scalar - LogicalStore(Runtime* runtime, LegateTypeCode code, const void* data); - private: - LogicalStore(std::shared_ptr impl); + friend class Runtime; + LogicalStore(std::shared_ptr&& impl); public: LogicalStore(const LogicalStore& other) = default; diff --git a/src/core/data/logical_store_detail.h b/src/core/data/logical_store_detail.h new file mode 100644 index 0000000000..eb6ecad70a --- /dev/null +++ b/src/core/data/logical_store_detail.h @@ -0,0 +1,103 @@ +/* Copyright 2022 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include "core/data/logical_region_field.h" +#include "core/partitioning/partition.h" +#include "core/runtime/runtime.h" + +namespace legate { +namespace detail { + +class LogicalStore { + public: + LogicalStore(); + LogicalStore(Runtime* runtime, LegateTypeCode code, tuple extents); + LogicalStore(Runtime* runtime, + LegateTypeCode code, + tuple extents, + std::shared_ptr parent, + std::shared_ptr transform); + LogicalStore(Runtime* runtime, LegateTypeCode code, const void* data); + + public: + ~LogicalStore(); + + private: + LogicalStore(std::shared_ptr impl); + + public: + LogicalStore(const LogicalStore& other) = default; + LogicalStore& operator=(const LogicalStore& other) = default; + + public: + LogicalStore(LogicalStore&& other) = default; + LogicalStore& operator=(LogicalStore&& other) = default; + + public: + bool scalar() const; + int32_t dim() const; + LegateTypeCode code() const; + Legion::Domain domain() const; + const std::vector& extents() const; + size_t volume() const; + + public: + bool has_storage() const; + std::shared_ptr get_storage(); + Legion::Future get_future(); + + private: + void create_storage(); + + public: + std::shared_ptr promote(int32_t extra_dim, + size_t dim_size, + std::shared_ptr parent) const; + + public: + std::shared_ptr get_physical_store(LibraryContext* context); + + public: + std::unique_ptr find_or_create_partition(const Partition* partition); + std::unique_ptr find_or_create_key_partition(); + + private: + std::unique_ptr invert_partition(const Partition* partition) const; + proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const; + Legion::ProjectionID compute_projection() const; + + public: + void pack(BufferBuilder& buffer) const; + + private: + void pack_transform(BufferBuilder& buffer) const; + + private: + bool scalar_{false}; + Runtime* runtime_{nullptr}; + LegateTypeCode code_{MAX_TYPE_NUMBER}; + tuple extents_; + std::shared_ptr region_field_{nullptr}; + Legion::Future future_{}; + std::shared_ptr parent_{nullptr}; + std::shared_ptr transform_{nullptr}; + std::shared_ptr mapped_{nullptr}; +}; + +} // namespace detail +} // namespace legate diff --git a/src/core/runtime/operation.cc b/src/core/runtime/operation.cc index 80d4316064..b56fe72c37 100644 --- a/src/core/runtime/operation.cc +++ b/src/core/runtime/operation.cc @@ -67,8 +67,8 @@ void Operation::add_reduction(LogicalStore store, std::shared_ptr Operation::declare_partition(LogicalStore store) { - auto variable = std::make_shared(this, next_part_id_++); - store_mappings_[variable] = std::move(store); + auto variable = std::make_shared(this, next_part_id_++); + store_mappings_.emplace(std::make_pair(variable, std::move(store))); return std::move(variable); } diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index e2cc0f04ce..90065ac994 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -20,6 +20,7 @@ #include "core/comm/comm.h" #include "core/data/logical_region_field.h" #include "core/data/logical_store.h" +#include "core/data/logical_store_detail.h" #include "core/mapping/core_mapper.h" #include "core/partitioning/partition.h" #include "core/partitioning/partitioner.h" @@ -614,12 +615,12 @@ void Runtime::schedule(std::vector> operations) LogicalStore Runtime::create_store(std::vector extents, LegateTypeCode code) { - return LogicalStore(this, code, extents); + return LogicalStore(std::make_shared(this, code, extents)); } LogicalStore Runtime::create_store(const Scalar& scalar) { - return LogicalStore(this, scalar.code(), scalar.ptr()); + return LogicalStore(std::make_shared(this, scalar.code(), scalar.ptr())); } std::shared_ptr Runtime::create_region_field(const tuple& extents, From b2b0a7819f0ae5d149c16ff487f312d6c1a8480e Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 5 Aug 2022 21:10:24 -0700 Subject: [PATCH 0068/1425] Hide storage-related methods from the client API and move them to the detail namespace --- src/core/data/logical_store.cc | 16 ---------------- src/core/data/logical_store.h | 23 +++++++---------------- src/core/partitioning/partition.cc | 16 ++++++++-------- src/core/partitioning/partition.h | 27 ++++++++++++++++----------- src/core/partitioning/partitioner.cc | 3 ++- src/core/runtime/launcher.cc | 26 +++++++++++++------------- src/core/runtime/launcher.h | 14 ++++++++++---- src/core/runtime/launcher_arg.cc | 19 ++++++++++--------- src/core/runtime/launcher_arg.h | 13 ++++++++----- src/core/runtime/operation.cc | 20 +++++++++++++++----- src/core/runtime/operation.h | 13 ++++++++++--- 11 files changed, 99 insertions(+), 91 deletions(-) diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index ba52da2918..b9de5fd455 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -276,10 +276,6 @@ const std::vector& LogicalStore::extents() const { return impl_->extents size_t LogicalStore::volume() const { return impl_->volume(); } -std::shared_ptr LogicalStore::get_storage() { return impl_->get_storage(); } - -Legion::Future LogicalStore::get_future() { return impl_->get_future(); } - LogicalStore LogicalStore::promote(int32_t extra_dim, size_t dim_size) const { return LogicalStore(impl_->promote(extra_dim, dim_size, impl_)); @@ -290,16 +286,4 @@ std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) return impl_->get_physical_store(context); } -std::unique_ptr LogicalStore::find_or_create_partition(const Partition* partition) -{ - return impl_->find_or_create_partition(partition); -} - -std::unique_ptr LogicalStore::find_or_create_key_partition() -{ - return impl_->find_or_create_key_partition(); -} - -void LogicalStore::pack(BufferBuilder& buffer) const { impl_->pack(buffer); } - } // namespace legate diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index d95dc8aae1..6dc043e965 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -25,12 +25,6 @@ namespace legate { -namespace detail { - -class LogicalStore; - -} // namespace detail - class BufferBuilder; class LibraryContext; class LogicalRegionField; @@ -39,6 +33,12 @@ class Projection; class Runtime; class Store; +namespace detail { + +class LogicalStore; + +} // namespace detail + class LogicalStore { private: friend class Runtime; @@ -60,11 +60,6 @@ class LogicalStore { const std::vector& extents() const; size_t volume() const; - public: - bool has_storage() const; - std::shared_ptr get_storage(); - Legion::Future get_future(); - public: LogicalStore promote(int32_t extra_dim, size_t dim_size) const; @@ -72,11 +67,7 @@ class LogicalStore { std::shared_ptr get_physical_store(LibraryContext* context); public: - std::unique_ptr find_or_create_partition(const Partition* partition); - std::unique_ptr find_or_create_key_partition(); - - public: - void pack(BufferBuilder& buffer) const; + std::shared_ptr impl() const { return impl_; } private: std::shared_ptr impl_{nullptr}; diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index a91000b531..3fda72363a 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -16,7 +16,7 @@ #include -#include "core/data/logical_store.h" +#include "core/data/logical_store_detail.h" #include "core/partitioning/partition.h" #include "core/runtime/launcher.h" #include "core/runtime/req_analyzer.h" @@ -28,9 +28,9 @@ Partition::Partition(Runtime* runtime) : runtime_(runtime) {} NoPartition::NoPartition(Runtime* runtime) : Partition(runtime) {} -bool NoPartition::is_complete_for(const LogicalStore* store) const { return false; } +bool NoPartition::is_complete_for(const detail::LogicalStore* store) const { return false; } -bool NoPartition::is_disjoint_for(const LogicalStore* store) const { return false; } +bool NoPartition::is_disjoint_for(const detail::LogicalStore* store) const { return false; } Legion::LogicalPartition NoPartition::construct(Legion::LogicalRegion region, bool disjoint, @@ -39,7 +39,7 @@ Legion::LogicalPartition NoPartition::construct(Legion::LogicalRegion region, return Legion::LogicalPartition::NO_PART; } -std::unique_ptr NoPartition::get_projection(LogicalStore store) const +std::unique_ptr NoPartition::get_projection(detail::LogicalStore* store) const { return std::make_unique(); } @@ -87,9 +87,9 @@ bool Tiling::operator<(const Tiling& other) const return false; } -bool Tiling::is_complete_for(const LogicalStore* store) const { return false; } +bool Tiling::is_complete_for(const detail::LogicalStore* store) const { return false; } -bool Tiling::is_disjoint_for(const LogicalStore* store) const { return true; } +bool Tiling::is_disjoint_for(const detail::LogicalStore* store) const { return true; } Legion::LogicalPartition Tiling::construct(Legion::LogicalRegion region, bool disjoint, @@ -133,9 +133,9 @@ Legion::LogicalPartition Tiling::construct(Legion::LogicalRegion region, return runtime_->create_logical_partition(region, index_partition); } -std::unique_ptr Tiling::get_projection(LogicalStore store) const +std::unique_ptr Tiling::get_projection(detail::LogicalStore* store) const { - return store.find_or_create_partition(this); + return store->find_or_create_partition(this); } bool Tiling::has_launch_domain() const { return true; } diff --git a/src/core/partitioning/partition.h b/src/core/partitioning/partition.h index 8eb5244716..2b35f4d588 100644 --- a/src/core/partitioning/partition.h +++ b/src/core/partitioning/partition.h @@ -23,10 +23,15 @@ namespace legate { -class LogicalStore; class Projection; class Runtime; +namespace detail { + +class LogicalStore; + +} // namespace detail + using Shape = tuple; struct Partition { @@ -44,14 +49,14 @@ struct Partition { virtual Kind kind() const = 0; public: - virtual bool is_complete_for(const LogicalStore* store) const = 0; - virtual bool is_disjoint_for(const LogicalStore* store) const = 0; + virtual bool is_complete_for(const detail::LogicalStore* store) const = 0; + virtual bool is_disjoint_for(const detail::LogicalStore* store) const = 0; public: virtual Legion::LogicalPartition construct(Legion::LogicalRegion region, bool disjoint = false, - bool complete = false) const = 0; - virtual std::unique_ptr get_projection(LogicalStore store) const = 0; + bool complete = false) const = 0; + virtual std::unique_ptr get_projection(detail::LogicalStore* store) const = 0; public: virtual bool has_launch_domain() const = 0; @@ -73,14 +78,14 @@ class NoPartition : public Partition { virtual Kind kind() const override { return Kind::NO_PARTITION; } public: - virtual bool is_complete_for(const LogicalStore* store) const override; - virtual bool is_disjoint_for(const LogicalStore* store) const override; + virtual bool is_complete_for(const detail::LogicalStore* store) const override; + virtual bool is_disjoint_for(const detail::LogicalStore* store) const override; public: virtual Legion::LogicalPartition construct(Legion::LogicalRegion region, bool disjoint, bool complete) const override; - virtual std::unique_ptr get_projection(LogicalStore store) const override; + virtual std::unique_ptr get_projection(detail::LogicalStore* store) const override; public: virtual bool has_launch_domain() const override; @@ -105,14 +110,14 @@ class Tiling : public Partition { virtual Kind kind() const override { return Kind::TILING; } public: - virtual bool is_complete_for(const LogicalStore* store) const override; - virtual bool is_disjoint_for(const LogicalStore* store) const override; + virtual bool is_complete_for(const detail::LogicalStore* store) const override; + virtual bool is_disjoint_for(const detail::LogicalStore* store) const override; public: virtual Legion::LogicalPartition construct(Legion::LogicalRegion region, bool disjoint, bool complete) const override; - virtual std::unique_ptr get_projection(LogicalStore store) const override; + virtual std::unique_ptr get_projection(detail::LogicalStore* store) const override; public: virtual bool has_launch_domain() const override; diff --git a/src/core/partitioning/partitioner.cc b/src/core/partitioning/partitioner.cc index 5ce0bf2da3..89c9e7e3a5 100644 --- a/src/core/partitioning/partitioner.cc +++ b/src/core/partitioning/partitioner.cc @@ -16,6 +16,7 @@ #include "core/partitioning/partitioner.h" #include "core/data/logical_store.h" +#include "core/data/logical_store_detail.h" #include "core/data/scalar.h" #include "core/partitioning/constraint.h" #include "core/partitioning/constraint_graph.h" @@ -104,7 +105,7 @@ std::unique_ptr Partitioner::solve() for (auto& variable : variables) { auto* op = variable->operation(); auto store = op->find_store(variable); - auto partition = store.find_or_create_key_partition(); + auto partition = store->find_or_create_key_partition(); if (!strategy->has_launch_domain(op)) { if (partition->has_launch_domain()) strategy->set_launch_domain(op, partition->launch_domain()); diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/launcher.cc index 351183804c..4bb791d53c 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -17,6 +17,7 @@ #include "core/runtime/launcher.h" #include "core/data/logical_region_field.h" #include "core/data/logical_store.h" +#include "core/data/logical_store_detail.h" #include "core/data/scalar.h" #include "core/runtime/context.h" #include "core/runtime/launcher_arg.h" @@ -58,30 +59,30 @@ void TaskLauncher::add_scalar(const Scalar& scalar) scalars_.push_back(new UntypedScalarArg(scalar)); } -void TaskLauncher::add_input(LogicalStore store, +void TaskLauncher::add_input(detail::LogicalStore* store, std::unique_ptr proj, Legion::MappingTagID tag, Legion::RegionFlags flags) { - add_store(inputs_, std::move(store), std::move(proj), READ_ONLY, tag, flags); + add_store(inputs_, store, std::move(proj), READ_ONLY, tag, flags); } -void TaskLauncher::add_output(LogicalStore store, +void TaskLauncher::add_output(detail::LogicalStore* store, std::unique_ptr proj, Legion::MappingTagID tag, Legion::RegionFlags flags) { - add_store(outputs_, std::move(store), std::move(proj), WRITE_ONLY, tag, flags); + add_store(outputs_, store, std::move(proj), WRITE_ONLY, tag, flags); } -void TaskLauncher::add_reduction(LogicalStore store, +void TaskLauncher::add_reduction(detail::LogicalStore* store, std::unique_ptr proj, Legion::MappingTagID tag, Legion::RegionFlags flags, bool read_write /*= false*/) { assert(!read_write); - add_store(reductions_, std::move(store), std::move(proj), REDUCE, tag, flags); + add_store(reductions_, store, std::move(proj), REDUCE, tag, flags); } void TaskLauncher::execute(const Legion::Domain& launch_domain) @@ -99,7 +100,7 @@ void TaskLauncher::execute_single() } void TaskLauncher::add_store(std::vector& args, - LogicalStore store, + detail::LogicalStore* store, std::unique_ptr proj, Legion::PrivilegeMode privilege, Legion::MappingTagID tag, @@ -107,21 +108,20 @@ void TaskLauncher::add_store(std::vector& args, { auto redop = proj->redop; - if (store.scalar()) { + if (store->scalar()) { auto has_storage = privilege != WRITE_ONLY; auto read_only = privilege == READ_ONLY; - if (has_storage) futures_.push_back(store.get_future()); - args.push_back(new FutureStoreArg(std::move(store), read_only, has_storage, redop)); + if (has_storage) futures_.push_back(store->get_future()); + args.push_back(new FutureStoreArg(store, read_only, has_storage, redop)); } else { - auto storage = store.get_storage(); + auto storage = store->get_storage(); auto region = storage->region(); auto field_id = storage->field_id(); auto proj_info = new ProjectionInfo(proj.get(), tag, flags); req_analyzer_->insert(region, field_id, privilege, proj_info); - args.push_back( - new RegionFieldArg(req_analyzer_, std::move(store), field_id, privilege, proj_info)); + args.push_back(new RegionFieldArg(req_analyzer_, store, field_id, privilege, proj_info)); } } diff --git a/src/core/runtime/launcher.h b/src/core/runtime/launcher.h index b6d30073cb..8036ad92df 100644 --- a/src/core/runtime/launcher.h +++ b/src/core/runtime/launcher.h @@ -31,6 +31,12 @@ class RequirementAnalyzer; class Runtime; class Scalar; +namespace detail { + +class LogicalStore; + +} // namespace detail + class TaskLauncher { public: TaskLauncher(Runtime* runtime, @@ -46,15 +52,15 @@ class TaskLauncher { public: void add_scalar(const Scalar& scalar); - void add_input(LogicalStore store, + void add_input(detail::LogicalStore* store, std::unique_ptr proj, Legion::MappingTagID tag = 0, Legion::RegionFlags flags = LEGION_NO_FLAG); - void add_output(LogicalStore store, + void add_output(detail::LogicalStore* store, std::unique_ptr proj, Legion::MappingTagID tag = 0, Legion::RegionFlags flags = LEGION_NO_FLAG); - void add_reduction(LogicalStore store, + void add_reduction(detail::LogicalStore* store, std::unique_ptr proj, Legion::MappingTagID tag = 0, Legion::RegionFlags flags = LEGION_NO_FLAG, @@ -62,7 +68,7 @@ class TaskLauncher { private: void add_store(std::vector& args, - LogicalStore store, + detail::LogicalStore* store, std::unique_ptr proj, Legion::PrivilegeMode privilege, Legion::MappingTagID tag, diff --git a/src/core/runtime/launcher_arg.cc b/src/core/runtime/launcher_arg.cc index 158719db70..57281cd421 100644 --- a/src/core/runtime/launcher_arg.cc +++ b/src/core/runtime/launcher_arg.cc @@ -29,13 +29,13 @@ void UntypedScalarArg::pack(BufferBuilder& buffer) const } RegionFieldArg::RegionFieldArg(RequirementAnalyzer* analyzer, - LogicalStore store, + detail::LogicalStore* store, Legion::FieldID field_id, Legion::PrivilegeMode privilege, const ProjectionInfo* proj_info) : analyzer_(analyzer), - store_(std::move(store)), - region_(store_.get_storage()->region()), + store_(store), + region_(store_->get_storage()->region()), field_id_(field_id), privilege_(privilege), proj_info_(proj_info) @@ -44,18 +44,19 @@ RegionFieldArg::RegionFieldArg(RequirementAnalyzer* analyzer, void RegionFieldArg::pack(BufferBuilder& buffer) const { - store_.pack(buffer); + store_->pack(buffer); + buffer.pack(proj_info_->redop); buffer.pack(region_.get_dim()); buffer.pack(analyzer_->get_requirement_index(region_, privilege_, proj_info_)); buffer.pack(field_id_); } -FutureStoreArg::FutureStoreArg(LogicalStore store, +FutureStoreArg::FutureStoreArg(detail::LogicalStore* store, bool read_only, bool has_storage, Legion::ReductionOpID redop) - : store_(std::move(store)), read_only_(read_only), has_storage_(has_storage), redop_(redop) + : store_(store), read_only_(read_only), has_storage_(has_storage), redop_(redop) { } @@ -69,13 +70,13 @@ struct datalen_fn { void FutureStoreArg::pack(BufferBuilder& buffer) const { - store_.pack(buffer); + store_->pack(buffer); buffer.pack(redop_); buffer.pack(read_only_); buffer.pack(has_storage_); - buffer.pack(type_dispatch(store_.code(), datalen_fn{})); - buffer.pack(store_.extents()); + buffer.pack(type_dispatch(store_->code(), datalen_fn{})); + buffer.pack(store_->extents()); } } // namespace legate diff --git a/src/core/runtime/launcher_arg.h b/src/core/runtime/launcher_arg.h index 4efc114af3..4c6ec58ec1 100644 --- a/src/core/runtime/launcher_arg.h +++ b/src/core/runtime/launcher_arg.h @@ -16,7 +16,7 @@ #pragma once -#include "core/data/logical_store.h" +#include "core/data/logical_store_detail.h" #include "core/data/scalar.h" #include "core/utilities/buffer_builder.h" @@ -62,7 +62,7 @@ struct UntypedScalarArg : public ArgWrapper { struct RegionFieldArg : public ArgWrapper { public: RegionFieldArg(RequirementAnalyzer* analyzer, - LogicalStore store, + detail::LogicalStore* store, Legion::FieldID field_id, Legion::PrivilegeMode privilege, const ProjectionInfo* proj_info); @@ -75,7 +75,7 @@ struct RegionFieldArg : public ArgWrapper { private: RequirementAnalyzer* analyzer_; - LogicalStore store_; + detail::LogicalStore* store_; Legion::LogicalRegion region_; Legion::FieldID field_id_; Legion::PrivilegeMode privilege_; @@ -84,7 +84,10 @@ struct RegionFieldArg : public ArgWrapper { struct FutureStoreArg : public ArgWrapper { public: - FutureStoreArg(LogicalStore store, bool read_only, bool has_storage, Legion::ReductionOpID redop); + FutureStoreArg(detail::LogicalStore* store, + bool read_only, + bool has_storage, + Legion::ReductionOpID redop); public: virtual ~FutureStoreArg() {} @@ -93,7 +96,7 @@ struct FutureStoreArg : public ArgWrapper { virtual void pack(BufferBuilder& buffer) const override; private: - LogicalStore store_; + detail::LogicalStore* store_; bool read_only_; bool has_storage_; Legion::ReductionOpID redop_; diff --git a/src/core/runtime/operation.cc b/src/core/runtime/operation.cc index b56fe72c37..c46789f451 100644 --- a/src/core/runtime/operation.cc +++ b/src/core/runtime/operation.cc @@ -46,33 +46,43 @@ Operation::~Operation() {} void Operation::add_input(LogicalStore store, std::shared_ptr partition) { + auto p_store = store.impl(); constraints_->add_variable(partition); - inputs_.push_back(Store(std::move(store), std::move(partition))); + inputs_.push_back(Store(p_store.get(), std::move(partition))); + all_stores_.insert(std::move(p_store)); } void Operation::add_output(LogicalStore store, std::shared_ptr partition) { + auto p_store = store.impl(); constraints_->add_variable(partition); - outputs_.push_back(Store(std::move(store), std::move(partition))); + outputs_.push_back(Store(p_store.get(), std::move(partition))); + all_stores_.insert(std::move(p_store)); } void Operation::add_reduction(LogicalStore store, Legion::ReductionOpID redop, std::shared_ptr partition) { + auto p_store = store.impl(); constraints_->add_variable(partition); - reductions_.push_back(Store(std::move(store), std::move(partition))); + reductions_.push_back(Store(p_store.get(), std::move(partition))); reduction_ops_.push_back(redop); + all_stores_.insert(std::move(p_store)); } std::shared_ptr Operation::declare_partition(LogicalStore store) { + // TODO: Variable doesn't need a store to be created, so there's some redundancy in this function. + // Will clean it up once the refactoring for logical store is done + auto p_store = store.impl(); auto variable = std::make_shared(this, next_part_id_++); - store_mappings_.emplace(std::make_pair(variable, std::move(store))); + store_mappings_.emplace(std::make_pair(variable, p_store.get())); + all_stores_.insert(std::move(p_store)); return std::move(variable); } -LogicalStore Operation::find_store(std::shared_ptr variable) const +detail::LogicalStore* Operation::find_store(std::shared_ptr variable) const { auto finder = store_mappings_.find(variable); assert(store_mappings_.end() != finder); diff --git a/src/core/runtime/operation.h b/src/core/runtime/operation.h index f8c7989733..361aac07c0 100644 --- a/src/core/runtime/operation.h +++ b/src/core/runtime/operation.h @@ -32,6 +32,12 @@ class Scalar; class Strategy; class Variable; +namespace detail { + +class LogicalStore; + +} // namespace detail + class Operation { public: Operation(Runtime* runtime, LibraryContext* library, uint64_t unique_id, int64_t mapper_id); @@ -48,7 +54,7 @@ class Operation { public: std::shared_ptr declare_partition(LogicalStore store); - LogicalStore find_store(std::shared_ptr variable) const; + detail::LogicalStore* find_store(std::shared_ptr variable) const; void add_constraint(std::shared_ptr constraint); std::shared_ptr constraints() const; @@ -65,9 +71,10 @@ class Operation { int64_t mapper_id_; protected: - using Store = std::pair>; + using Store = std::pair>; protected: + std::set> all_stores_; std::vector inputs_{}; std::vector outputs_{}; std::vector reductions_{}; @@ -77,7 +84,7 @@ class Operation { uint32_t next_part_id_{0}; private: - std::unordered_map, LogicalStore> store_mappings_; + std::unordered_map, detail::LogicalStore*> store_mappings_; std::shared_ptr constraints_; }; From 8b482129c0771b765fd62e90bfb4d0d9c3efa354 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 5 Aug 2022 23:47:12 -0700 Subject: [PATCH 0069/1425] Stop caching the runtime object and use the singleton instead --- src/core/data/logical_region_field.h | 3 +- src/core/data/logical_store.cc | 49 +++++++++++----------------- src/core/data/logical_store_detail.h | 8 ++--- src/core/data/transform.cc | 5 ++- src/core/partitioning/partition.cc | 33 ++++++++----------- src/core/partitioning/partition.h | 19 +++-------- src/core/partitioning/partitioner.cc | 4 +-- src/core/partitioning/partitioner.h | 4 +-- src/core/runtime/launcher.cc | 9 +++-- src/core/runtime/launcher.h | 8 +---- src/core/runtime/operation.cc | 18 +++------- src/core/runtime/operation.h | 10 ++---- src/core/runtime/runtime.cc | 12 +++---- 13 files changed, 65 insertions(+), 117 deletions(-) diff --git a/src/core/data/logical_region_field.h b/src/core/data/logical_region_field.h index 1520cb91c4..ba5c0a299a 100644 --- a/src/core/data/logical_region_field.h +++ b/src/core/data/logical_region_field.h @@ -25,7 +25,7 @@ namespace legate { class LogicalRegionField { public: LogicalRegionField() {} - LogicalRegionField(Runtime* runtime, const Legion::LogicalRegion& lr, Legion::FieldID fid); + LogicalRegionField(const Legion::LogicalRegion& lr, Legion::FieldID fid); public: LogicalRegionField(const LogicalRegionField& other) = default; @@ -40,7 +40,6 @@ class LogicalRegionField { Legion::Domain domain() const; private: - Runtime* runtime_{nullptr}; Legion::LogicalRegion lr_{}; Legion::FieldID fid_{-1U}; }; diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index b9de5fd455..8a1cf008ca 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -34,36 +34,30 @@ namespace legate { extern Logger log_legate; -LogicalRegionField::LogicalRegionField(Runtime* runtime, const LogicalRegion& lr, FieldID fid) - : runtime_(runtime), lr_(lr), fid_(fid) -{ -} +LogicalRegionField::LogicalRegionField(const LogicalRegion& lr, FieldID fid) : lr_(lr), fid_(fid) {} int32_t LogicalRegionField::dim() const { return lr_.get_dim(); } Domain LogicalRegionField::domain() const { - return runtime_->get_index_space_domain(lr_.get_index_space()); + return Runtime::get_runtime()->get_index_space_domain(lr_.get_index_space()); } namespace detail { -LogicalStore::LogicalStore(Runtime* runtime, LegateTypeCode code, tuple extents) - : runtime_(runtime), - code_(code), +LogicalStore::LogicalStore(LegateTypeCode code, tuple extents) + : code_(code), extents_(std::move(extents)), parent_(nullptr), transform_(std::make_shared()) { } -LogicalStore::LogicalStore(Runtime* runtime, - LegateTypeCode code, +LogicalStore::LogicalStore(LegateTypeCode code, tuple extents, std::shared_ptr parent, std::shared_ptr transform) - : runtime_(runtime), - code_(code), + : code_(code), extents_(std::move(extents)), parent_(std::move(parent)), transform_(std::move(transform)) @@ -78,15 +72,11 @@ struct datalen_fn { } }; -LogicalStore::LogicalStore(Runtime* runtime, LegateTypeCode code, const void* data) - : scalar_(true), - runtime_(runtime), - code_(code), - extents_({1}), - transform_(std::make_shared()) +LogicalStore::LogicalStore(LegateTypeCode code, const void* data) + : scalar_(true), code_(code), extents_({1}), transform_(std::make_shared()) { auto datalen = type_dispatch(code, datalen_fn{}); - future_ = runtime_->create_future(data, datalen); + future_ = Runtime::get_runtime()->create_future(data, datalen); } LogicalStore::~LogicalStore() @@ -133,7 +123,7 @@ Legion::Future LogicalStore::get_future() void LogicalStore::create_storage() { - region_field_ = runtime_->create_region_field(extents_, code_); + region_field_ = Runtime::get_runtime()->create_region_field(extents_, code_); } std::shared_ptr LogicalStore::promote(int32_t extra_dim, @@ -149,7 +139,7 @@ std::shared_ptr LogicalStore::promote(int32_t extra_dim, auto new_extents = extents_.insert(extra_dim, dim_size); auto transform = transform_->push(std::make_unique(extra_dim, dim_size)); return std::make_shared( - runtime_, code_, std::move(new_extents), std::move(parent), std::move(transform)); + code_, std::move(new_extents), std::move(parent), std::move(transform)); } std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) @@ -157,7 +147,7 @@ std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) // TODO: Need to support inline mapping for scalars assert(!scalar_); if (nullptr != mapped_) return mapped_; - auto rf = runtime_->map_region_field(context, region_field_); + auto rf = Runtime::get_runtime()->map_region_field(context, region_field_); mapped_ = std::make_shared(dim(), code_, -1, std::move(rf), transform_); return mapped_; } @@ -167,15 +157,14 @@ std::unique_ptr LogicalStore::invert_partition(const Partition* parti if (nullptr == parent_) { switch (partition->kind()) { case Partition::Kind::NO_PARTITION: { - return create_no_partition(runtime_); + return create_no_partition(); } case Partition::Kind::TILING: { auto tiling = static_cast(partition); Shape tile_shape = tiling->tile_shape(); Shape color_shape = tiling->color_shape(); Shape offsets = tiling->offsets(); - return create_tiling( - runtime_, std::move(tile_shape), std::move(color_shape), std::move(offsets)); + return create_tiling(std::move(tile_shape), std::move(color_shape), std::move(offsets)); } } } else { @@ -206,7 +195,7 @@ Legion::ProjectionID LogicalStore::compute_projection() const if (identity_mapping) return 0; else - return runtime_->get_projection(ndim, point); + return Runtime::get_runtime()->get_projection(ndim, point); } std::unique_ptr LogicalStore::find_or_create_partition(const Partition* partition) @@ -226,15 +215,15 @@ std::unique_ptr LogicalStore::find_or_create_partition(const Partiti std::unique_ptr LogicalStore::find_or_create_key_partition() { - if (scalar_) return create_no_partition(runtime_); + if (scalar_) return create_no_partition(); - auto part_mgr = runtime_->partition_manager(); + auto part_mgr = Runtime::get_runtime()->partition_manager(); auto launch_shape = part_mgr->compute_launch_shape(extents()); if (launch_shape.empty()) - return create_no_partition(runtime_); + return create_no_partition(); else { auto tile_shape = part_mgr->compute_tile_shape(extents_, launch_shape); - return create_tiling(runtime_, std::move(tile_shape), std::move(launch_shape)); + return create_tiling(std::move(tile_shape), std::move(launch_shape)); } } diff --git a/src/core/data/logical_store_detail.h b/src/core/data/logical_store_detail.h index eb6ecad70a..1d97ae0e3c 100644 --- a/src/core/data/logical_store_detail.h +++ b/src/core/data/logical_store_detail.h @@ -26,13 +26,12 @@ namespace detail { class LogicalStore { public: LogicalStore(); - LogicalStore(Runtime* runtime, LegateTypeCode code, tuple extents); - LogicalStore(Runtime* runtime, - LegateTypeCode code, + LogicalStore(LegateTypeCode code, tuple extents); + LogicalStore(LegateTypeCode code, tuple extents, std::shared_ptr parent, std::shared_ptr transform); - LogicalStore(Runtime* runtime, LegateTypeCode code, const void* data); + LogicalStore(LegateTypeCode code, const void* data); public: ~LogicalStore(); @@ -89,7 +88,6 @@ class LogicalStore { private: bool scalar_{false}; - Runtime* runtime_{nullptr}; LegateTypeCode code_{MAX_TYPE_NUMBER}; tuple extents_; std::shared_ptr region_field_{nullptr}; diff --git a/src/core/data/transform.cc b/src/core/data/transform.cc index 6de781b815..f1c189e550 100644 --- a/src/core/data/transform.cc +++ b/src/core/data/transform.cc @@ -228,12 +228,11 @@ std::unique_ptr Promote::invert(const Partition* partition) const { switch (partition->kind()) { case Partition::Kind::NO_PARTITION: { - return create_no_partition(partition->runtime()); + return create_no_partition(); } case Partition::Kind::TILING: { auto tiling = static_cast(partition); - return create_tiling(tiling->runtime(), - tiling->tile_shape().remove(extra_dim_), + return create_tiling(tiling->tile_shape().remove(extra_dim_), tiling->color_shape().remove(extra_dim_), tiling->offsets().remove(extra_dim_)); } diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index 3fda72363a..bf322fd85b 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -24,9 +24,7 @@ namespace legate { -Partition::Partition(Runtime* runtime) : runtime_(runtime) {} - -NoPartition::NoPartition(Runtime* runtime) : Partition(runtime) {} +NoPartition::NoPartition() : Partition() {} bool NoPartition::is_complete_for(const detail::LogicalStore* store) const { return false; } @@ -54,8 +52,8 @@ Legion::Domain NoPartition::launch_domain() const std::string NoPartition::to_string() const { return "NoPartition"; } -Tiling::Tiling(Runtime* runtime, Shape&& tile_shape, Shape&& color_shape, Shape&& offsets) - : Partition(runtime), +Tiling::Tiling(Shape&& tile_shape, Shape&& color_shape, Shape&& offsets) + : Partition(), tile_shape_(std::forward(tile_shape)), color_shape_(std::forward(color_shape)), offsets_(std::forward(offsets)) @@ -96,9 +94,11 @@ Legion::LogicalPartition Tiling::construct(Legion::LogicalRegion region, bool complete) const { auto index_space = region.get_index_space(); - auto index_partition = runtime_->partition_manager()->find_index_partition(index_space, *this); + auto runtime = Runtime::get_runtime(); + auto part_mgr = runtime->partition_manager(); + auto index_partition = part_mgr->find_index_partition(index_space, *this); if (index_partition != Legion::IndexPartition::NO_PART) - return runtime_->create_logical_partition(region, index_partition); + return runtime->create_logical_partition(region, index_partition); auto ndim = static_cast(tile_shape_.size()); @@ -122,15 +122,15 @@ Legion::LogicalPartition Tiling::construct(Legion::LogicalRegion region, color_domain.rect_data[idx + ndim] = color_shape_[idx] - 1; } - auto color_space = runtime_->find_or_create_index_space(color_domain); + auto color_space = runtime->find_or_create_index_space(color_domain); auto kind = complete ? (disjoint ? LEGION_DISJOINT_COMPLETE_KIND : LEGION_ALIASED_COMPLETE_KIND) : (disjoint ? LEGION_DISJOINT_KIND : LEGION_ALIASED_KIND); index_partition = - runtime_->create_restricted_partition(index_space, color_space, kind, transform, extent); - runtime_->partition_manager()->record_index_partition(index_space, *this, index_partition); - return runtime_->create_logical_partition(region, index_partition); + runtime->create_restricted_partition(index_space, color_space, kind, transform, extent); + part_mgr->record_index_partition(index_space, *this, index_partition); + return runtime->create_logical_partition(region, index_partition); } std::unique_ptr Tiling::get_projection(detail::LogicalStore* store) const @@ -160,20 +160,15 @@ std::string Tiling::to_string() const return ss.str(); } -std::unique_ptr create_tiling(Runtime* runtime, - Shape&& tile_shape, +std::unique_ptr create_tiling(Shape&& tile_shape, Shape&& color_shape, Shape&& offsets /*= {}*/) { - return std::make_unique(runtime, - std::forward(tile_shape), + return std::make_unique(std::forward(tile_shape), std::forward(color_shape), std::forward(offsets)); } -std::unique_ptr create_no_partition(Runtime* runtime) -{ - return std::make_unique(runtime); -} +std::unique_ptr create_no_partition() { return std::make_unique(); } } // namespace legate diff --git a/src/core/partitioning/partition.h b/src/core/partitioning/partition.h index 2b35f4d588..b1316e0022 100644 --- a/src/core/partitioning/partition.h +++ b/src/core/partitioning/partition.h @@ -24,7 +24,6 @@ namespace legate { class Projection; -class Runtime; namespace detail { @@ -42,7 +41,7 @@ struct Partition { }; public: - Partition(Runtime* runtime); + Partition() {} virtual ~Partition() {} public: @@ -63,16 +62,12 @@ struct Partition { virtual Legion::Domain launch_domain() const = 0; public: - Runtime* runtime() const { return runtime_; } virtual std::string to_string() const = 0; - - protected: - Runtime* runtime_; }; class NoPartition : public Partition { public: - NoPartition(Runtime* runtime); + NoPartition(); public: virtual Kind kind() const override { return Kind::NO_PARTITION; } @@ -93,14 +88,11 @@ class NoPartition : public Partition { public: virtual std::string to_string() const override; - - private: - Runtime* runtime_; }; class Tiling : public Partition { public: - Tiling(Runtime* runtime, Shape&& tile_shape, Shape&& color_shape, Shape&& offsets); + Tiling(Shape&& tile_shape, Shape&& color_shape, Shape&& offsets); public: bool operator==(const Tiling& other) const; @@ -137,10 +129,9 @@ class Tiling : public Partition { Shape offsets_; }; -std::unique_ptr create_no_partition(Runtime* runtime); +std::unique_ptr create_no_partition(); -std::unique_ptr create_tiling(Runtime* runtime, - Shape&& tile_shape, +std::unique_ptr create_tiling(Shape&& tile_shape, Shape&& color_shape, Shape&& offsets = {}); diff --git a/src/core/partitioning/partitioner.cc b/src/core/partitioning/partitioner.cc index 89c9e7e3a5..ad52925f63 100644 --- a/src/core/partitioning/partitioner.cc +++ b/src/core/partitioning/partitioner.cc @@ -67,8 +67,8 @@ std::shared_ptr Strategy::operator[](const std::shared_ptr& var return finder->second; } -Partitioner::Partitioner(Runtime* runtime, std::vector&& operations) - : runtime_(runtime), operations_(std::forward>(operations)) +Partitioner::Partitioner(std::vector&& operations) + : operations_(std::forward>(operations)) { } diff --git a/src/core/partitioning/partitioner.h b/src/core/partitioning/partitioner.h index 1d152ec407..3f4c88d437 100644 --- a/src/core/partitioning/partitioner.h +++ b/src/core/partitioning/partitioner.h @@ -28,7 +28,6 @@ class LogicalStore; class Operation; class Partition; class Projection; -class Runtime; class Strategy { public: @@ -52,13 +51,12 @@ class Strategy { class Partitioner { public: - Partitioner(Runtime* runtime, std::vector&& operations); + Partitioner(std::vector&& operations); public: std::unique_ptr solve(); private: - Runtime* runtime_; std::vector operations_; }; diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/launcher.cc index 4bb791d53c..f27840efc7 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -28,12 +28,11 @@ namespace legate { -TaskLauncher::TaskLauncher(Runtime* runtime, - LibraryContext* library, +TaskLauncher::TaskLauncher(LibraryContext* library, int64_t task_id, int64_t mapper_id /*= 0*/, int64_t tag /*= 0*/) - : runtime_(runtime), library_(library), task_id_(task_id), mapper_id_(mapper_id), tag_(tag) + : library_(library), task_id_(task_id), mapper_id_(mapper_id), tag_(tag) { req_analyzer_ = new RequirementAnalyzer(); buffer_ = new BufferBuilder(); @@ -88,14 +87,14 @@ void TaskLauncher::add_reduction(detail::LogicalStore* store, void TaskLauncher::execute(const Legion::Domain& launch_domain) { auto legion_launcher = build_index_task(launch_domain); - runtime_->dispatch(legion_launcher); + Runtime::get_runtime()->dispatch(legion_launcher); delete legion_launcher; } void TaskLauncher::execute_single() { auto legion_launcher = build_single_task(); - runtime_->dispatch(legion_launcher); + Runtime::get_runtime()->dispatch(legion_launcher); delete legion_launcher; } diff --git a/src/core/runtime/launcher.h b/src/core/runtime/launcher.h index 8036ad92df..61203c0e1b 100644 --- a/src/core/runtime/launcher.h +++ b/src/core/runtime/launcher.h @@ -28,7 +28,6 @@ class LibraryContext; class LogicalStore; class Projection; class RequirementAnalyzer; -class Runtime; class Scalar; namespace detail { @@ -39,11 +38,7 @@ class LogicalStore; class TaskLauncher { public: - TaskLauncher(Runtime* runtime, - LibraryContext* library, - int64_t task_id, - int64_t mapper_id = 0, - int64_t tag = 0); + TaskLauncher(LibraryContext* library, int64_t task_id, int64_t mapper_id = 0, int64_t tag = 0); ~TaskLauncher(); public: @@ -84,7 +79,6 @@ class TaskLauncher { Legion::TaskLauncher* build_single_task(); private: - Runtime* runtime_; LibraryContext* library_; int64_t task_id_; int64_t mapper_id_; diff --git a/src/core/runtime/operation.cc b/src/core/runtime/operation.cc index c46789f451..4ec6779530 100644 --- a/src/core/runtime/operation.cc +++ b/src/core/runtime/operation.cc @@ -30,12 +30,8 @@ namespace legate { -Operation::Operation(Runtime* runtime, - LibraryContext* library, - uint64_t unique_id, - int64_t mapper_id) - : runtime_(runtime), - library_(library), +Operation::Operation(LibraryContext* library, uint64_t unique_id, int64_t mapper_id) + : library_(library), unique_id_(unique_id), mapper_id_(mapper_id), constraints_(std::make_shared()) @@ -96,12 +92,8 @@ void Operation::add_constraint(std::shared_ptr constraint) std::shared_ptr Operation::constraints() const { return constraints_; } -Task::Task(Runtime* runtime, - LibraryContext* library, - int64_t task_id, - uint64_t unique_id, - int64_t mapper_id /*=0*/) - : Operation(runtime, library, unique_id, mapper_id), task_id_(task_id) +Task::Task(LibraryContext* library, int64_t task_id, uint64_t unique_id, int64_t mapper_id /*=0*/) + : Operation(library, unique_id, mapper_id), task_id_(task_id) { } @@ -112,7 +104,7 @@ void Task::add_scalar_arg(const Scalar& scalar) { scalars_.push_back(scalar); } void Task::launch(Strategy* p_strategy) { auto& strategy = *p_strategy; - TaskLauncher launcher(runtime_, library_, task_id_, mapper_id_); + TaskLauncher launcher(library_, task_id_, mapper_id_); for (auto& pair : inputs_) { auto& store = pair.first; diff --git a/src/core/runtime/operation.h b/src/core/runtime/operation.h index 361aac07c0..aff662363d 100644 --- a/src/core/runtime/operation.h +++ b/src/core/runtime/operation.h @@ -27,7 +27,6 @@ namespace legate { class Constraint; class ConstraintGraph; class LibraryContext; -class Runtime; class Scalar; class Strategy; class Variable; @@ -40,7 +39,7 @@ class LogicalStore; class Operation { public: - Operation(Runtime* runtime, LibraryContext* library, uint64_t unique_id, int64_t mapper_id); + Operation(LibraryContext* library, uint64_t unique_id, int64_t mapper_id); public: virtual ~Operation(); @@ -65,7 +64,6 @@ class Operation { virtual std::string to_string() const = 0; protected: - Runtime* runtime_; LibraryContext* library_; uint64_t unique_id_; int64_t mapper_id_; @@ -90,11 +88,7 @@ class Operation { class Task : public Operation { public: - Task(Runtime* runtime, - LibraryContext* library, - int64_t task_id, - uint64_t unique_id, - int64_t mapper_id = 0); + Task(LibraryContext* library, int64_t task_id, uint64_t unique_id, int64_t mapper_id = 0); public: ~Task(); diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 90065ac994..560fcf1236 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -336,13 +336,13 @@ std::shared_ptr FieldManager::allocate_field() auto field = free_fields_.front(); log_legate.debug("Field %u recycled in field manager %p", field.second, this); free_fields_.pop_front(); - rf = new LogicalRegionField(runtime_, field.first, field.second); + rf = new LogicalRegionField(field.first, field.second); } else { auto rgn_mgr = runtime_->find_or_create_region_manager(shape_); LogicalRegion lr; FieldID fid; std::tie(lr, fid) = rgn_mgr->allocate_field(field_size_); - rf = new LogicalRegionField(runtime_, lr, fid); + rf = new LogicalRegionField(lr, fid); log_legate.debug("Field %u created in field manager %p", fid, this); } assert(rf != nullptr); @@ -588,7 +588,7 @@ std::unique_ptr Runtime::create_task(LibraryContext* library, int64_t task_id, int64_t mapper_id /*=0*/) { - return std::make_unique(this, library, task_id, next_unique_id_++, mapper_id); + return std::make_unique(library, task_id, next_unique_id_++, mapper_id); } void Runtime::submit(std::unique_ptr op) @@ -607,7 +607,7 @@ void Runtime::schedule(std::vector> operations) op_pointers.reserve(operations.size()); for (auto& op : operations) op_pointers.push_back(op.get()); - Partitioner partitioner(this, std::move(op_pointers)); + Partitioner partitioner(std::move(op_pointers)); auto strategy = partitioner.solve(); for (auto& op : operations) op->launch(strategy.get()); @@ -615,12 +615,12 @@ void Runtime::schedule(std::vector> operations) LogicalStore Runtime::create_store(std::vector extents, LegateTypeCode code) { - return LogicalStore(std::make_shared(this, code, extents)); + return LogicalStore(std::make_shared(code, extents)); } LogicalStore Runtime::create_store(const Scalar& scalar) { - return LogicalStore(std::make_shared(this, scalar.code(), scalar.ptr())); + return LogicalStore(std::make_shared(scalar.code(), scalar.ptr())); } std::shared_ptr Runtime::create_region_field(const tuple& extents, From ab95f542e9f2d4d133989d62890db1c7804dc114 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 5 Aug 2022 23:49:25 -0700 Subject: [PATCH 0070/1425] Remove an unused constructor --- src/core/data/logical_store_detail.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/data/logical_store_detail.h b/src/core/data/logical_store_detail.h index 1d97ae0e3c..d28cc86a1a 100644 --- a/src/core/data/logical_store_detail.h +++ b/src/core/data/logical_store_detail.h @@ -25,7 +25,6 @@ namespace detail { class LogicalStore { public: - LogicalStore(); LogicalStore(LegateTypeCode code, tuple extents); LogicalStore(LegateTypeCode code, tuple extents, From c4ef27a7281b7de6c22edd985a24521579e65c31 Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Mon, 8 Aug 2022 10:17:20 -0700 Subject: [PATCH 0071/1425] Release tasks (#321) Co-authored-by: Marcin Zalewski --- install.py | 6 +++--- setup.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/install.py b/install.py index 6d252b97e1..b53bf63ace 100755 --- a/install.py +++ b/install.py @@ -214,7 +214,7 @@ def install_gasnet(gasnet_dir, conduit, thread_count): shutil.rmtree(temp_dir) -def install_legion(legion_src_dir, branch, commit=None): +def install_legion(legion_src_dir, branch, commit="96777114"): print("Legate is installing Legion into a local directory...") # For now all we have to do is clone legion since we build it with Legate git_clone( @@ -234,7 +234,7 @@ def install_thrust(thrust_dir): ) -def update_legion(legion_src_dir, branch, commit=None): +def update_legion(legion_src_dir, branch, commit="96777114"): # Make sure we are on the right branch for single/multi-node git_update(legion_src_dir, branch=branch, commit=commit) @@ -1044,7 +1044,7 @@ def driver(): "--legion-branch", dest="legion_branch", required=False, - default="control_replication", + default=None, help="Legion branch to build Legate with.", ) args, unknown = parser.parse_known_args() diff --git a/setup.py b/setup.py index 362981b56e..d5924e8bff 100755 --- a/setup.py +++ b/setup.py @@ -72,7 +72,7 @@ def run(self): sys.argv.remove("--recurse") setup( name="legate.core", - version="22.03", + version="22.08.00", packages=find_packages( where=".", include=["legate*"], From 5b0903d4a68b7297cbfacb02eb70a47ad6bdac42 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Mon, 8 Aug 2022 23:48:18 -0700 Subject: [PATCH 0072/1425] Minor renaming --- src/core/data/logical_store.cc | 2 +- src/core/data/logical_store_detail.h | 2 +- src/core/partitioning/partition.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index 8a1cf008ca..12324b1a71 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -198,7 +198,7 @@ Legion::ProjectionID LogicalStore::compute_projection() const return Runtime::get_runtime()->get_projection(ndim, point); } -std::unique_ptr LogicalStore::find_or_create_partition(const Partition* partition) +std::unique_ptr LogicalStore::create_projection(const Partition* partition) { if (scalar_) return std::make_unique(); diff --git a/src/core/data/logical_store_detail.h b/src/core/data/logical_store_detail.h index d28cc86a1a..416a3ad56a 100644 --- a/src/core/data/logical_store_detail.h +++ b/src/core/data/logical_store_detail.h @@ -71,7 +71,7 @@ class LogicalStore { std::shared_ptr get_physical_store(LibraryContext* context); public: - std::unique_ptr find_or_create_partition(const Partition* partition); + std::unique_ptr create_projection(const Partition* partition); std::unique_ptr find_or_create_key_partition(); private: diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index bf322fd85b..6d2fc1ccde 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -135,7 +135,7 @@ Legion::LogicalPartition Tiling::construct(Legion::LogicalRegion region, std::unique_ptr Tiling::get_projection(detail::LogicalStore* store) const { - return store->find_or_create_partition(this); + return store->create_projection(this); } bool Tiling::has_launch_domain() const { return true; } From c53d5b7172aa755d0eb6e8d74505807d9611f2ae Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Tue, 9 Aug 2022 23:04:49 -0700 Subject: [PATCH 0073/1425] Implement storage layer to support views --- src/core.mk | 1 + src/core/data/logical_store.cc | 211 +----------------------- src/core/data/logical_store.h | 4 +- src/core/data/logical_store_detail.cc | 227 ++++++++++++++++++++++++++ src/core/data/logical_store_detail.h | 79 ++++++--- src/core/partitioning/partition.cc | 6 + src/core/partitioning/partition.h | 2 + src/core/runtime/launcher.cc | 6 +- src/core/runtime/launcher_arg.cc | 4 +- src/core/runtime/projection.cc | 17 ++ src/core/runtime/projection.h | 4 + src/core/runtime/runtime.cc | 52 ++++-- src/core/runtime/runtime.h | 11 +- src/core/utilities/buffer_builder.h | 4 + src/core/utilities/buffer_builder.inl | 6 + 15 files changed, 380 insertions(+), 254 deletions(-) create mode 100644 src/core/data/logical_store_detail.cc diff --git a/src/core.mk b/src/core.mk index 5d6f8f52d5..8cc26842fd 100644 --- a/src/core.mk +++ b/src/core.mk @@ -21,6 +21,7 @@ GEN_CPU_SRC = core/legate_c.cc \ core/comm/coll.cc \ core/data/allocator.cc \ core/data/logical_store.cc \ + core/data/logical_store_detail.cc \ core/data/scalar.cc \ core/data/store.cc \ core/data/transform.cc \ diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index 12324b1a71..ca16636c01 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -43,225 +43,16 @@ Domain LogicalRegionField::domain() const return Runtime::get_runtime()->get_index_space_domain(lr_.get_index_space()); } -namespace detail { - -LogicalStore::LogicalStore(LegateTypeCode code, tuple extents) - : code_(code), - extents_(std::move(extents)), - parent_(nullptr), - transform_(std::make_shared()) -{ -} - -LogicalStore::LogicalStore(LegateTypeCode code, - tuple extents, - std::shared_ptr parent, - std::shared_ptr transform) - : code_(code), - extents_(std::move(extents)), - parent_(std::move(parent)), - transform_(std::move(transform)) -{ -} - -struct datalen_fn { - template - size_t operator()() - { - return sizeof(legate_type_of); - } -}; - -LogicalStore::LogicalStore(LegateTypeCode code, const void* data) - : scalar_(true), code_(code), extents_({1}), transform_(std::make_shared()) -{ - auto datalen = type_dispatch(code, datalen_fn{}); - future_ = Runtime::get_runtime()->create_future(data, datalen); -} - -LogicalStore::~LogicalStore() -{ - if (mapped_ != nullptr) mapped_->unmap(); -} - -bool LogicalStore::scalar() const { return scalar_; } - -int32_t LogicalStore::dim() const { return static_cast(extents_.size()); } - -LegateTypeCode LogicalStore::code() const { return code_; } - -Domain LogicalStore::domain() const -{ - assert(nullptr != region_field_); - return region_field_->domain(); -} - -const std::vector& LogicalStore::extents() const { return extents_.data(); } - -size_t LogicalStore::volume() const { return extents_.volume(); } - -bool LogicalStore::has_storage() const { return nullptr != region_field_; } - -std::shared_ptr LogicalStore::get_storage() -{ - assert(!scalar_); - if (nullptr == parent_) { - if (!has_storage()) create_storage(); - return region_field_; - } else - return parent_->get_storage(); -} - -Legion::Future LogicalStore::get_future() -{ - assert(scalar_); - if (nullptr == parent_) { - return future_; - } else - return parent_->get_future(); -} - -void LogicalStore::create_storage() -{ - region_field_ = Runtime::get_runtime()->create_region_field(extents_, code_); -} - -std::shared_ptr LogicalStore::promote(int32_t extra_dim, - size_t dim_size, - std::shared_ptr parent) const -{ - if (extra_dim < 0 || static_cast(extra_dim) > extents_.size()) { - log_legate.error( - "Invalid promotion on dimension %d for a %zd-D store", extra_dim, extents_.size()); - LEGATE_ABORT; - } - - auto new_extents = extents_.insert(extra_dim, dim_size); - auto transform = transform_->push(std::make_unique(extra_dim, dim_size)); - return std::make_shared( - code_, std::move(new_extents), std::move(parent), std::move(transform)); -} - -std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) -{ - // TODO: Need to support inline mapping for scalars - assert(!scalar_); - if (nullptr != mapped_) return mapped_; - auto rf = Runtime::get_runtime()->map_region_field(context, region_field_); - mapped_ = std::make_shared(dim(), code_, -1, std::move(rf), transform_); - return mapped_; -} - -std::unique_ptr LogicalStore::invert_partition(const Partition* partition) const -{ - if (nullptr == parent_) { - switch (partition->kind()) { - case Partition::Kind::NO_PARTITION: { - return create_no_partition(); - } - case Partition::Kind::TILING: { - auto tiling = static_cast(partition); - Shape tile_shape = tiling->tile_shape(); - Shape color_shape = tiling->color_shape(); - Shape offsets = tiling->offsets(); - return create_tiling(std::move(tile_shape), std::move(color_shape), std::move(offsets)); - } - } - } else { - auto inverted = transform_->invert(partition); - return parent_->invert_partition(inverted.get()); - } - assert(false); - return nullptr; -} - -proj::SymbolicPoint LogicalStore::invert(const proj::SymbolicPoint& point) const -{ - if (nullptr == parent_) return point; - return parent_->invert(transform_->invert(point)); -} - -Legion::ProjectionID LogicalStore::compute_projection() const -{ - auto ndim = dim(); - std::vector exprs; - for (int32_t dim = 0; dim < ndim; ++dim) exprs.push_back(proj::SymbolicExpr(dim)); - auto point = invert(proj::SymbolicPoint(std::move(exprs))); - - bool identity_mapping = true; - for (int32_t dim = 0; dim < ndim; ++dim) - identity_mapping = identity_mapping && point[dim].is_identity(dim); - - if (identity_mapping) - return 0; - else - return Runtime::get_runtime()->get_projection(ndim, point); -} - -std::unique_ptr LogicalStore::create_projection(const Partition* partition) -{ - if (scalar_) return std::make_unique(); - - // We're about to create a legion partition for this store, so the store should have its region - // created. - auto proj = compute_projection(); - auto inverted = invert_partition(partition); - - auto lr = get_storage()->region(); - auto lp = - inverted->construct(lr, inverted->is_disjoint_for(nullptr), inverted->is_complete_for(nullptr)); - return std::make_unique(lp, proj); -} - -std::unique_ptr LogicalStore::find_or_create_key_partition() -{ - if (scalar_) return create_no_partition(); - - auto part_mgr = Runtime::get_runtime()->partition_manager(); - auto launch_shape = part_mgr->compute_launch_shape(extents()); - if (launch_shape.empty()) - return create_no_partition(); - else { - auto tile_shape = part_mgr->compute_tile_shape(extents_, launch_shape); - return create_tiling(std::move(tile_shape), std::move(launch_shape)); - } -} - -void LogicalStore::pack(BufferBuilder& buffer) const -{ - buffer.pack(scalar_); - buffer.pack(false); - buffer.pack(dim()); - buffer.pack(code_); - transform_->pack(buffer); -} - -void LogicalStore::pack_transform(BufferBuilder& buffer) const -{ - if (nullptr == parent_) - buffer.pack(-1); - else { - transform_->pack(buffer); - parent_->pack_transform(buffer); - } -} - -} // namespace detail - LogicalStore::LogicalStore(std::shared_ptr&& impl) : impl_(std::forward(impl)) { } -bool LogicalStore::scalar() const { return impl_->scalar(); } - int32_t LogicalStore::dim() const { return impl_->dim(); } LegateTypeCode LogicalStore::code() const { return impl_->code(); } -Domain LogicalStore::domain() const { return impl_->domain(); } - -const std::vector& LogicalStore::extents() const { return impl_->extents(); } +const tuple& LogicalStore::extents() const { return impl_->extents(); } size_t LogicalStore::volume() const { return impl_->volume(); } diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index 6dc043e965..d7da902576 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -53,11 +53,9 @@ class LogicalStore { LogicalStore& operator=(LogicalStore&& other) = default; public: - bool scalar() const; int32_t dim() const; LegateTypeCode code() const; - Legion::Domain domain() const; - const std::vector& extents() const; + const tuple& extents() const; size_t volume() const; public: diff --git a/src/core/data/logical_store_detail.cc b/src/core/data/logical_store_detail.cc new file mode 100644 index 0000000000..9b16457cc1 --- /dev/null +++ b/src/core/data/logical_store_detail.cc @@ -0,0 +1,227 @@ +/* Copyright 2022 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/data/logical_store_detail.h" + +#include "core/runtime/req_analyzer.h" +#include "core/utilities/buffer_builder.h" +#include "core/utilities/dispatch.h" +#include "core/utilities/type_traits.h" +#include "legate_defines.h" + +namespace legate { + +extern Legion::Logger log_legate; + +namespace detail { + +//////////////////////////////////////////////////// +// legate::detail::Storage +//////////////////////////////////////////////////// + +Storage::Storage(tuple extents, LegateTypeCode code) + : extents_(extents), code_(code), volume_(extents.volume()) +{ +} + +Storage::Storage(tuple extents, LegateTypeCode code, const Legion::Future& future) + : extents_(extents), code_(code), kind_(Kind::FUTURE), future_(future), volume_(extents.volume()) +{ +} + +int32_t Storage::dim() { return static_cast(extents_.size()); } + +LogicalRegionField* Storage::get_region_field() +{ +#ifdef DEBUG_LEGATE + assert(Kind::REGION_FIELD == kind_); +#endif + if (nullptr == region_field_) + region_field_ = Runtime::get_runtime()->create_region_field(extents_, code_); + return region_field_.get(); +} + +Legion::Future Storage::get_future() const +{ +#ifdef DEBUG_LEGATE + assert(Kind::FUTURE == kind_); +#endif + return future_; +} + +RegionField Storage::map(LibraryContext* context) +{ +#ifdef DEBUG_LEGATE + assert(Kind::REGION_FIELD == kind_); +#endif + return Runtime::get_runtime()->map_region_field(context, region_field_.get()); +} + +Partition* Storage::find_key_partition() const { return key_partition_.get(); } + +void Storage::set_key_partition(std::unique_ptr&& key_partition) +{ + key_partition_ = std::forward(key_partition); +} + +void Storage::reset_key_partition() { key_partition_ = nullptr; } + +Legion::LogicalPartition Storage::find_or_create_legion_partition(const Partition* partition) +{ +#ifdef DEBUG_LEGATE + assert(Kind::REGION_FIELD == kind_); +#endif + auto region = get_region_field()->region(); + return partition->construct( + region, partition->is_disjoint_for(nullptr), partition->is_complete_for(nullptr)); +} + +//////////////////////////////////////////////////// +// legate::detail::LogicalStore +//////////////////////////////////////////////////// + +LogicalStore::LogicalStore(std::shared_ptr&& storage) + : store_id_(Runtime::get_runtime()->get_unique_store_id()), + extents_(storage->extents()), + storage_(std::forward(storage)), + transform_(std::make_shared()) +{ +#ifdef DEBUG_LEGATE + assert(transform_ != nullptr); + + log_legate.debug() << "Create Store(" << store_id_ << ") {shape: " << extents_ << "}"; +#endif +} + +LogicalStore::LogicalStore(tuple&& extents, + const std::shared_ptr& storage, + std::shared_ptr&& transform) + : store_id_(Runtime::get_runtime()->get_unique_store_id()), + extents_(std::forward(extents)), + storage_(storage), + transform_(std::forward(transform)) +{ +#ifdef DEBUG_LEGATE + assert(transform_ != nullptr); + + log_legate.debug() << "Create Store(" << store_id_ << ") {shape: " << extents_ + << ", transform: " << *transform_ << "}"; +#endif +} + +LogicalStore::~LogicalStore() +{ + if (mapped_ != nullptr) mapped_->unmap(); +} + +const tuple& LogicalStore::extents() const { return extents_; } + +size_t LogicalStore::volume() const { return extents_.volume(); } + +int32_t LogicalStore::dim() const { return static_cast(extents_.size()); } + +bool LogicalStore::scalar() const { return storage_->kind() == Storage::Kind::FUTURE; } + +LegateTypeCode LogicalStore::code() const { return storage_->code(); } + +LogicalRegionField* LogicalStore::get_region_field() { return storage_->get_region_field(); } + +Legion::Future LogicalStore::get_future() { return storage_->get_future(); } + +std::shared_ptr LogicalStore::promote(int32_t extra_dim, + size_t dim_size, + std::shared_ptr parent) const +{ + if (extra_dim < 0 || extra_dim > dim()) { + log_legate.error("Invalid promotion on dimension %d for a %d-D store", extra_dim, dim()); + LEGATE_ABORT; + } + + auto new_extents = extents_.insert(extra_dim, dim_size); + auto transform = transform_->push(std::make_unique(extra_dim, dim_size)); + return std::make_shared(std::move(new_extents), storage_, std::move(transform)); +} + +std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) +{ + // TODO: Need to support inline mapping for scalars + assert(storage_->kind() == Storage::Kind::REGION_FIELD); + if (nullptr != mapped_) return mapped_; + auto region_field = storage_->map(context); + mapped_ = std::make_shared(dim(), code(), -1, std::move(region_field), transform_); + return std::move(mapped_); +} + +Legion::ProjectionID LogicalStore::compute_projection() const +{ + if (transform_->identity()) return 0; + + auto ndim = dim(); + auto point = transform_->invert(proj::create_symbolic_point(ndim)); + return Runtime::get_runtime()->get_projection(ndim, point); +} + +std::unique_ptr LogicalStore::create_projection(const Partition* partition) +{ + if (scalar()) return std::make_unique(); + + // We're about to create a legion partition for this store, so the store should have its region + // created. + auto proj_id = compute_projection(); + auto* orig_partition = partition; + std::unique_ptr inverted = nullptr; + if (!transform_->identity()) { + inverted = transform_->invert(partition); + partition = inverted.get(); + } + +#ifdef DEBUG_LEGATE + log_legate.debug() << "Partition Store(" << store_id_ << ") {partition: " << *orig_partition + << ", inverted: " << *partition << ", projection: " << proj_id << "}"; +#endif + + auto region_field = get_region_field(); + auto region = region_field->region(); + auto legion_partition = partition->construct( + region, partition->is_disjoint_for(nullptr), partition->is_complete_for(nullptr)); + return std::make_unique(legion_partition, proj_id); +} + +std::unique_ptr LogicalStore::find_or_create_key_partition() +{ + if (scalar()) return create_no_partition(); + + auto part_mgr = Runtime::get_runtime()->partition_manager(); + auto launch_shape = part_mgr->compute_launch_shape(extents_); + if (launch_shape.empty()) + return create_no_partition(); + else { + auto tile_shape = part_mgr->compute_tile_shape(extents_, launch_shape); + return create_tiling(std::move(tile_shape), std::move(launch_shape)); + } +} + +void LogicalStore::pack(BufferBuilder& buffer) const +{ + buffer.pack(scalar()); + buffer.pack(false); + buffer.pack(dim()); + buffer.pack(code()); + transform_->pack(buffer); +} + +} // namespace detail +} // namespace legate diff --git a/src/core/data/logical_store_detail.h b/src/core/data/logical_store_detail.h index 416a3ad56a..5b4181fdda 100644 --- a/src/core/data/logical_store_detail.h +++ b/src/core/data/logical_store_detail.h @@ -23,14 +23,57 @@ namespace legate { namespace detail { +class Storage { + public: + enum class Kind : int32_t { + REGION_FIELD = 0, + FUTURE = 1, + }; + + public: + // Create a RegionField-backed storage. Initialized lazily. + Storage(tuple extents, LegateTypeCode code); + // Create a Future-backed storage. Initialized eagerly. + Storage(tuple extents, LegateTypeCode code, const Legion::Future& future); + + public: + const tuple& extents() const { return extents_; } + size_t volume() const { return volume_; } + int32_t dim(); + LegateTypeCode code() const { return code_; } + Kind kind() const { return kind_; } + + public: + LogicalRegionField* get_region_field(); + Legion::Future get_future() const; + + public: + RegionField map(LibraryContext* context); + + public: + Partition* find_key_partition() const; + void set_key_partition(std::unique_ptr&& key_partition); + void reset_key_partition(); + Legion::LogicalPartition find_or_create_legion_partition(const Partition* partition); + + private: + tuple extents_; + size_t volume_; + LegateTypeCode code_{MAX_TYPE_NUMBER}; + Kind kind_{Kind::REGION_FIELD}; + std::shared_ptr region_field_{nullptr}; + Legion::Future future_{}; + + private: + std::unique_ptr key_partition_{nullptr}; +}; + class LogicalStore { public: - LogicalStore(LegateTypeCode code, tuple extents); - LogicalStore(LegateTypeCode code, - tuple extents, - std::shared_ptr parent, - std::shared_ptr transform); - LogicalStore(LegateTypeCode code, const void* data); + LogicalStore(std::shared_ptr&& storage); + LogicalStore(tuple&& extents, + const std::shared_ptr& storage, + std::shared_ptr&& transform); public: ~LogicalStore(); @@ -47,16 +90,14 @@ class LogicalStore { LogicalStore& operator=(LogicalStore&& other) = default; public: - bool scalar() const; + const tuple& extents() const; + size_t volume() const; int32_t dim() const; + bool scalar() const; LegateTypeCode code() const; - Legion::Domain domain() const; - const std::vector& extents() const; - size_t volume() const; public: - bool has_storage() const; - std::shared_ptr get_storage(); + LogicalRegionField* get_region_field(); Legion::Future get_future(); private: @@ -75,24 +116,16 @@ class LogicalStore { std::unique_ptr find_or_create_key_partition(); private: - std::unique_ptr invert_partition(const Partition* partition) const; - proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const; Legion::ProjectionID compute_projection() const; public: void pack(BufferBuilder& buffer) const; private: - void pack_transform(BufferBuilder& buffer) const; - - private: - bool scalar_{false}; - LegateTypeCode code_{MAX_TYPE_NUMBER}; + uint64_t store_id_; tuple extents_; - std::shared_ptr region_field_{nullptr}; - Legion::Future future_{}; - std::shared_ptr parent_{nullptr}; - std::shared_ptr transform_{nullptr}; + std::shared_ptr storage_; + std::shared_ptr transform_; std::shared_ptr mapped_{nullptr}; }; diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index 6d2fc1ccde..fb0fad5280 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -171,4 +171,10 @@ std::unique_ptr create_tiling(Shape&& tile_shape, std::unique_ptr create_no_partition() { return std::make_unique(); } +std::ostream& operator<<(std::ostream& out, const Partition& partition) +{ + out << partition.to_string(); + return out; +} + } // namespace legate diff --git a/src/core/partitioning/partition.h b/src/core/partitioning/partition.h index b1316e0022..64965288eb 100644 --- a/src/core/partitioning/partition.h +++ b/src/core/partitioning/partition.h @@ -135,4 +135,6 @@ std::unique_ptr create_tiling(Shape&& tile_shape, Shape&& color_shape, Shape&& offsets = {}); +std::ostream& operator<<(std::ostream& out, const Partition& partition); + } // namespace legate diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/launcher.cc index f27840efc7..9cde824af2 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -113,9 +113,9 @@ void TaskLauncher::add_store(std::vector& args, if (has_storage) futures_.push_back(store->get_future()); args.push_back(new FutureStoreArg(store, read_only, has_storage, redop)); } else { - auto storage = store->get_storage(); - auto region = storage->region(); - auto field_id = storage->field_id(); + auto region_field = store->get_region_field(); + auto region = region_field->region(); + auto field_id = region_field->field_id(); auto proj_info = new ProjectionInfo(proj.get(), tag, flags); diff --git a/src/core/runtime/launcher_arg.cc b/src/core/runtime/launcher_arg.cc index 57281cd421..98dea4569a 100644 --- a/src/core/runtime/launcher_arg.cc +++ b/src/core/runtime/launcher_arg.cc @@ -35,7 +35,7 @@ RegionFieldArg::RegionFieldArg(RequirementAnalyzer* analyzer, const ProjectionInfo* proj_info) : analyzer_(analyzer), store_(store), - region_(store_->get_storage()->region()), + region_(store_->get_region_field()->region()), field_id_(field_id), privilege_(privilege), proj_info_(proj_info) @@ -76,7 +76,7 @@ void FutureStoreArg::pack(BufferBuilder& buffer) const buffer.pack(read_only_); buffer.pack(has_storage_); buffer.pack(type_dispatch(store_->code(), datalen_fn{})); - buffer.pack(store_->extents()); + buffer.pack(store_->extents().data()); } } // namespace legate diff --git a/src/core/runtime/projection.cc b/src/core/runtime/projection.cc index 71d3c011ff..2078cfa40b 100644 --- a/src/core/runtime/projection.cc +++ b/src/core/runtime/projection.cc @@ -86,6 +86,23 @@ std::ostream& operator<<(std::ostream& out, const SymbolicExpr& expr) return out; } +SymbolicPoint create_symbolic_point(int32_t ndim) +{ + std::vector exprs; + exprs.resize(ndim); + for (int32_t dim = 0; dim < ndim; ++dim) exprs[dim] = proj::SymbolicExpr(dim); + return SymbolicPoint(std::move(exprs)); +} + +bool is_identity(int32_t src_ndim, const SymbolicPoint& point) +{ + auto ndim = static_cast(point.size()); + if (src_ndim != ndim) return false; + for (int32_t dim = 0; dim < ndim; ++dim) + if (!point[dim].is_identity(dim)) return false; + return true; +} + } // namespace proj class DelinearizationFunctor : public ProjectionFunctor { diff --git a/src/core/runtime/projection.h b/src/core/runtime/projection.h index 59d03ad66d..8fd0f8d369 100644 --- a/src/core/runtime/projection.h +++ b/src/core/runtime/projection.h @@ -56,6 +56,10 @@ std::ostream& operator<<(std::ostream& out, const SymbolicExpr& expr); using SymbolicPoint = tuple; using SymbolicFunctor = SymbolicPoint (*)(const SymbolicPoint&); +SymbolicPoint create_symbolic_point(int32_t ndim); + +bool is_identity(int32_t ndim, const SymbolicPoint& point); + } // namespace proj // Interface for Legate projection functors diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 560fcf1236..a98278295b 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -615,14 +615,20 @@ void Runtime::schedule(std::vector> operations) LogicalStore Runtime::create_store(std::vector extents, LegateTypeCode code) { - return LogicalStore(std::make_shared(code, extents)); + auto storage = std::make_shared(extents, code); + return LogicalStore(std::make_shared(std::move(storage))); } LogicalStore Runtime::create_store(const Scalar& scalar) { - return LogicalStore(std::make_shared(scalar.code(), scalar.ptr())); + tuple extents{1}; + auto future = create_future(scalar.ptr(), scalar.size()); + auto storage = std::make_shared(extents, scalar.code(), future); + return LogicalStore(std::make_shared(std::move(storage))); } +uint64_t Runtime::get_unique_store_id() { return next_store_id_++; } + std::shared_ptr Runtime::create_region_field(const tuple& extents, LegateTypeCode code) { @@ -637,19 +643,26 @@ std::shared_ptr Runtime::create_region_field(const tupleallocate_field(); } -RegionField Runtime::map_region_field(LibraryContext* context, - std::shared_ptr rf) +RegionField Runtime::map_region_field(LibraryContext* context, const LogicalRegionField* rf) { auto region = rf->region(); auto field_id = rf->field_id(); - RegionRequirement req(region, READ_WRITE, EXCLUSIVE, region); - req.add_field(field_id); - - auto mapper_id = context->get_mapper_id(0); - // TODO: We need to pass the metadata about logical store - InlineLauncher launcher(req, mapper_id); - auto pr = legion_runtime_->map_region(legion_context_, launcher); + PhysicalRegion pr; + + RegionFieldID key(region, field_id); + auto finder = inline_mapped_.find(key); + if (inline_mapped_.end() == finder) { + RegionRequirement req(region, READ_WRITE, EXCLUSIVE, region); + req.add_field(field_id); + + auto mapper_id = context->get_mapper_id(0); + // TODO: We need to pass the metadata about logical store + InlineLauncher launcher(req, mapper_id); + pr = legion_runtime_->map_region(legion_context_, launcher); + inline_mapped_.insert({key, pr}); + } else + pr = finder->second; return RegionField(rf->dim(), pr, field_id); } @@ -762,6 +775,18 @@ std::shared_ptr Runtime::dispatch(IndexTaskLauncher* launcher) Legion::ProjectionID Runtime::get_projection(int32_t src_ndim, const proj::SymbolicPoint& point) { +#ifdef DEBUG_LEGATE + log_legate.debug() << "Query projection {src_ndim: " << src_ndim << ", point: " << point << "}"; +#endif + + if (is_identity(src_ndim, point)) { +#ifdef DEBUG_LEGATE + log_legate.debug() << "Identity projection {src_ndim: " << src_ndim << ", point: " << point + << "}"; +#endif + return 0; + } + ProjectionDesc key(src_ndim, point); auto finder = registered_projections_.find(key); if (registered_projections_.end() != finder) return finder->second; @@ -781,6 +806,11 @@ Legion::ProjectionID Runtime::get_projection(int32_t src_ndim, const proj::Symbo src_ndim, ndim, dims.data(), weights.data(), offsets.data(), proj_id); registered_projections_[key] = proj_id; +#ifdef DEBUG_LEGATE + log_legate.debug() << "Register projection " << proj_id << " {src_ndim: " << src_ndim + << ", point: " << point << "}"; +#endif + return proj_id; } diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index ebd295e950..13eacf1c9d 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -116,10 +116,12 @@ class Runtime { public: LogicalStore create_store(std::vector extents, LegateTypeCode code); LogicalStore create_store(const Scalar& scalar); + uint64_t get_unique_store_id(); + + public: std::shared_ptr create_region_field(const tuple& extents, LegateTypeCode code); - RegionField map_region_field(LibraryContext* context, - std::shared_ptr region_field); + RegionField map_region_field(LibraryContext* context, const LogicalRegionField* region_field); void unmap_physical_region(Legion::PhysicalRegion pr); public: @@ -185,6 +187,11 @@ class Runtime { size_t window_size_{1}; uint64_t next_unique_id_{0}; + private: + using RegionFieldID = std::pair; + std::map inline_mapped_; + uint64_t next_store_id_{1}; + private: std::map libraries_; }; diff --git a/src/core/utilities/buffer_builder.h b/src/core/utilities/buffer_builder.h index 9d3c149b9a..7b10975c1e 100644 --- a/src/core/utilities/buffer_builder.h +++ b/src/core/utilities/buffer_builder.h @@ -18,6 +18,8 @@ #include "legion.h" +#include "core/utilities/tuple.h" + namespace legate { class BufferBuilder { @@ -29,6 +31,8 @@ class BufferBuilder { void pack(const T& value); template void pack(const std::vector& values); + template + void pack(const tuple& values); void pack_buffer(const void* buffer, size_t size); public: diff --git a/src/core/utilities/buffer_builder.inl b/src/core/utilities/buffer_builder.inl index 664b719582..a22fc30ed1 100644 --- a/src/core/utilities/buffer_builder.inl +++ b/src/core/utilities/buffer_builder.inl @@ -30,4 +30,10 @@ void BufferBuilder::pack(const std::vector& values) pack_buffer(values.data(), size * sizeof(T)); } +template +void BufferBuilder::pack(const tuple& values) +{ + pack(values.data()); +} + } // namespace legate From 798733fdf9c558fcdf091567ed4d72e56d991634 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 10 Aug 2022 15:48:53 -0700 Subject: [PATCH 0074/1425] Clean up memory management for constraints --- src/core/partitioning/constraint.cc | 39 +++++++++----- src/core/partitioning/constraint.h | 24 +++++++-- src/core/partitioning/constraint_graph.cc | 26 +++------ src/core/partitioning/constraint_graph.h | 15 +++--- src/core/partitioning/partitioner.cc | 41 +++++++++----- src/core/partitioning/partitioner.h | 11 ++-- src/core/runtime/operation.cc | 65 ++++++++++++----------- src/core/runtime/operation.h | 40 ++++++++------ 8 files changed, 149 insertions(+), 112 deletions(-) diff --git a/src/core/partitioning/constraint.cc b/src/core/partitioning/constraint.cc index f08a344c8a..a6bc75772a 100644 --- a/src/core/partitioning/constraint.cc +++ b/src/core/partitioning/constraint.cc @@ -27,15 +27,15 @@ Literal::Literal(const std::shared_ptr& partition) : partition_(parti std::string Literal::to_string() const { return partition_->to_string(); } +void Literal::find_partition_symbols(std::vector& partition_symbols) const {} + Variable::Variable(const Operation* op, int32_t id) : op_(op), id_(id) {} -bool operator<(const Variable& lhs, const Variable& rhs) +bool operator<(const Variable& lhs, const Variable& rhs) { return lhs.id_ < rhs.id_; } + +void Variable::find_partition_symbols(std::vector& partition_symbols) const { - if (lhs.op_ > rhs.op_) - return false; - else if (lhs.op_ < rhs.op_) - return true; - return lhs.id_ < rhs.id_; + partition_symbols.push_back(this); } std::string Variable::to_string() const @@ -45,21 +45,32 @@ std::string Variable::to_string() const return ss.str(); } +// Constraint AST nodes own their child nodes struct Alignment : public Constraint { public: - Alignment(std::shared_ptr lhs, std::shared_ptr rhs); + Alignment(std::unique_ptr&& lhs, std::unique_ptr&& rhs); + + public: + virtual void find_partition_symbols( + std::vector& partition_symbols) const override; public: virtual std::string to_string() const override; private: - std::shared_ptr lhs_; - std::shared_ptr rhs_; + std::unique_ptr lhs_; + std::unique_ptr rhs_; }; -Alignment::Alignment(std::shared_ptr lhs, std::shared_ptr rhs) - : lhs_(std::move(lhs)), rhs_(std::move(rhs)) +Alignment::Alignment(std::unique_ptr&& lhs, std::unique_ptr&& rhs) + : lhs_(std::forward(lhs)), rhs_(std::forward(rhs)) +{ +} + +void Alignment::find_partition_symbols(std::vector& partition_symbols) const { + lhs_->find_partition_symbols(partition_symbols); + rhs_->find_partition_symbols(partition_symbols); } std::string Alignment::to_string() const @@ -69,9 +80,11 @@ std::string Alignment::to_string() const return ss.str(); } -std::shared_ptr align(std::shared_ptr lhs, std::shared_ptr rhs) +std::unique_ptr align(const Variable* lhs, const Variable* rhs) { - return std::make_shared(std::move(lhs), std::move(rhs)); + // Since an Alignment object owns child nodes, inputs need to be copied + return std::make_unique(std::make_unique(*lhs), + std::make_unique(*rhs)); } } // namespace legate diff --git a/src/core/partitioning/constraint.h b/src/core/partitioning/constraint.h index 120edf7e93..575bdda5ea 100644 --- a/src/core/partitioning/constraint.h +++ b/src/core/partitioning/constraint.h @@ -32,6 +32,9 @@ struct Expr { public: virtual ~Expr() {} + public: + virtual void find_partition_symbols(std::vector& partition_symbols) const = 0; + public: virtual bool closed() const = 0; virtual std::string to_string() const = 0; @@ -46,9 +49,13 @@ struct Literal : public Expr { Literal(const std::shared_ptr& partition); public: - Literal(const Literal&) = default; + Literal(const Literal&) = default; Literal& operator=(const Literal&) = default; + public: + virtual void find_partition_symbols( + std::vector& partition_symbols) const override; + public: virtual bool closed() const override { return true; } virtual std::string to_string() const override; @@ -69,20 +76,24 @@ struct Variable : public Expr { Variable(const Operation* op, int32_t id); public: - Variable(const Variable&) = default; + Variable(const Variable&) = default; Variable& operator=(const Variable&) = default; public: friend bool operator<(const Variable& lhs, const Variable& rhs); public: - virtual const Literal* as_literal() const override { return nullptr; } - virtual const Variable* as_variable() const override { return this; } + virtual void find_partition_symbols( + std::vector& partition_symbols) const override; public: virtual bool closed() const override { return false; } virtual std::string to_string() const override; + public: + virtual const Literal* as_literal() const override { return nullptr; } + virtual const Variable* as_variable() const override { return this; } + public: const Operation* operation() const { return op_; } @@ -95,10 +106,13 @@ struct Constraint { public: virtual ~Constraint() {} + public: + virtual void find_partition_symbols(std::vector& partition_symbols) const = 0; + public: virtual std::string to_string() const = 0; }; -std::shared_ptr align(std::shared_ptr lhs, std::shared_ptr rhs); +std::unique_ptr align(const Variable* lhs, const Variable* rhs); } // namespace legate diff --git a/src/core/partitioning/constraint_graph.cc b/src/core/partitioning/constraint_graph.cc index 4efd2b3c6d..76c06f9506 100644 --- a/src/core/partitioning/constraint_graph.cc +++ b/src/core/partitioning/constraint_graph.cc @@ -25,23 +25,14 @@ namespace legate { extern Legion::Logger log_legate; -void ConstraintGraph::add_variable(std::shared_ptr variable) +void ConstraintGraph::add_partition_symbol(const Variable* partition_symbol) { - variables_.push_back(std::move(variable)); + partition_symbols_.push_back(partition_symbol); } -void ConstraintGraph::add_constraint(std::shared_ptr constraint) +void ConstraintGraph::add_constraint(const Constraint* constraint) { - constraints_.push_back(std::move(constraint)); -} - -void ConstraintGraph::join(const ConstraintGraph& other) -{ - auto& other_variables = other.variables(); - auto& other_constraints = other.constraints(); - - for (auto& other_variable : other_variables) variables_.push_back(other_variable); - for (auto& other_constraint : other_constraints) constraints_.push_back(other_constraint); + constraints_.push_back(constraint); } void ConstraintGraph::dump() @@ -49,14 +40,11 @@ void ConstraintGraph::dump() for (auto& constraint : constraints_) log_legate.debug("%s", constraint->to_string().c_str()); } -const std::vector>& ConstraintGraph::variables() const +const std::vector& ConstraintGraph::partition_symbols() const { - return variables_; + return partition_symbols_; } -const std::vector>& ConstraintGraph::constraints() const -{ - return constraints_; -} +const std::vector& ConstraintGraph::constraints() const { return constraints_; } } // namespace legate diff --git a/src/core/partitioning/constraint_graph.h b/src/core/partitioning/constraint_graph.h index 16c9b1e6a4..ca20bcfeb1 100644 --- a/src/core/partitioning/constraint_graph.h +++ b/src/core/partitioning/constraint_graph.h @@ -27,22 +27,19 @@ struct Variable; struct ConstraintGraph { public: - void add_variable(std::shared_ptr variable); - void add_constraint(std::shared_ptr constraint); - - public: - void join(const ConstraintGraph& other); + void add_partition_symbol(const Variable* partition_symbol); + void add_constraint(const Constraint* constraint); public: void dump(); public: - const std::vector>& variables() const; - const std::vector>& constraints() const; + const std::vector& partition_symbols() const; + const std::vector& constraints() const; private: - std::vector> variables_; - std::vector> constraints_; + std::vector partition_symbols_; + std::vector constraints_; }; } // namespace legate diff --git a/src/core/partitioning/partitioner.cc b/src/core/partitioning/partitioner.cc index ad52925f63..db54dd2372 100644 --- a/src/core/partitioning/partitioner.cc +++ b/src/core/partitioning/partitioner.cc @@ -27,6 +27,10 @@ namespace legate { +//////////////////////////////////////////////////// +// legate::Strategy +//////////////////////////////////////////////////// + Strategy::Strategy() {} bool Strategy::parallel(const Operation* op) const @@ -54,19 +58,32 @@ void Strategy::set_launch_domain(const Operation* op, const Legion::Domain& laun launch_domains_[op] = std::make_unique(launch_domain); } -void Strategy::insert(const Expr* variable, std::shared_ptr partition) +void Strategy::insert(const Variable* partition_symbol, std::shared_ptr partition) { - assert(assignments_.find(variable) == assignments_.end()); - assignments_[variable] = std::move(partition); +#ifdef DEBUG_LEGATE + assert(assignments_.find(*partition_symbol) == assignments_.end()); +#endif + assignments_.insert({*partition_symbol, std::move(partition)}); } -std::shared_ptr Strategy::operator[](const std::shared_ptr& variable) const +bool Strategy::has_assignment(const Variable* partition_symbol) const { - auto finder = assignments_.find(variable.get()); + return assignments_.find(*partition_symbol) != assignments_.end(); +} + +std::shared_ptr Strategy::operator[](const Variable* partition_symbol) const +{ + auto finder = assignments_.find(*partition_symbol); +#ifdef DEBUG_LEGATE assert(finder != assignments_.end()); +#endif return finder->second; } +//////////////////////////////////////////////////// +// legate::Partitioner +//////////////////////////////////////////////////// + Partitioner::Partitioner(std::vector&& operations) : operations_(std::forward>(operations)) { @@ -76,7 +93,7 @@ std::unique_ptr Partitioner::solve() { ConstraintGraph constraints; - for (auto op : operations_) constraints.join(*op->constraints()); + for (auto op : operations_) op->add_to_constraint_graph(constraints); constraints.dump(); @@ -99,12 +116,12 @@ std::unique_ptr Partitioner::solve() // } //} - auto strategy = std::make_unique(); - auto& variables = constraints.variables(); + auto strategy = std::make_unique(); + auto& partition_symbols = constraints.partition_symbols(); - for (auto& variable : variables) { - auto* op = variable->operation(); - auto store = op->find_store(variable); + for (auto& part_symb : partition_symbols) { + auto* op = part_symb->operation(); + auto store = op->find_store(part_symb); auto partition = store->find_or_create_key_partition(); if (!strategy->has_launch_domain(op)) { if (partition->has_launch_domain()) @@ -112,7 +129,7 @@ std::unique_ptr Partitioner::solve() else strategy->set_single_launch(op); } - strategy->insert(variable.get(), std::move(partition)); + strategy->insert(part_symb, std::move(partition)); } return std::move(strategy); diff --git a/src/core/partitioning/partitioner.h b/src/core/partitioning/partitioner.h index 3f4c88d437..e83df3cd9d 100644 --- a/src/core/partitioning/partitioner.h +++ b/src/core/partitioning/partitioner.h @@ -19,11 +19,11 @@ #include #include +#include "core/partitioning/constraint.h" #include "legion.h" namespace legate { -class Expr; class LogicalStore; class Operation; class Partition; @@ -41,12 +41,13 @@ class Strategy { void set_launch_domain(const Operation* op, const Legion::Domain& launch_domain); public: - void insert(const Expr* variable, std::shared_ptr partition); - std::shared_ptr operator[](const std::shared_ptr& variable) const; + void insert(const Variable* partition_symbol, std::shared_ptr partition); + bool has_assignment(const Variable* partition_symbol) const; + std::shared_ptr operator[](const Variable* partition_symbol) const; private: - std::unordered_map> assignments_; - std::unordered_map> launch_domains_; + std::map> assignments_{}; + std::map> launch_domains_{}; }; class Partitioner { diff --git a/src/core/runtime/operation.cc b/src/core/runtime/operation.cc index 4ec6779530..35805bfd0f 100644 --- a/src/core/runtime/operation.cc +++ b/src/core/runtime/operation.cc @@ -31,66 +31,67 @@ namespace legate { Operation::Operation(LibraryContext* library, uint64_t unique_id, int64_t mapper_id) - : library_(library), - unique_id_(unique_id), - mapper_id_(mapper_id), - constraints_(std::make_shared()) + : library_(library), unique_id_(unique_id), mapper_id_(mapper_id) { } Operation::~Operation() {} -void Operation::add_input(LogicalStore store, std::shared_ptr partition) +void Operation::add_input(LogicalStore store, const Variable* partition_symbol) { - auto p_store = store.impl(); - constraints_->add_variable(partition); - inputs_.push_back(Store(p_store.get(), std::move(partition))); - all_stores_.insert(std::move(p_store)); + add_store(inputs_, store, partition_symbol); } -void Operation::add_output(LogicalStore store, std::shared_ptr partition) +void Operation::add_output(LogicalStore store, const Variable* partition_symbol) { - auto p_store = store.impl(); - constraints_->add_variable(partition); - outputs_.push_back(Store(p_store.get(), std::move(partition))); - all_stores_.insert(std::move(p_store)); + add_store(outputs_, store, partition_symbol); } void Operation::add_reduction(LogicalStore store, Legion::ReductionOpID redop, - std::shared_ptr partition) + const Variable* partition_symbol) { - auto p_store = store.impl(); - constraints_->add_variable(partition); - reductions_.push_back(Store(p_store.get(), std::move(partition))); + add_store(reductions_, store, partition_symbol); reduction_ops_.push_back(redop); - all_stores_.insert(std::move(p_store)); } -std::shared_ptr Operation::declare_partition(LogicalStore store) +void Operation::add_store(std::vector& store_args, + LogicalStore& store, + const Variable* partition_symbol) { - // TODO: Variable doesn't need a store to be created, so there's some redundancy in this function. - // Will clean it up once the refactoring for logical store is done - auto p_store = store.impl(); - auto variable = std::make_shared(this, next_part_id_++); - store_mappings_.emplace(std::make_pair(variable, p_store.get())); - all_stores_.insert(std::move(p_store)); - return std::move(variable); + auto store_impl = store.impl(); + store_args.push_back(StoreArg(store_impl.get(), partition_symbol)); + store_mappings_[*partition_symbol] = store_impl.get(); + all_stores_.insert(std::move(store_impl)); } -detail::LogicalStore* Operation::find_store(std::shared_ptr variable) const +const Variable* Operation::declare_partition() { - auto finder = store_mappings_.find(variable); + partition_symbols_.emplace_back(new Variable(this, next_part_id_++)); + return partition_symbols_.back().get(); +} + +detail::LogicalStore* Operation::find_store(const Variable* part_symb) const +{ + auto finder = store_mappings_.find(*part_symb); +#ifdef DEBUG_LEGATE assert(store_mappings_.end() != finder); +#endif return finder->second; } -void Operation::add_constraint(std::shared_ptr constraint) +void Operation::add_constraint(std::unique_ptr constraint) { - constraints_->add_constraint(constraint); + constraints_.push_back(std::move(constraint)); } -std::shared_ptr Operation::constraints() const { return constraints_; } +void Operation::add_to_constraint_graph(ConstraintGraph& constraint_graph) const +{ + for (auto& pair : inputs_) constraint_graph.add_partition_symbol(pair.second); + for (auto& pair : outputs_) constraint_graph.add_partition_symbol(pair.second); + for (auto& pair : reductions_) constraint_graph.add_partition_symbol(pair.second); + for (auto& constraint : constraints_) constraint_graph.add_constraint(constraint.get()); +} Task::Task(LibraryContext* library, int64_t task_id, uint64_t unique_id, int64_t mapper_id /*=0*/) : Operation(library, unique_id, mapper_id), task_id_(task_id) diff --git a/src/core/runtime/operation.h b/src/core/runtime/operation.h index aff662363d..006ea5f188 100644 --- a/src/core/runtime/operation.h +++ b/src/core/runtime/operation.h @@ -20,6 +20,7 @@ #include #include "core/data/logical_store.h" +#include "core/partitioning/constraint.h" #include "legion.h" namespace legate { @@ -29,7 +30,6 @@ class ConstraintGraph; class LibraryContext; class Scalar; class Strategy; -class Variable; namespace detail { @@ -38,6 +38,9 @@ class LogicalStore; } // namespace detail class Operation { + protected: + using StoreArg = std::pair; + public: Operation(LibraryContext* library, uint64_t unique_id, int64_t mapper_id); @@ -45,17 +48,22 @@ class Operation { virtual ~Operation(); public: - void add_input(LogicalStore store, std::shared_ptr partition); - void add_output(LogicalStore store, std::shared_ptr partition); + void add_input(LogicalStore store, const Variable* partition_symbol); + void add_output(LogicalStore store, const Variable* partition_symbol); void add_reduction(LogicalStore store, Legion::ReductionOpID redop, - std::shared_ptr partition); + const Variable* partition_symbol); + + private: + void add_store(std::vector& store_args, + LogicalStore& store, + const Variable* partition_symbol); public: - std::shared_ptr declare_partition(LogicalStore store); - detail::LogicalStore* find_store(std::shared_ptr variable) const; - void add_constraint(std::shared_ptr constraint); - std::shared_ptr constraints() const; + const Variable* declare_partition(); + detail::LogicalStore* find_store(const Variable* variable) const; + void add_constraint(std::unique_ptr constraint); + void add_to_constraint_graph(ConstraintGraph& constraint_graph) const; public: virtual void launch(Strategy* strategy) = 0; @@ -69,21 +77,19 @@ class Operation { int64_t mapper_id_; protected: - using Store = std::pair>; - - protected: - std::set> all_stores_; - std::vector inputs_{}; - std::vector outputs_{}; - std::vector reductions_{}; + std::set> all_stores_{}; + std::vector inputs_{}; + std::vector outputs_{}; + std::vector reductions_{}; std::vector reduction_ops_{}; private: uint32_t next_part_id_{0}; + std::vector> partition_symbols_{}; private: - std::unordered_map, detail::LogicalStore*> store_mappings_; - std::shared_ptr constraints_; + std::map store_mappings_{}; + std::vector> constraints_{}; }; class Task : public Operation { From eb59ca6ce97f21c69f75ffff8df4b3efb2005df1 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 10 Aug 2022 21:39:21 -0700 Subject: [PATCH 0075/1425] Start using ordered set to deduplicate partition symbols --- src/core/partitioning/constraint_graph.cc | 12 ++++-- src/core/partitioning/constraint_graph.h | 3 +- src/core/partitioning/partitioner.cc | 2 + src/core/utilities/ordered_set.h | 49 +++++++++++++++++++++++ 4 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 src/core/utilities/ordered_set.h diff --git a/src/core/partitioning/constraint_graph.cc b/src/core/partitioning/constraint_graph.cc index 76c06f9506..1f37aa7604 100644 --- a/src/core/partitioning/constraint_graph.cc +++ b/src/core/partitioning/constraint_graph.cc @@ -27,7 +27,7 @@ extern Legion::Logger log_legate; void ConstraintGraph::add_partition_symbol(const Variable* partition_symbol) { - partition_symbols_.push_back(partition_symbol); + partition_symbols_.insert(partition_symbol); } void ConstraintGraph::add_constraint(const Constraint* constraint) @@ -37,12 +37,18 @@ void ConstraintGraph::add_constraint(const Constraint* constraint) void ConstraintGraph::dump() { - for (auto& constraint : constraints_) log_legate.debug("%s", constraint->to_string().c_str()); + log_legate.debug("===== Constraint Graph ====="); + log_legate.debug() << "Variables:"; + for (auto& symbol : partition_symbols_.elements()) + log_legate.debug() << " " << symbol->to_string(); + log_legate.debug() << "Constraints:"; + for (auto& constraint : constraints_) log_legate.debug() << " " << constraint->to_string(); + log_legate.debug("============================"); } const std::vector& ConstraintGraph::partition_symbols() const { - return partition_symbols_; + return partition_symbols_.elements(); } const std::vector& ConstraintGraph::constraints() const { return constraints_; } diff --git a/src/core/partitioning/constraint_graph.h b/src/core/partitioning/constraint_graph.h index ca20bcfeb1..fedc8503aa 100644 --- a/src/core/partitioning/constraint_graph.h +++ b/src/core/partitioning/constraint_graph.h @@ -19,6 +19,7 @@ #include #include #include +#include "core/utilities/ordered_set.h" namespace legate { @@ -38,7 +39,7 @@ struct ConstraintGraph { const std::vector& constraints() const; private: - std::vector partition_symbols_; + ordered_set partition_symbols_; std::vector constraints_; }; diff --git a/src/core/partitioning/partitioner.cc b/src/core/partitioning/partitioner.cc index db54dd2372..579a8f622a 100644 --- a/src/core/partitioning/partitioner.cc +++ b/src/core/partitioning/partitioner.cc @@ -95,7 +95,9 @@ std::unique_ptr Partitioner::solve() for (auto op : operations_) op->add_to_constraint_graph(constraints); +#ifdef DEBUG_LEGATE constraints.dump(); +#endif // We need to find a mapping from every partition variable to a concrete partition // Substitution mapping; diff --git a/src/core/utilities/ordered_set.h b/src/core/utilities/ordered_set.h new file mode 100644 index 0000000000..525e34583c --- /dev/null +++ b/src/core/utilities/ordered_set.h @@ -0,0 +1,49 @@ +/* Copyright 2021 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include +#include + +namespace legate { + +// A set implementation that keeps the elements in the order in which they are inserted. +// Internally it uses a vector and a set, the latter of which is used for deduplication. +// This container is not very efficient if used with non-pointer data types + +template +class ordered_set { + public: + ordered_set() {} + + public: + void insert(const T& value) + { + if (element_set_.find(value) != element_set_.end()) return; + elements_.push_back(value); + element_set_.insert(value); + } + + public: + const std::vector& elements() const { return elements_; } + + private: + std::vector elements_{}; + std::unordered_set element_set_{}; +}; + +} // namespace legate From 005be21afd7528c36a8bdb6dbaa670016c69f337 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 10 Aug 2022 22:08:15 -0700 Subject: [PATCH 0076/1425] Sort partition symbols based on their stores' communication volume --- src/core/data/logical_store_detail.cc | 30 ++++++++++++++++++++++++--- src/core/data/logical_store_detail.h | 5 +++++ src/core/partitioning/partitioner.cc | 15 ++++++++++++-- src/core/utilities/tuple.inl | 2 +- 4 files changed, 46 insertions(+), 6 deletions(-) diff --git a/src/core/data/logical_store_detail.cc b/src/core/data/logical_store_detail.cc index 9b16457cc1..04a23a8d56 100644 --- a/src/core/data/logical_store_detail.cc +++ b/src/core/data/logical_store_detail.cc @@ -102,7 +102,7 @@ LogicalStore::LogicalStore(std::shared_ptr&& storage) #ifdef DEBUG_LEGATE assert(transform_ != nullptr); - log_legate.debug() << "Create Store(" << store_id_ << ") {shape: " << extents_ << "}"; + log_legate.debug() << "Create " << to_string(); #endif } @@ -117,8 +117,7 @@ LogicalStore::LogicalStore(tuple&& extents, #ifdef DEBUG_LEGATE assert(transform_ != nullptr); - log_legate.debug() << "Create Store(" << store_id_ << ") {shape: " << extents_ - << ", transform: " << *transform_ << "}"; + log_legate.debug() << "Create " << to_string(); #endif } @@ -131,6 +130,20 @@ const tuple& LogicalStore::extents() const { return extents_; } size_t LogicalStore::volume() const { return extents_.volume(); } +struct elem_size_fn { + template + size_t operator()() + { + return sizeof(legate_type_of); + } +}; + +size_t LogicalStore::storage_size() const +{ + auto elem_size = type_dispatch(code(), elem_size_fn{}); + return storage_->volume() * elem_size; +} + int32_t LogicalStore::dim() const { return static_cast(extents_.size()); } bool LogicalStore::scalar() const { return storage_->kind() == Storage::Kind::FUTURE; } @@ -223,5 +236,16 @@ void LogicalStore::pack(BufferBuilder& buffer) const transform_->pack(buffer); } +std::string LogicalStore::to_string() const +{ + std::stringstream ss; + ss << "Store(" << store_id_ << ") {shape: " << extents_; + if (!transform_->identity()) + ss << ", transform: " << *transform_ << "}"; + else + ss << "}"; + return ss.str(); +} + } // namespace detail } // namespace legate diff --git a/src/core/data/logical_store_detail.h b/src/core/data/logical_store_detail.h index 5b4181fdda..68855ab2a5 100644 --- a/src/core/data/logical_store_detail.h +++ b/src/core/data/logical_store_detail.h @@ -92,6 +92,8 @@ class LogicalStore { public: const tuple& extents() const; size_t volume() const; + // Size of the backing storage + size_t storage_size() const; int32_t dim() const; bool scalar() const; LegateTypeCode code() const; @@ -121,6 +123,9 @@ class LogicalStore { public: void pack(BufferBuilder& buffer) const; + public: + std::string to_string() const; + private: uint64_t store_id_; tuple extents_; diff --git a/src/core/partitioning/partitioner.cc b/src/core/partitioning/partitioner.cc index 579a8f622a..d89e2dba18 100644 --- a/src/core/partitioning/partitioner.cc +++ b/src/core/partitioning/partitioner.cc @@ -118,8 +118,19 @@ std::unique_ptr Partitioner::solve() // } //} - auto strategy = std::make_unique(); - auto& partition_symbols = constraints.partition_symbols(); + auto strategy = std::make_unique(); + + // Copy the list of partition symbols as we will sort them inplace + std::vector partition_symbols(constraints.partition_symbols()); + std::stable_sort(partition_symbols.begin(), + partition_symbols.end(), + [](const auto& part_symb_a, const auto& part_symb_b) { + auto get_storage_size = [](const auto& part_symb) { + auto* op = part_symb->operation(); + return op->find_store(part_symb)->storage_size(); + }; + return get_storage_size(part_symb_a) > get_storage_size(part_symb_b); + }); for (auto& part_symb : partition_symbols) { auto* op = part_symb->operation(); diff --git a/src/core/utilities/tuple.inl b/src/core/utilities/tuple.inl index 059c817961..50ce78635a 100644 --- a/src/core/utilities/tuple.inl +++ b/src/core/utilities/tuple.inl @@ -139,7 +139,7 @@ template T tuple::reduce(FUNC func, const T& init) const { T agg{init}; - for (auto value : data_) func(agg, value); + for (auto value : data_) agg = func(agg, value); return agg; } From ee3950087383c1b81bdde1ae811c4ab352202217 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Thu, 11 Aug 2022 11:19:45 -0700 Subject: [PATCH 0077/1425] Simple constraint solver for alignment constraints --- src/core/partitioning/constraint.cc | 22 ++------- src/core/partitioning/constraint.h | 29 ++++++++++++ src/core/partitioning/constraint_graph.cc | 57 +++++++++++++++++++++++ src/core/partitioning/constraint_graph.h | 32 ++++++++++++- src/core/partitioning/partitioner.cc | 32 +++++-------- 5 files changed, 132 insertions(+), 40 deletions(-) diff --git a/src/core/partitioning/constraint.cc b/src/core/partitioning/constraint.cc index a6bc75772a..7846db8e55 100644 --- a/src/core/partitioning/constraint.cc +++ b/src/core/partitioning/constraint.cc @@ -31,6 +31,11 @@ void Literal::find_partition_symbols(std::vector& partition_sym Variable::Variable(const Operation* op, int32_t id) : op_(op), id_(id) {} +bool operator==(const Variable& lhs, const Variable& rhs) +{ + return lhs.op_ == rhs.op_ && lhs.id_ == rhs.id_; +} + bool operator<(const Variable& lhs, const Variable& rhs) { return lhs.id_ < rhs.id_; } void Variable::find_partition_symbols(std::vector& partition_symbols) const @@ -45,23 +50,6 @@ std::string Variable::to_string() const return ss.str(); } -// Constraint AST nodes own their child nodes -struct Alignment : public Constraint { - public: - Alignment(std::unique_ptr&& lhs, std::unique_ptr&& rhs); - - public: - virtual void find_partition_symbols( - std::vector& partition_symbols) const override; - - public: - virtual std::string to_string() const override; - - private: - std::unique_ptr lhs_; - std::unique_ptr rhs_; -}; - Alignment::Alignment(std::unique_ptr&& lhs, std::unique_ptr&& rhs) : lhs_(std::forward(lhs)), rhs_(std::forward(rhs)) { diff --git a/src/core/partitioning/constraint.h b/src/core/partitioning/constraint.h index 575bdda5ea..290a37bbbb 100644 --- a/src/core/partitioning/constraint.h +++ b/src/core/partitioning/constraint.h @@ -22,6 +22,7 @@ namespace legate { +struct Alignment; struct Constraint; struct Literal; struct Operation; @@ -80,6 +81,7 @@ struct Variable : public Expr { Variable& operator=(const Variable&) = default; public: + friend bool operator==(const Variable& lhs, const Variable& rhs); friend bool operator<(const Variable& lhs, const Variable& rhs); public: @@ -111,6 +113,33 @@ struct Constraint { public: virtual std::string to_string() const = 0; + + public: + virtual const Alignment* as_alignment() const = 0; +}; + +// Constraint AST nodes own their child nodes +struct Alignment : public Constraint { + public: + Alignment(std::unique_ptr&& lhs, std::unique_ptr&& rhs); + + public: + virtual void find_partition_symbols( + std::vector& partition_symbols) const override; + + public: + virtual std::string to_string() const override; + + public: + virtual const Alignment* as_alignment() const override { return this; } + + public: + const Expr* lhs() const { return lhs_.get(); } + const Expr* rhs() const { return rhs_.get(); } + + private: + std::unique_ptr lhs_; + std::unique_ptr rhs_; }; std::unique_ptr align(const Variable* lhs, const Variable* rhs); diff --git a/src/core/partitioning/constraint_graph.cc b/src/core/partitioning/constraint_graph.cc index 1f37aa7604..d86bd63412 100644 --- a/src/core/partitioning/constraint_graph.cc +++ b/src/core/partitioning/constraint_graph.cc @@ -35,6 +35,63 @@ void ConstraintGraph::add_constraint(const Constraint* constraint) constraints_.push_back(constraint); } +void ConstraintGraph::compute_equivalence_classes() +{ + for (auto& part_symb : partition_symbols()) { + all_equiv_class_entries_.emplace_back(new EquivClass(part_symb)); + equiv_classes_.insert({*part_symb, all_equiv_class_entries_.back().get()}); + } + + for (auto& constraint : constraints_) { + auto* alignment = constraint->as_alignment(); + if (nullptr == alignment) continue; +#ifdef DEBUG_LEGATE + assert(alignment->lhs()->as_variable() != nullptr && + alignment->rhs()->as_variable() != nullptr); +#endif + + std::vector part_symbs_to_unify; + alignment->find_partition_symbols(part_symbs_to_unify); +#ifdef DEBUG_LEGATE + assert(!part_symbs_to_unify.empty()); +#endif + + EquivClass* equiv_class = equiv_classes_[*part_symbs_to_unify[0]]; +#ifdef DEBUG_LEGATE + assert(equiv_class != nullptr); +#endif + auto update_table = [&](auto* new_cls, auto* orig_cls) { + while (orig_cls != nullptr) { + equiv_classes_[*orig_cls->partition_symbol] = new_cls; + orig_cls = orig_cls->next; + } + }; + for (size_t idx = 1; idx < part_symbs_to_unify.size(); ++idx) { + auto class_to_unify = equiv_classes_[*part_symbs_to_unify[idx]]; + auto result = equiv_class->unify(class_to_unify); + + if (result != equiv_class) update_table(result, equiv_class); + if (result != class_to_unify) update_table(result, class_to_unify); + equiv_class = result; + } + } +} + +void ConstraintGraph::find_equivalence_class(const Variable* partition_symbol, + std::vector& out_equiv_class) const +{ + auto finder = equiv_classes_.find(*partition_symbol); +#ifdef DEBUG_LEGATE + assert(finder != equiv_classes_.end()); +#endif + auto equiv_class = finder->second; + out_equiv_class.reserve(equiv_class->size); + while (equiv_class != nullptr) { + out_equiv_class.push_back(equiv_class->partition_symbol); + equiv_class = equiv_class->next; + } +} + void ConstraintGraph::dump() { log_legate.debug("===== Constraint Graph ====="); diff --git a/src/core/partitioning/constraint_graph.h b/src/core/partitioning/constraint_graph.h index fedc8503aa..884b4e9d1e 100644 --- a/src/core/partitioning/constraint_graph.h +++ b/src/core/partitioning/constraint_graph.h @@ -16,15 +16,34 @@ #pragma once +#include #include #include #include +#include "core/partitioning/constraint.h" #include "core/utilities/ordered_set.h" namespace legate { -struct Constraint; -struct Variable; +struct EquivClass { + EquivClass(const Variable* symb) : partition_symbol(symb), next(nullptr), size(1) {} + + EquivClass* unify(EquivClass* other) + { + EquivClass* self = this; + if (self->size < other->size) std::swap(self, other); + + auto end = self; + while (end->next != nullptr) end = end->next; + end->next = other; + end->size += other->size; + return self; + } + + const Variable* partition_symbol; + EquivClass* next; + size_t size; +}; struct ConstraintGraph { public: @@ -38,9 +57,18 @@ struct ConstraintGraph { const std::vector& partition_symbols() const; const std::vector& constraints() const; + public: + void compute_equivalence_classes(); + void find_equivalence_class(const Variable* partition_symbol, + std::vector& out_equiv_class) const; + private: ordered_set partition_symbols_; std::vector constraints_; + + private: + std::map equiv_classes_; + std::vector> all_equiv_class_entries_; }; } // namespace legate diff --git a/src/core/partitioning/partitioner.cc b/src/core/partitioning/partitioner.cc index d89e2dba18..fb81ccbad0 100644 --- a/src/core/partitioning/partitioner.cc +++ b/src/core/partitioning/partitioner.cc @@ -95,29 +95,12 @@ std::unique_ptr Partitioner::solve() for (auto op : operations_) op->add_to_constraint_graph(constraints); + constraints.compute_equivalence_classes(); + #ifdef DEBUG_LEGATE constraints.dump(); #endif - // We need to find a mapping from every partition variable to a concrete partition - // Substitution mapping; - // for (auto& expr : all_symbols) { - // auto* var = expr->as_variable(); - // assert(nullptr != sym); - // if (mapping.find(var) != mapping.end()) continue; - - // auto store = store_mapping[expr]; - // auto partition = store->find_or_create_key_partition(); - // mapping[sym] = std::move(partition); - - // std::vector> next_constraints; - // for (auto& constraint : all_constraints) { - // auto substituted = constraint.subst(mapping); - // bool resolved = substituted.resolve(mapping); - // if (!resolved) next_constraints.push_back(std::move(substituted)); - // } - //} - auto strategy = std::make_unique(); // Copy the list of partition symbols as we will sort them inplace @@ -133,16 +116,23 @@ std::unique_ptr Partitioner::solve() }); for (auto& part_symb : partition_symbols) { + if (strategy->has_assignment(part_symb)) continue; + auto* op = part_symb->operation(); auto store = op->find_store(part_symb); - auto partition = store->find_or_create_key_partition(); + auto partition = store->find_or_create_key_partition().release(); if (!strategy->has_launch_domain(op)) { if (partition->has_launch_domain()) strategy->set_launch_domain(op, partition->launch_domain()); else strategy->set_single_launch(op); } - strategy->insert(part_symb, std::move(partition)); + + std::vector equiv_class; + constraints.find_equivalence_class(part_symb, equiv_class); + + std::shared_ptr solution(partition); + for (auto symb : equiv_class) strategy->insert(symb, solution); } return std::move(strategy); From a59738e0fc8f2d153762ba334f349b9155edf56e Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Thu, 11 Aug 2022 13:52:49 -0700 Subject: [PATCH 0078/1425] Make sure key partitions of a store and its views are all aligned --- src/core/data/logical_store_detail.cc | 35 +++++++++++++++------- src/core/data/logical_store_detail.h | 7 +++-- src/core/data/transform.cc | 42 +++++++++++++++++++++++++++ src/core/data/transform.h | 7 +++++ src/core/partitioning/partition.cc | 8 +++-- src/core/partitioning/partition.h | 12 ++++++++ src/core/partitioning/partitioner.cc | 5 ++-- 7 files changed, 99 insertions(+), 17 deletions(-) diff --git a/src/core/data/logical_store_detail.cc b/src/core/data/logical_store_detail.cc index 04a23a8d56..473e626936 100644 --- a/src/core/data/logical_store_detail.cc +++ b/src/core/data/logical_store_detail.cc @@ -70,7 +70,21 @@ RegionField Storage::map(LibraryContext* context) return Runtime::get_runtime()->map_region_field(context, region_field_.get()); } -Partition* Storage::find_key_partition() const { return key_partition_.get(); } +Partition* Storage::find_or_create_key_partition() +{ + if (key_partition_ != nullptr) return key_partition_.get(); + + auto part_mgr = Runtime::get_runtime()->partition_manager(); + auto launch_shape = part_mgr->compute_launch_shape(extents_); + if (launch_shape.empty()) + key_partition_ = create_no_partition(); + else { + auto tile_shape = part_mgr->compute_tile_shape(extents_, launch_shape); + key_partition_ = create_tiling(std::move(tile_shape), std::move(launch_shape)); + } + + return key_partition_.get(); +} void Storage::set_key_partition(std::unique_ptr&& key_partition) { @@ -213,18 +227,19 @@ std::unique_ptr LogicalStore::create_projection(const Partition* par return std::make_unique(legion_partition, proj_id); } -std::unique_ptr LogicalStore::find_or_create_key_partition() +std::shared_ptr LogicalStore::find_or_create_key_partition() { - if (scalar()) return create_no_partition(); + if (key_partition_ != nullptr) return key_partition_; - auto part_mgr = Runtime::get_runtime()->partition_manager(); - auto launch_shape = part_mgr->compute_launch_shape(extents_); - if (launch_shape.empty()) - return create_no_partition(); - else { - auto tile_shape = part_mgr->compute_tile_shape(extents_, launch_shape); - return create_tiling(std::move(tile_shape), std::move(launch_shape)); + if (scalar()) { + key_partition_ = create_no_partition(); + return key_partition_; } + + Partition* storage_part = storage_->find_or_create_key_partition(); + auto store_part = transform_->convert(storage_part); + key_partition_ = std::move(store_part); + return key_partition_; } void LogicalStore::pack(BufferBuilder& buffer) const diff --git a/src/core/data/logical_store_detail.h b/src/core/data/logical_store_detail.h index 68855ab2a5..afb987a408 100644 --- a/src/core/data/logical_store_detail.h +++ b/src/core/data/logical_store_detail.h @@ -51,7 +51,7 @@ class Storage { RegionField map(LibraryContext* context); public: - Partition* find_key_partition() const; + Partition* find_or_create_key_partition(); void set_key_partition(std::unique_ptr&& key_partition); void reset_key_partition(); Legion::LogicalPartition find_or_create_legion_partition(const Partition* partition); @@ -115,7 +115,7 @@ class LogicalStore { public: std::unique_ptr create_projection(const Partition* partition); - std::unique_ptr find_or_create_key_partition(); + std::shared_ptr find_or_create_key_partition(); private: Legion::ProjectionID compute_projection() const; @@ -131,6 +131,9 @@ class LogicalStore { tuple extents_; std::shared_ptr storage_; std::shared_ptr transform_; + + private: + std::shared_ptr key_partition_; std::shared_ptr mapped_{nullptr}; }; diff --git a/src/core/data/transform.cc b/src/core/data/transform.cc index f1c189e550..6f1fa6db03 100644 --- a/src/core/data/transform.cc +++ b/src/core/data/transform.cc @@ -46,8 +46,22 @@ TransformStack::TransformStack(std::unique_ptr&& transform, { } +std::unique_ptr TransformStack::convert(const Partition* partition) const +{ + if (identity()) return partition->clone(); + + if (parent_->identity()) + return transform_->convert(partition); + else { + auto result = parent_->convert(partition); + return transform_->convert(result.get()); + } +} + std::unique_ptr TransformStack::invert(const Partition* partition) const { + if (identity()) return partition->clone(); + auto result = transform_->invert(partition); return parent_->identity() ? std::move(result) : parent_->invert(result.get()); } @@ -157,6 +171,8 @@ DomainAffineTransform Shift::inverse_transform(int32_t in_dim) const return result; } +std::unique_ptr Shift::convert(const Partition* partition) const { return nullptr; } + std::unique_ptr Shift::invert(const Partition* partition) const { return nullptr; } // the shift transform makes no change on the store's dimensions @@ -224,6 +240,23 @@ DomainAffineTransform Promote::inverse_transform(int32_t in_dim) const return result; } +std::unique_ptr Promote::convert(const Partition* partition) const +{ + switch (partition->kind()) { + case Partition::Kind::NO_PARTITION: { + return create_no_partition(); + } + case Partition::Kind::TILING: { + auto tiling = static_cast(partition); + return create_tiling(tiling->tile_shape().insert(extra_dim_, dim_size_), + tiling->color_shape().insert(extra_dim_, 1), + tiling->offsets().insert(extra_dim_, 0)); + } + } + assert(false); + return nullptr; +} + std::unique_ptr Promote::invert(const Partition* partition) const { switch (partition->kind()) { @@ -307,6 +340,8 @@ DomainAffineTransform Project::inverse_transform(int32_t in_dim) const return result; } +std::unique_ptr Project::convert(const Partition* partition) const { return nullptr; } + std::unique_ptr Project::invert(const Partition* partition) const { return nullptr; } proj::SymbolicPoint Project::invert(const proj::SymbolicPoint& point) const @@ -371,6 +406,8 @@ DomainAffineTransform Transpose::inverse_transform(int32_t in_dim) const return result; } +std::unique_ptr Transpose::convert(const Partition* partition) const { return nullptr; } + std::unique_ptr Transpose::invert(const Partition* partition) const { return nullptr; } proj::SymbolicPoint Transpose::invert(const proj::SymbolicPoint& point) const @@ -480,6 +517,11 @@ DomainAffineTransform Delinearize::inverse_transform(int32_t in_dim) const return result; } +std::unique_ptr Delinearize::convert(const Partition* partition) const +{ + return nullptr; +} + std::unique_ptr Delinearize::invert(const Partition* partition) const { return nullptr; } proj::SymbolicPoint Delinearize::invert(const proj::SymbolicPoint& point) const diff --git a/src/core/data/transform.h b/src/core/data/transform.h index f88e86cbc6..fb0e17c44f 100644 --- a/src/core/data/transform.h +++ b/src/core/data/transform.h @@ -29,6 +29,7 @@ class Partition; struct Transform { virtual Legion::Domain transform(const Legion::Domain& input) const = 0; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const = 0; + virtual std::unique_ptr convert(const Partition* partition) const = 0; virtual std::unique_ptr invert(const Partition* partition) const = 0; virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const = 0; virtual void pack(BufferBuilder& buffer) const = 0; @@ -51,6 +52,7 @@ struct TransformStack : public Transform, std::enable_shared_from_this convert(const Partition* partition) const override; virtual std::unique_ptr invert(const Partition* partition) const override; virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; virtual void pack(BufferBuilder& buffer) const override; @@ -76,6 +78,7 @@ class Shift : public StoreTransform { public: virtual Legion::Domain transform(const Legion::Domain& input) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; + virtual std::unique_ptr convert(const Partition* partition) const override; virtual std::unique_ptr invert(const Partition* partition) const override; virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; virtual void pack(BufferBuilder& buffer) const override; @@ -96,6 +99,7 @@ class Promote : public StoreTransform { public: virtual Legion::Domain transform(const Legion::Domain& input) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; + virtual std::unique_ptr convert(const Partition* partition) const override; virtual std::unique_ptr invert(const Partition* partition) const override; virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; virtual void pack(BufferBuilder& buffer) const override; @@ -117,6 +121,7 @@ class Project : public StoreTransform { public: virtual Legion::Domain transform(const Legion::Domain& domain) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; + virtual std::unique_ptr convert(const Partition* partition) const override; virtual std::unique_ptr invert(const Partition* partition) const override; virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; virtual void pack(BufferBuilder& buffer) const override; @@ -137,6 +142,7 @@ class Transpose : public StoreTransform { public: virtual Legion::Domain transform(const Legion::Domain& domain) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; + virtual std::unique_ptr convert(const Partition* partition) const override; virtual std::unique_ptr invert(const Partition* partition) const override; virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; virtual void pack(BufferBuilder& buffer) const override; @@ -157,6 +163,7 @@ class Delinearize : public StoreTransform { public: virtual Legion::Domain transform(const Legion::Domain& domain) const override; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; + virtual std::unique_ptr convert(const Partition* partition) const override; virtual std::unique_ptr invert(const Partition* partition) const override; virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; virtual void pack(BufferBuilder& buffer) const override; diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index fb0fad5280..67248ba141 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -50,6 +50,8 @@ Legion::Domain NoPartition::launch_domain() const return Legion::Domain(); } +std::unique_ptr NoPartition::clone() const { return create_no_partition(); } + std::string NoPartition::to_string() const { return "NoPartition"; } Tiling::Tiling(Shape&& tile_shape, Shape&& color_shape, Shape&& offsets) @@ -152,6 +154,8 @@ Legion::Domain Tiling::launch_domain() const return launch_domain; } +std::unique_ptr Tiling::clone() const { return std::make_unique(*this); } + std::string Tiling::to_string() const { std::stringstream ss; @@ -160,6 +164,8 @@ std::string Tiling::to_string() const return ss.str(); } +std::unique_ptr create_no_partition() { return std::make_unique(); } + std::unique_ptr create_tiling(Shape&& tile_shape, Shape&& color_shape, Shape&& offsets /*= {}*/) @@ -169,8 +175,6 @@ std::unique_ptr create_tiling(Shape&& tile_shape, std::forward(offsets)); } -std::unique_ptr create_no_partition() { return std::make_unique(); } - std::ostream& operator<<(std::ostream& out, const Partition& partition) { out << partition.to_string(); diff --git a/src/core/partitioning/partition.h b/src/core/partitioning/partition.h index 64965288eb..163eb8c709 100644 --- a/src/core/partitioning/partition.h +++ b/src/core/partitioning/partition.h @@ -61,6 +61,9 @@ struct Partition { virtual bool has_launch_domain() const = 0; virtual Legion::Domain launch_domain() const = 0; + public: + virtual std::unique_ptr clone() const = 0; + public: virtual std::string to_string() const = 0; }; @@ -86,6 +89,9 @@ class NoPartition : public Partition { virtual bool has_launch_domain() const override; virtual Legion::Domain launch_domain() const override; + public: + virtual std::unique_ptr clone() const override; + public: virtual std::string to_string() const override; }; @@ -94,6 +100,9 @@ class Tiling : public Partition { public: Tiling(Shape&& tile_shape, Shape&& color_shape, Shape&& offsets); + public: + Tiling(const Tiling&) = default; + public: bool operator==(const Tiling& other) const; bool operator<(const Tiling& other) const; @@ -115,6 +124,9 @@ class Tiling : public Partition { virtual bool has_launch_domain() const override; virtual Legion::Domain launch_domain() const override; + public: + virtual std::unique_ptr clone() const override; + public: virtual std::string to_string() const override; diff --git a/src/core/partitioning/partitioner.cc b/src/core/partitioning/partitioner.cc index fb81ccbad0..6aeed10c6c 100644 --- a/src/core/partitioning/partitioner.cc +++ b/src/core/partitioning/partitioner.cc @@ -120,7 +120,7 @@ std::unique_ptr Partitioner::solve() auto* op = part_symb->operation(); auto store = op->find_store(part_symb); - auto partition = store->find_or_create_key_partition().release(); + auto partition = store->find_or_create_key_partition(); if (!strategy->has_launch_domain(op)) { if (partition->has_launch_domain()) strategy->set_launch_domain(op, partition->launch_domain()); @@ -131,8 +131,7 @@ std::unique_ptr Partitioner::solve() std::vector equiv_class; constraints.find_equivalence_class(part_symb, equiv_class); - std::shared_ptr solution(partition); - for (auto symb : equiv_class) strategy->insert(symb, solution); + for (auto symb : equiv_class) strategy->insert(symb, partition); } return std::move(strategy); From bb2b917241da00157aea69bc2f6661833b9b44a5 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Thu, 11 Aug 2022 14:06:47 -0700 Subject: [PATCH 0079/1425] Set key partitions for task outputs --- src/core/data/logical_store_detail.cc | 8 ++++++++ src/core/data/logical_store_detail.h | 2 ++ src/core/partitioning/partitioner.cc | 2 +- src/core/partitioning/partitioner.h | 2 +- src/core/runtime/operation.cc | 8 ++++++-- 5 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/core/data/logical_store_detail.cc b/src/core/data/logical_store_detail.cc index 473e626936..f3037422e6 100644 --- a/src/core/data/logical_store_detail.cc +++ b/src/core/data/logical_store_detail.cc @@ -242,6 +242,14 @@ std::shared_ptr LogicalStore::find_or_create_key_partition() return key_partition_; } +void LogicalStore::set_key_partition(const Partition* partition) +{ + auto inverted = transform_->invert(partition); + storage_->set_key_partition(std::move(inverted)); +} + +void LogicalStore::reset_key_partition() { storage_->reset_key_partition(); } + void LogicalStore::pack(BufferBuilder& buffer) const { buffer.pack(scalar()); diff --git a/src/core/data/logical_store_detail.h b/src/core/data/logical_store_detail.h index afb987a408..1ac8c443fa 100644 --- a/src/core/data/logical_store_detail.h +++ b/src/core/data/logical_store_detail.h @@ -116,6 +116,8 @@ class LogicalStore { public: std::unique_ptr create_projection(const Partition* partition); std::shared_ptr find_or_create_key_partition(); + void set_key_partition(const Partition* partition); + void reset_key_partition(); private: Legion::ProjectionID compute_projection() const; diff --git a/src/core/partitioning/partitioner.cc b/src/core/partitioning/partitioner.cc index 6aeed10c6c..a7fe1b2573 100644 --- a/src/core/partitioning/partitioner.cc +++ b/src/core/partitioning/partitioner.cc @@ -71,7 +71,7 @@ bool Strategy::has_assignment(const Variable* partition_symbol) const return assignments_.find(*partition_symbol) != assignments_.end(); } -std::shared_ptr Strategy::operator[](const Variable* partition_symbol) const +const std::shared_ptr& Strategy::operator[](const Variable* partition_symbol) const { auto finder = assignments_.find(*partition_symbol); #ifdef DEBUG_LEGATE diff --git a/src/core/partitioning/partitioner.h b/src/core/partitioning/partitioner.h index e83df3cd9d..530136b31d 100644 --- a/src/core/partitioning/partitioner.h +++ b/src/core/partitioning/partitioner.h @@ -43,7 +43,7 @@ class Strategy { public: void insert(const Variable* partition_symbol, std::shared_ptr partition); bool has_assignment(const Variable* partition_symbol) const; - std::shared_ptr operator[](const Variable* partition_symbol) const; + const std::shared_ptr& operator[](const Variable* partition_symbol) const; private: std::map> assignments_{}; diff --git a/src/core/runtime/operation.cc b/src/core/runtime/operation.cc index 35805bfd0f..3252b6e38c 100644 --- a/src/core/runtime/operation.cc +++ b/src/core/runtime/operation.cc @@ -14,9 +14,12 @@ * */ +#include "core/runtime/operation.h" + #include #include +#include "core/data/logical_store_detail.h" #include "core/data/scalar.h" #include "core/partitioning/constraint.h" #include "core/partitioning/constraint_graph.h" @@ -24,7 +27,6 @@ #include "core/partitioning/partitioner.h" #include "core/runtime/context.h" #include "core/runtime/launcher.h" -#include "core/runtime/operation.h" #include "core/runtime/req_analyzer.h" #include "core/runtime/runtime.h" @@ -116,8 +118,10 @@ void Task::launch(Strategy* p_strategy) for (auto& pair : outputs_) { auto& store = pair.first; auto& var = pair.second; - auto proj = strategy[var]->get_projection(store); + auto part = strategy[var]; + auto proj = part->get_projection(store); launcher.add_output(store, std::move(proj)); + store->set_key_partition(part.get()); } uint32_t idx = 0; for (auto& pair : reductions_) { From a62d19882c4f2b7b5f4dc18287c38e30ef1abe6f Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 12 Aug 2022 10:22:57 -0700 Subject: [PATCH 0080/1425] Implement projection transformation --- src/core/data/logical_store.cc | 7 +++++- src/core/data/logical_store.h | 1 + src/core/data/logical_store_detail.cc | 18 +++++++++++--- src/core/data/logical_store_detail.h | 5 ++-- src/core/data/transform.cc | 34 +++++++++++++++++++++++++-- 5 files changed, 56 insertions(+), 9 deletions(-) diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index ca16636c01..0464de67c5 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -58,7 +58,12 @@ size_t LogicalStore::volume() const { return impl_->volume(); } LogicalStore LogicalStore::promote(int32_t extra_dim, size_t dim_size) const { - return LogicalStore(impl_->promote(extra_dim, dim_size, impl_)); + return LogicalStore(impl_->promote(extra_dim, dim_size)); +} + +LogicalStore LogicalStore::project(int32_t dim, int64_t index) const +{ + return LogicalStore(impl_->project(dim, index)); } std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index d7da902576..5550597597 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -60,6 +60,7 @@ class LogicalStore { public: LogicalStore promote(int32_t extra_dim, size_t dim_size) const; + LogicalStore project(int32_t dim, int64_t index) const; public: std::shared_ptr get_physical_store(LibraryContext* context); diff --git a/src/core/data/logical_store_detail.cc b/src/core/data/logical_store_detail.cc index f3037422e6..0633f32327 100644 --- a/src/core/data/logical_store_detail.cc +++ b/src/core/data/logical_store_detail.cc @@ -168,9 +168,7 @@ LogicalRegionField* LogicalStore::get_region_field() { return storage_->get_regi Legion::Future LogicalStore::get_future() { return storage_->get_future(); } -std::shared_ptr LogicalStore::promote(int32_t extra_dim, - size_t dim_size, - std::shared_ptr parent) const +std::shared_ptr LogicalStore::promote(int32_t extra_dim, size_t dim_size) const { if (extra_dim < 0 || extra_dim > dim()) { log_legate.error("Invalid promotion on dimension %d for a %d-D store", extra_dim, dim()); @@ -182,6 +180,20 @@ std::shared_ptr LogicalStore::promote(int32_t extra_dim, return std::make_shared(std::move(new_extents), storage_, std::move(transform)); } +std::shared_ptr LogicalStore::project(int32_t d, int64_t index) const +{ + if (d < 0 || d >= dim()) { + log_legate.error("Invalid projection on dimension %d for a %d-D store", d, dim()); + LEGATE_ABORT; + } else if (index < 0 || index >= extents_[d]) { + log_legate.error("Projection index %ld is out of bounds [0, %zd)", index, extents_[d]); + } + + auto new_extents = extents_.remove(d); + auto transform = transform_->push(std::make_unique(d, index)); + return std::make_shared(std::move(new_extents), storage_, std::move(transform)); +} + std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) { // TODO: Need to support inline mapping for scalars diff --git a/src/core/data/logical_store_detail.h b/src/core/data/logical_store_detail.h index 1ac8c443fa..06286ec484 100644 --- a/src/core/data/logical_store_detail.h +++ b/src/core/data/logical_store_detail.h @@ -106,9 +106,8 @@ class LogicalStore { void create_storage(); public: - std::shared_ptr promote(int32_t extra_dim, - size_t dim_size, - std::shared_ptr parent) const; + std::shared_ptr promote(int32_t extra_dim, size_t dim_size) const; + std::shared_ptr project(int32_t dim, int64_t index) const; public: std::shared_ptr get_physical_store(LibraryContext* context); diff --git a/src/core/data/transform.cc b/src/core/data/transform.cc index 6f1fa6db03..639137a7ff 100644 --- a/src/core/data/transform.cc +++ b/src/core/data/transform.cc @@ -340,9 +340,39 @@ DomainAffineTransform Project::inverse_transform(int32_t in_dim) const return result; } -std::unique_ptr Project::convert(const Partition* partition) const { return nullptr; } +std::unique_ptr Project::convert(const Partition* partition) const +{ + switch (partition->kind()) { + case Partition::Kind::NO_PARTITION: { + return create_no_partition(); + } + case Partition::Kind::TILING: { + auto tiling = static_cast(partition); + return create_tiling(tiling->tile_shape().remove(dim_), + tiling->color_shape().remove(dim_), + tiling->offsets().remove(dim_)); + } + } + assert(false); + return nullptr; +} -std::unique_ptr Project::invert(const Partition* partition) const { return nullptr; } +std::unique_ptr Project::invert(const Partition* partition) const +{ + switch (partition->kind()) { + case Partition::Kind::NO_PARTITION: { + return create_no_partition(); + } + case Partition::Kind::TILING: { + auto tiling = static_cast(partition); + return create_tiling(tiling->tile_shape().insert(dim_, 1), + tiling->color_shape().insert(dim_, 1), + tiling->offsets().insert(dim_, coord_)); + } + } + assert(false); + return nullptr; +} proj::SymbolicPoint Project::invert(const proj::SymbolicPoint& point) const { From dd445cc380879095300ce89f1d76ea150ced9d78 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 12 Aug 2022 10:50:22 -0700 Subject: [PATCH 0081/1425] Add a mode that generates an executable to legate.mk --- src/legate.mk | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/legate.mk b/src/legate.mk index 047d05a6f0..e226693ed6 100644 --- a/src/legate.mk +++ b/src/legate.mk @@ -223,6 +223,14 @@ endif CC_FLAGS += -fPIC NVCC_FLAGS += --compiler-options '-fPIC' +ifeq ($(strip $(BUILD_EXECUTABLE)),1) +ifeq ($(shell uname), Darwin) + DLIB = $(LIBNAME) + LD_FLAGS += -fPIC +else + DLIB = $(LIBNAME) +endif +else ifeq ($(shell uname), Darwin) DLIB = $(LIBNAME).dylib LD_FLAGS += -dynamiclib -fPIC -install_name @rpath/$(DLIB) @@ -230,6 +238,7 @@ else DLIB = $(LIBNAME).so LD_FLAGS += -shared endif +endif OMP_FLAGS ?= ifeq ($(strip $(USE_OPENMP)),1) From 84d29b33aa73c12b47c8e8b8d9108763daa24d29 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 12 Aug 2022 10:56:09 -0700 Subject: [PATCH 0082/1425] Make requirement analyzer handle cases where coalesced requirements use promoted privilege --- src/core/runtime/req_analyzer.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/runtime/req_analyzer.cc b/src/core/runtime/req_analyzer.cc index f329e29657..3a358d2968 100644 --- a/src/core/runtime/req_analyzer.cc +++ b/src/core/runtime/req_analyzer.cc @@ -156,6 +156,7 @@ uint32_t FieldSet::get_requirement_index(Legion::PrivilegeMode privilege, const ProjectionInfo* proj_info) const { auto finder = req_indices_.find(Key(privilege, *proj_info)); + if (req_indices_.end() == finder) finder = req_indices_.find(Key(LEGION_READ_WRITE, *proj_info)); #ifdef DEBUG_LEGATE assert(finder != req_indices_.end()); #endif From e20ab92c01972ef12a27857cf8298e765f8cf7ce Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Tue, 23 Aug 2022 14:50:10 -0700 Subject: [PATCH 0083/1425] Remove dead code and stop using a deprecated Legion API --- src/core/data/logical_store_detail.h | 3 --- src/core/runtime/runtime.cc | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/core/data/logical_store_detail.h b/src/core/data/logical_store_detail.h index 06286ec484..dc4d13974e 100644 --- a/src/core/data/logical_store_detail.h +++ b/src/core/data/logical_store_detail.h @@ -102,9 +102,6 @@ class LogicalStore { LogicalRegionField* get_region_field(); Legion::Future get_future(); - private: - void create_storage(); - public: std::shared_ptr promote(int32_t extra_dim, size_t dim_size) const; std::shared_ptr project(int32_t dim, int64_t index) const; diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index a98278295b..df9628c724 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -743,7 +743,7 @@ Legion::LogicalPartition Runtime::create_logical_partition( Legion::Future Runtime::create_future(const void* data, size_t datalen) const { - return Legion::Future::from_untyped_pointer(legion_runtime_, data, datalen); + return Legion::Future::from_untyped_pointer(data, datalen); } FieldID Runtime::allocate_field(const FieldSpace& field_space, size_t field_size) From 2290cb9d2e44facac12db649a24aaccf96509144 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 24 Aug 2022 13:05:52 -0700 Subject: [PATCH 0084/1425] Initial work for unbound stores. Bounds are eagerly fetched in a blocking manner --- src/core/data/logical_store.h | 1 - src/core/data/logical_store_detail.cc | 44 +++++++++++++++++--- src/core/data/logical_store_detail.h | 8 ++++ src/core/partitioning/partitioner.cc | 60 ++++++++++++++++++++++++++- src/core/partitioning/partitioner.h | 11 +++++ src/core/runtime/launcher.cc | 44 +++++++++++++++++++- src/core/runtime/launcher.h | 9 ++++ src/core/runtime/launcher_arg.cc | 20 +++++++++ src/core/runtime/launcher_arg.h | 28 +++++++++++++ src/core/runtime/operation.cc | 27 ++++++++++-- src/core/runtime/req_analyzer.cc | 47 +++++++++++++++++++++ src/core/runtime/req_analyzer.h | 22 ++++++++++ src/core/runtime/runtime.cc | 58 ++++++++++++++++++++++---- src/core/runtime/runtime.h | 12 +++++- 14 files changed, 368 insertions(+), 23 deletions(-) diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index 5550597597..2d05144a97 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -27,7 +27,6 @@ namespace legate { class BufferBuilder; class LibraryContext; -class LogicalRegionField; class Partition; class Projection; class Runtime; diff --git a/src/core/data/logical_store_detail.cc b/src/core/data/logical_store_detail.cc index 0633f32327..7009b8a2fc 100644 --- a/src/core/data/logical_store_detail.cc +++ b/src/core/data/logical_store_detail.cc @@ -32,17 +32,24 @@ namespace detail { // legate::detail::Storage //////////////////////////////////////////////////// +Storage::Storage(int32_t dim, LegateTypeCode code) : unbound_(true), dim_(dim), code_(code) {} + Storage::Storage(tuple extents, LegateTypeCode code) - : extents_(extents), code_(code), volume_(extents.volume()) + : dim_(extents.size()), extents_(extents), code_(code), volume_(extents.volume()) { } Storage::Storage(tuple extents, LegateTypeCode code, const Legion::Future& future) - : extents_(extents), code_(code), kind_(Kind::FUTURE), future_(future), volume_(extents.volume()) + : dim_(extents.size()), + extents_(extents), + code_(code), + kind_(Kind::FUTURE), + future_(future), + volume_(extents.volume()) { } -int32_t Storage::dim() { return static_cast(extents_.size()); } +int32_t Storage::dim() { return dim_; } LogicalRegionField* Storage::get_region_field() { @@ -62,6 +69,20 @@ Legion::Future Storage::get_future() const return future_; } +void Storage::set_region_field(std::shared_ptr&& region_field) +{ + unbound_ = false; + region_field_ = std::move(region_field); + + // TODO: this is a blocking operator + auto domain = region_field_->domain(); + auto lo = domain.lo(); + auto hi = domain.hi(); + std::vector extents; + for (int32_t idx = 0; idx < lo.dim; ++idx) extents.push_back(hi[idx] - lo[idx]); + extents_ = extents; +} + RegionField Storage::map(LibraryContext* context) { #ifdef DEBUG_LEGATE @@ -109,7 +130,6 @@ Legion::LogicalPartition Storage::find_or_create_legion_partition(const Partitio LogicalStore::LogicalStore(std::shared_ptr&& storage) : store_id_(Runtime::get_runtime()->get_unique_store_id()), - extents_(storage->extents()), storage_(std::forward(storage)), transform_(std::make_shared()) { @@ -118,6 +138,7 @@ LogicalStore::LogicalStore(std::shared_ptr&& storage) log_legate.debug() << "Create " << to_string(); #endif + if (!unbound()) extents_ = storage_->extents(); } LogicalStore::LogicalStore(tuple&& extents, @@ -140,6 +161,8 @@ LogicalStore::~LogicalStore() if (mapped_ != nullptr) mapped_->unmap(); } +bool LogicalStore::unbound() const { return storage_->unbound(); } + const tuple& LogicalStore::extents() const { return extents_; } size_t LogicalStore::volume() const { return extents_.volume(); } @@ -158,7 +181,10 @@ size_t LogicalStore::storage_size() const return storage_->volume() * elem_size; } -int32_t LogicalStore::dim() const { return static_cast(extents_.size()); } +int32_t LogicalStore::dim() const +{ + return unbound() ? storage_->dim() : static_cast(extents_.size()); +} bool LogicalStore::scalar() const { return storage_->kind() == Storage::Kind::FUTURE; } @@ -168,6 +194,12 @@ LogicalRegionField* LogicalStore::get_region_field() { return storage_->get_regi Legion::Future LogicalStore::get_future() { return storage_->get_future(); } +void LogicalStore::set_region_field(std::shared_ptr&& region_field) +{ + storage_->set_region_field(std::move(region_field)); + extents_ = storage_->extents(); +} + std::shared_ptr LogicalStore::promote(int32_t extra_dim, size_t dim_size) const { if (extra_dim < 0 || extra_dim > dim()) { @@ -265,7 +297,7 @@ void LogicalStore::reset_key_partition() { storage_->reset_key_partition(); } void LogicalStore::pack(BufferBuilder& buffer) const { buffer.pack(scalar()); - buffer.pack(false); + buffer.pack(unbound()); buffer.pack(dim()); buffer.pack(code()); transform_->pack(buffer); diff --git a/src/core/data/logical_store_detail.h b/src/core/data/logical_store_detail.h index dc4d13974e..3b50e44ad2 100644 --- a/src/core/data/logical_store_detail.h +++ b/src/core/data/logical_store_detail.h @@ -31,12 +31,15 @@ class Storage { }; public: + // Create a RegionField-backed storage whose size is unbound. Initialized lazily. + Storage(int32_t dim, LegateTypeCode code); // Create a RegionField-backed storage. Initialized lazily. Storage(tuple extents, LegateTypeCode code); // Create a Future-backed storage. Initialized eagerly. Storage(tuple extents, LegateTypeCode code, const Legion::Future& future); public: + bool unbound() const { return unbound_; } const tuple& extents() const { return extents_; } size_t volume() const { return volume_; } int32_t dim(); @@ -46,6 +49,7 @@ class Storage { public: LogicalRegionField* get_region_field(); Legion::Future get_future() const; + void set_region_field(std::shared_ptr&& region_field); public: RegionField map(LibraryContext* context); @@ -57,6 +61,8 @@ class Storage { Legion::LogicalPartition find_or_create_legion_partition(const Partition* partition); private: + bool unbound_{false}; + int32_t dim_{-1}; tuple extents_; size_t volume_; LegateTypeCode code_{MAX_TYPE_NUMBER}; @@ -90,6 +96,7 @@ class LogicalStore { LogicalStore& operator=(LogicalStore&& other) = default; public: + bool unbound() const; const tuple& extents() const; size_t volume() const; // Size of the backing storage @@ -101,6 +108,7 @@ class LogicalStore { public: LogicalRegionField* get_region_field(); Legion::Future get_future(); + void set_region_field(std::shared_ptr&& region_field); public: std::shared_ptr promote(int32_t extra_dim, size_t dim_size) const; diff --git a/src/core/partitioning/partitioner.cc b/src/core/partitioning/partitioner.cc index a7fe1b2573..369fcd5727 100644 --- a/src/core/partitioning/partitioner.cc +++ b/src/core/partitioning/partitioner.cc @@ -66,6 +66,17 @@ void Strategy::insert(const Variable* partition_symbol, std::shared_ptr partition, + Legion::FieldSpace field_space) +{ +#ifdef DEBUG_LEGATE + assert(field_spaces_.find(*partition_symbol) == field_spaces_.end()); +#endif + field_spaces_.insert({*partition_symbol, field_space}); + insert(partition_symbol, std::move(partition)); +} + bool Strategy::has_assignment(const Variable* partition_symbol) const { return assignments_.find(*partition_symbol) != assignments_.end(); @@ -80,6 +91,15 @@ const std::shared_ptr& Strategy::operator[](const Variable* partition return finder->second; } +const Legion::FieldSpace& Strategy::find_field_space(const Variable* partition_symbol) const +{ + auto finder = field_spaces_.find(*partition_symbol); +#ifdef DEBUG_LEGATE + assert(finder != field_spaces_.end()); +#endif + return finder->second; +} + //////////////////////////////////////////////////// // legate::Partitioner //////////////////////////////////////////////////// @@ -105,12 +125,19 @@ std::unique_ptr Partitioner::solve() // Copy the list of partition symbols as we will sort them inplace std::vector partition_symbols(constraints.partition_symbols()); + + solve_for_unbound_stores(partition_symbols, strategy.get(), constraints); + std::stable_sort(partition_symbols.begin(), partition_symbols.end(), [](const auto& part_symb_a, const auto& part_symb_b) { auto get_storage_size = [](const auto& part_symb) { - auto* op = part_symb->operation(); - return op->find_store(part_symb)->storage_size(); + auto* op = part_symb->operation(); + auto* store = op->find_store(part_symb); +#ifdef DEBUG_LEGATE + assert(!store->unbound()); +#endif + return store->storage_size(); }; return get_storage_size(part_symb_a) > get_storage_size(part_symb_b); }); @@ -137,4 +164,33 @@ std::unique_ptr Partitioner::solve() return std::move(strategy); } +void Partitioner::solve_for_unbound_stores(std::vector& partition_symbols, + Strategy* strategy, + const ConstraintGraph& constraints) +{ + auto runtime = Runtime::get_runtime(); + + std::vector filtered; + filtered.reserve(partition_symbols.size()); + + for (auto* part_symb : partition_symbols) { + auto* op = part_symb->operation(); + auto store = op->find_store(part_symb); + + if (!store->unbound()) { + filtered.push_back(part_symb); + continue; + } + + std::vector equiv_class; + constraints.find_equivalence_class(part_symb, equiv_class); + std::shared_ptr partition(create_no_partition()); + auto field_space = runtime->create_field_space(); + + for (auto symb : equiv_class) strategy->insert(symb, partition, field_space); + } + + partition_symbols.swap(filtered); +} + } // namespace legate diff --git a/src/core/partitioning/partitioner.h b/src/core/partitioning/partitioner.h index 530136b31d..1df5b94fe8 100644 --- a/src/core/partitioning/partitioner.h +++ b/src/core/partitioning/partitioner.h @@ -24,6 +24,7 @@ namespace legate { +class ConstraintGraph; class LogicalStore; class Operation; class Partition; @@ -42,11 +43,16 @@ class Strategy { public: void insert(const Variable* partition_symbol, std::shared_ptr partition); + void insert(const Variable* partition_symbol, + std::shared_ptr partition, + Legion::FieldSpace field_space); bool has_assignment(const Variable* partition_symbol) const; const std::shared_ptr& operator[](const Variable* partition_symbol) const; + const Legion::FieldSpace& find_field_space(const Variable* partition_symbol) const; private: std::map> assignments_{}; + std::map field_spaces_{}; std::map> launch_domains_{}; }; @@ -57,6 +63,11 @@ class Partitioner { public: std::unique_ptr solve(); + private: + void solve_for_unbound_stores(std::vector& partition_symbols, + Strategy* strategy, + const ConstraintGraph& constraints); + private: std::vector operations_; }; diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/launcher.cc index 9cde824af2..fc2510303a 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -35,6 +35,7 @@ TaskLauncher::TaskLauncher(LibraryContext* library, : library_(library), task_id_(task_id), mapper_id_(mapper_id), tag_(tag) { req_analyzer_ = new RequirementAnalyzer(); + out_analyzer_ = new OutputRequirementAnalyzer(); buffer_ = new BufferBuilder(); } @@ -84,17 +85,37 @@ void TaskLauncher::add_reduction(detail::LogicalStore* store, add_store(reductions_, store, std::move(proj), REDUCE, tag, flags); } +void TaskLauncher::add_unbound_output(detail::LogicalStore* store, + Legion::FieldSpace field_space, + Legion::FieldID field_id) +{ + out_analyzer_->insert(store->dim(), field_space, field_id); + auto arg = new OutputRegionArg(out_analyzer_, store, field_space, field_id); + outputs_.push_back(arg); + unbound_stores_.push_back(arg); +} + void TaskLauncher::execute(const Legion::Domain& launch_domain) { auto legion_launcher = build_index_task(launch_domain); - Runtime::get_runtime()->dispatch(legion_launcher); + if (output_requirements_.empty()) + Runtime::get_runtime()->dispatch(legion_launcher); + else { + Runtime::get_runtime()->dispatch(legion_launcher, &output_requirements_); + bind_region_fields_to_unbound_stores(); + } delete legion_launcher; } void TaskLauncher::execute_single() { auto legion_launcher = build_single_task(); - Runtime::get_runtime()->dispatch(legion_launcher); + if (output_requirements_.empty()) + Runtime::get_runtime()->dispatch(legion_launcher); + else { + Runtime::get_runtime()->dispatch(legion_launcher, &output_requirements_); + bind_region_fields_to_unbound_stores(); + } delete legion_launcher; } @@ -135,6 +156,7 @@ Legion::TaskLauncher* TaskLauncher::build_single_task() // Coalesce region requirements before packing task arguments // as the latter requires requirement indices to be finalized req_analyzer_->analyze_requirements(); + out_analyzer_->analyze_requirements(); pack_args(inputs_); pack_args(outputs_); @@ -152,6 +174,7 @@ Legion::TaskLauncher* TaskLauncher::build_single_task() for (auto& future_ : futures_) single_task->add_future(future_); req_analyzer_->populate_launcher(single_task); + out_analyzer_->populate_output_requirements(output_requirements_); return single_task; } @@ -161,6 +184,7 @@ Legion::IndexTaskLauncher* TaskLauncher::build_index_task(const Legion::Domain& // Coalesce region requirements before packing task arguments // as the latter requires requirement indices to be finalized req_analyzer_->analyze_requirements(); + out_analyzer_->analyze_requirements(); pack_args(inputs_); pack_args(outputs_); @@ -181,8 +205,24 @@ Legion::IndexTaskLauncher* TaskLauncher::build_index_task(const Legion::Domain& for (auto& future_ : futures_) index_task->add_future(future_); req_analyzer_->populate_launcher(index_task); + out_analyzer_->populate_output_requirements(output_requirements_); return index_task; } +void TaskLauncher::bind_region_fields_to_unbound_stores() +{ + auto* runtime = Runtime::get_runtime(); + + for (auto& arg : unbound_stores_) { +#ifdef DEBUG_LEGATE + assert(arg->requirement_index() != -1U); +#endif + auto* store = arg->store(); + auto& req = output_requirements_[arg->requirement_index()]; + auto region_field = runtime->import_region_field(req.region, arg->field_id(), store->code()); + store->set_region_field(std::move(region_field)); + } +} + } // namespace legate diff --git a/src/core/runtime/launcher.h b/src/core/runtime/launcher.h index 61203c0e1b..cb4624015a 100644 --- a/src/core/runtime/launcher.h +++ b/src/core/runtime/launcher.h @@ -27,6 +27,8 @@ class BufferBuilder; class LibraryContext; class LogicalStore; class Projection; +class OutputRegionArg; +class OutputRequirementAnalyzer; class RequirementAnalyzer; class Scalar; @@ -60,6 +62,9 @@ class TaskLauncher { Legion::MappingTagID tag = 0, Legion::RegionFlags flags = LEGION_NO_FLAG, bool read_write = false); + void add_unbound_output(detail::LogicalStore* store, + Legion::FieldSpace field_space, + Legion::FieldID field_id); private: void add_store(std::vector& args, @@ -77,6 +82,7 @@ class TaskLauncher { void pack_args(const std::vector& args); Legion::IndexTaskLauncher* build_index_task(const Legion::Domain& launch_domain); Legion::TaskLauncher* build_single_task(); + void bind_region_fields_to_unbound_stores(); private: LibraryContext* library_; @@ -90,10 +96,13 @@ class TaskLauncher { std::vector reductions_; std::vector scalars_; std::vector futures_; + std::vector unbound_stores_; private: RequirementAnalyzer* req_analyzer_; + OutputRequirementAnalyzer* out_analyzer_; BufferBuilder* buffer_; + std::vector output_requirements_; }; } // namespace legate diff --git a/src/core/runtime/launcher_arg.cc b/src/core/runtime/launcher_arg.cc index 98dea4569a..3ea407dd31 100644 --- a/src/core/runtime/launcher_arg.cc +++ b/src/core/runtime/launcher_arg.cc @@ -52,6 +52,26 @@ void RegionFieldArg::pack(BufferBuilder& buffer) const buffer.pack(field_id_); } +OutputRegionArg::OutputRegionArg(OutputRequirementAnalyzer* analyzer, + detail::LogicalStore* store, + Legion::FieldSpace field_space, + Legion::FieldID field_id) + : analyzer_(analyzer), store_(store), field_space_(field_space), field_id_(field_id) +{ +} + +void OutputRegionArg::pack(BufferBuilder& buffer) const +{ + store_->pack(buffer); + + requirement_index_ = analyzer_->get_requirement_index(field_space_, field_id_); + + buffer.pack(-1); + buffer.pack(store_->dim()); + buffer.pack(requirement_index_); + buffer.pack(field_id_); +} + FutureStoreArg::FutureStoreArg(detail::LogicalStore* store, bool read_only, bool has_storage, diff --git a/src/core/runtime/launcher_arg.h b/src/core/runtime/launcher_arg.h index 4c6ec58ec1..8378b114ce 100644 --- a/src/core/runtime/launcher_arg.h +++ b/src/core/runtime/launcher_arg.h @@ -22,6 +22,7 @@ namespace legate { +class OutputRequirementAnalyzer; class ProjectionInfo; class RequirementAnalyzer; @@ -82,6 +83,33 @@ struct RegionFieldArg : public ArgWrapper { const ProjectionInfo* proj_info_; }; +struct OutputRegionArg : public ArgWrapper { + public: + OutputRegionArg(OutputRequirementAnalyzer* analyzer, + detail::LogicalStore* store, + Legion::FieldSpace field_space, + Legion::FieldID field_id); + + public: + virtual void pack(BufferBuilder& buffer) const override; + + public: + virtual ~OutputRegionArg() {} + + public: + detail::LogicalStore* store() const { return store_; } + const Legion::FieldSpace& field_space() const { return field_space_; } + Legion::FieldID field_id() const { return field_id_; } + uint32_t requirement_index() const { return requirement_index_; } + + private: + OutputRequirementAnalyzer* analyzer_; + detail::LogicalStore* store_; + Legion::FieldSpace field_space_; + Legion::FieldID field_id_; + mutable uint32_t requirement_index_{-1U}; +}; + struct FutureStoreArg : public ArgWrapper { public: FutureStoreArg(detail::LogicalStore* store, diff --git a/src/core/runtime/operation.cc b/src/core/runtime/operation.cc index 3252b6e38c..2a4bc524e7 100644 --- a/src/core/runtime/operation.cc +++ b/src/core/runtime/operation.cc @@ -104,8 +104,18 @@ Task::~Task() {} void Task::add_scalar_arg(const Scalar& scalar) { scalars_.push_back(scalar); } +struct field_size_fn { + template + size_t operator()() + { + return sizeof(legate_type_of); + } +}; + void Task::launch(Strategy* p_strategy) { + auto* runtime = Runtime::get_runtime(); + auto& strategy = *p_strategy; TaskLauncher launcher(library_, task_id_, mapper_id_); @@ -117,9 +127,10 @@ void Task::launch(Strategy* p_strategy) } for (auto& pair : outputs_) { auto& store = pair.first; - auto& var = pair.second; - auto part = strategy[var]; - auto proj = part->get_projection(store); + if (store->unbound()) continue; + auto& var = pair.second; + auto part = strategy[var]; + auto proj = part->get_projection(store); launcher.add_output(store, std::move(proj)); store->set_key_partition(part.get()); } @@ -132,6 +143,16 @@ void Task::launch(Strategy* p_strategy) proj->set_reduction_op(redop); launcher.add_reduction(store, std::move(proj)); } + for (auto& pair : outputs_) { + auto& store = pair.first; + if (!store->unbound()) continue; + auto& var = pair.second; + auto field_space = strategy.find_field_space(var); + // TODO: We should reuse field ids here + auto field_size = type_dispatch(store->code(), field_size_fn{}); + auto field_id = runtime->allocate_field(field_space, field_size); + launcher.add_unbound_output(store, field_space, field_id); + } for (auto& scalar : scalars_) launcher.add_scalar(scalar); if (strategy.parallel(this)) diff --git a/src/core/runtime/req_analyzer.cc b/src/core/runtime/req_analyzer.cc index 3a358d2968..2dfc85d158 100644 --- a/src/core/runtime/req_analyzer.cc +++ b/src/core/runtime/req_analyzer.cc @@ -252,4 +252,51 @@ void RequirementAnalyzer::populate_launcher(Legion::TaskLauncher* task) const } } +//////////////////////////// +// OutputRequirementAnalyzer +//////////////////////////// + +OutputRequirementAnalyzer::~OutputRequirementAnalyzer() {} + +void OutputRequirementAnalyzer::insert(int32_t dim, + const Legion::FieldSpace& field_space, + Legion::FieldID field_id) +{ + auto& req_info = req_infos_[field_space]; +#ifdef DEBUG_LEGATE + // TODO: This should be checked when alignment constraints are set on unbound stores + assert(-1 == req_info.dim || req_info.dim == dim); +#endif + req_info.dim = dim; + field_groups_[field_space].insert(field_id); +} + +uint32_t OutputRequirementAnalyzer::get_requirement_index(const Legion::FieldSpace& field_space, + Legion::FieldID field_id) const +{ + auto finder = req_infos_.find(field_space); +#ifdef DEBUG_LEGATE + assert(finder != req_infos_.end()); +#endif + return finder->second.req_idx; +} + +void OutputRequirementAnalyzer::analyze_requirements() +{ + uint32_t idx = 0; + for (auto& entry : field_groups_) req_infos_[entry.first].req_idx = idx++; +} + +void OutputRequirementAnalyzer::populate_output_requirements( + std::vector& out_reqs) const +{ + for (auto& entry : field_groups_) { + auto finder = req_infos_.find(entry.first); +#ifdef DEBUG_LEGATE + assert(finder != req_infos_.end()); +#endif + out_reqs.emplace_back(entry.first, entry.second, finder->second.dim, true); + } +} + } // namespace legate diff --git a/src/core/runtime/req_analyzer.h b/src/core/runtime/req_analyzer.h index 45c6fb0a6b..d2de3cc139 100644 --- a/src/core/runtime/req_analyzer.h +++ b/src/core/runtime/req_analyzer.h @@ -123,4 +123,26 @@ class RequirementAnalyzer { std::map> field_sets_; }; +class OutputRequirementAnalyzer { + public: + ~OutputRequirementAnalyzer(); + + public: + void insert(int32_t dim, const Legion::FieldSpace& field_space, Legion::FieldID field_id); + uint32_t get_requirement_index(const Legion::FieldSpace& field_space, + Legion::FieldID field_id) const; + + public: + void analyze_requirements(); + void populate_output_requirements(std::vector& out_reqs) const; + + private: + struct ReqInfo { + int32_t dim{-1}; + uint32_t req_idx{0}; + }; + std::map> field_groups_; + std::map req_infos_; +}; + } // namespace legate diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index df9628c724..4c4f7325fc 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -256,11 +256,12 @@ class RegionManager { private: Legion::LogicalRegion active_region() const; - bool has_space() const; void create_region(); public: + bool has_space() const; std::pair allocate_field(size_t field_size); + void import_region(const Legion::LogicalRegion& region); private: Runtime* runtime_; @@ -275,8 +276,6 @@ RegionManager::RegionManager(Runtime* runtime, const Domain& shape) LogicalRegion RegionManager::active_region() const { return regions_.back(); } -bool RegionManager::has_space() const { return regions_.size() > 0; } - void RegionManager::create_region() { auto is = runtime_->find_or_create_index_space(shape_); @@ -284,6 +283,8 @@ void RegionManager::create_region() regions_.push_back(runtime_->create_region(is, fs)); } +bool RegionManager::has_space() const { return regions_.size() > 0; } + std::pair RegionManager::allocate_field(size_t field_size) { if (!has_space()) create_region(); @@ -292,6 +293,11 @@ std::pair RegionManager::allocate_field( return std::make_pair(lr, fid); } +void RegionManager::import_region(const Legion::LogicalRegion& region) +{ + regions_.push_back(region); +} + //////////////////////////////////////////////////// // legate::FieldManager //////////////////////////////////////////////////// @@ -302,6 +308,8 @@ class FieldManager { public: std::shared_ptr allocate_field(); + std::shared_ptr import_field(const Legion::LogicalRegion& region, + Legion::FieldID field_id); private: Runtime* runtime_; @@ -353,6 +361,23 @@ std::shared_ptr FieldManager::allocate_field() }); } +std::shared_ptr FieldManager::import_field(const Legion::LogicalRegion& region, + Legion::FieldID field_id) +{ + // Import the region only if the region manager is created fresh + auto rgn_mgr = runtime_->find_or_create_region_manager(shape_); + if (!rgn_mgr->has_space()) rgn_mgr->import_region(region); + + log_legate.debug("Field %u imported in field manager %p", field_id, this); + + auto* rf = new LogicalRegionField(region, field_id); + return std::shared_ptr(rf, [this](auto* field) { + log_legate.debug("Field %u freed in field manager %p", field->field_id(), this); + this->free_fields_.push_back(FreeField(field->region(), field->field_id())); + delete field; + }); +} + //////////////////////////////////////////////////// // legate::PartitionManager //////////////////////////////////////////////////// @@ -613,6 +638,12 @@ void Runtime::schedule(std::vector> operations) for (auto& op : operations) op->launch(strategy.get()); } +LogicalStore Runtime::create_store(LegateTypeCode code, int32_t dim) +{ + auto storage = std::make_shared(dim, code); + return LogicalStore(std::make_shared(std::move(storage))); +} + LogicalStore Runtime::create_store(std::vector extents, LegateTypeCode code) { auto storage = std::make_shared(extents, code); @@ -643,6 +674,17 @@ std::shared_ptr Runtime::create_region_field(const tupleallocate_field(); } +std::shared_ptr Runtime::import_region_field(Legion::LogicalRegion region, + Legion::FieldID field_id, + LegateTypeCode code) +{ + // TODO: This is a blocking operation. We should instead use index sapces as keys to field + // managers + auto shape = legion_runtime_->get_index_space_domain(legion_context_, region.get_index_space()); + auto fld_mgr = runtime_->find_or_create_field_manager(shape, code); + return fld_mgr->import_field(region, field_id); +} + RegionField Runtime::map_region_field(LibraryContext* context, const LogicalRegionField* rf) { auto region = rf->region(); @@ -759,17 +801,19 @@ Domain Runtime::get_index_space_domain(const IndexSpace& index_space) const return legion_runtime_->get_index_space_domain(legion_context_, index_space); } -std::shared_ptr Runtime::dispatch(TaskLauncher* launcher) +std::shared_ptr Runtime::dispatch( + TaskLauncher* launcher, std::vector* output_requirements) { assert(nullptr != legion_context_); - legion_runtime_->execute_task(legion_context_, *launcher); + legion_runtime_->execute_task(legion_context_, *launcher, output_requirements); return nullptr; } -std::shared_ptr Runtime::dispatch(IndexTaskLauncher* launcher) +std::shared_ptr Runtime::dispatch( + IndexTaskLauncher* launcher, std::vector* output_requirements) { assert(nullptr != legion_context_); - legion_runtime_->execute_index_space(legion_context_, *launcher); + legion_runtime_->execute_index_space(legion_context_, *launcher, output_requirements); return nullptr; } diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 13eacf1c9d..8fabb8d9c3 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -114,6 +114,7 @@ class Runtime { void submit(std::unique_ptr op); public: + LogicalStore create_store(LegateTypeCode code, int32_t dim = 1); LogicalStore create_store(std::vector extents, LegateTypeCode code); LogicalStore create_store(const Scalar& scalar); uint64_t get_unique_store_id(); @@ -121,6 +122,9 @@ class Runtime { public: std::shared_ptr create_region_field(const tuple& extents, LegateTypeCode code); + std::shared_ptr import_region_field(Legion::LogicalRegion region, + Legion::FieldID field_id, + LegateTypeCode code); RegionField map_region_field(LibraryContext* context, const LogicalRegionField* region_field); void unmap_physical_region(Legion::PhysicalRegion pr); @@ -146,8 +150,12 @@ class Runtime { Legion::Domain get_index_space_domain(const Legion::IndexSpace& index_space) const; public: - std::shared_ptr dispatch(Legion::TaskLauncher* launcher); - std::shared_ptr dispatch(Legion::IndexTaskLauncher* launcher); + std::shared_ptr dispatch( + Legion::TaskLauncher* launcher, + std::vector* output_requirements = nullptr); + std::shared_ptr dispatch( + Legion::IndexTaskLauncher* launcher, + std::vector* output_requirements = nullptr); public: Legion::ProjectionID get_projection(int32_t src_ndim, const proj::SymbolicPoint& point); From df673f5c90f54f6d22b2a6b9eb9f6d18d297290e Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 24 Aug 2022 22:01:20 -0700 Subject: [PATCH 0085/1425] Add launch domain resolver and update the projection computation for forced 1d launches --- src/core/data/logical_store_detail.cc | 18 ++++- src/core/data/logical_store_detail.h | 4 +- src/core/partitioning/partition.cc | 8 +- src/core/partitioning/partition.h | 11 ++- src/core/partitioning/partitioner.cc | 111 +++++++++++++++++++++----- src/core/partitioning/partitioner.h | 11 ++- src/core/runtime/launcher.cc | 2 +- src/core/runtime/operation.cc | 12 +-- src/core/runtime/runtime.cc | 5 ++ src/core/runtime/runtime.h | 1 + 10 files changed, 140 insertions(+), 43 deletions(-) diff --git a/src/core/data/logical_store_detail.cc b/src/core/data/logical_store_detail.cc index 7009b8a2fc..23300c42a3 100644 --- a/src/core/data/logical_store_detail.cc +++ b/src/core/data/logical_store_detail.cc @@ -236,22 +236,32 @@ std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) return std::move(mapped_); } -Legion::ProjectionID LogicalStore::compute_projection() const +Legion::ProjectionID LogicalStore::compute_projection(int32_t launch_ndim) const { - if (transform_->identity()) return 0; + if (transform_->identity()) { + if (launch_ndim != dim()) + return Runtime::get_runtime()->get_delinearizing_projection(); + else + return 0; + } auto ndim = dim(); auto point = transform_->invert(proj::create_symbolic_point(ndim)); + // TODO: We can't currently mix affine projections with delinearizing projections +#ifdef DEBUG_LEGATE + assert(ndim == launch_ndim); +#endif return Runtime::get_runtime()->get_projection(ndim, point); } -std::unique_ptr LogicalStore::create_projection(const Partition* partition) +std::unique_ptr LogicalStore::create_projection(const Partition* partition, + int32_t launch_ndim) { if (scalar()) return std::make_unique(); // We're about to create a legion partition for this store, so the store should have its region // created. - auto proj_id = compute_projection(); + auto proj_id = compute_projection(launch_ndim); auto* orig_partition = partition; std::unique_ptr inverted = nullptr; if (!transform_->identity()) { diff --git a/src/core/data/logical_store_detail.h b/src/core/data/logical_store_detail.h index 3b50e44ad2..ecb1dc20a9 100644 --- a/src/core/data/logical_store_detail.h +++ b/src/core/data/logical_store_detail.h @@ -118,13 +118,13 @@ class LogicalStore { std::shared_ptr get_physical_store(LibraryContext* context); public: - std::unique_ptr create_projection(const Partition* partition); + std::unique_ptr create_projection(const Partition* partition, int32_t launch_ndim); std::shared_ptr find_or_create_key_partition(); void set_key_partition(const Partition* partition); void reset_key_partition(); private: - Legion::ProjectionID compute_projection() const; + Legion::ProjectionID compute_projection(int32_t launch_ndim) const; public: void pack(BufferBuilder& buffer) const; diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index 67248ba141..6ef3428efc 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -37,7 +37,8 @@ Legion::LogicalPartition NoPartition::construct(Legion::LogicalRegion region, return Legion::LogicalPartition::NO_PART; } -std::unique_ptr NoPartition::get_projection(detail::LogicalStore* store) const +std::unique_ptr NoPartition::get_projection(detail::LogicalStore* store, + int32_t launch_ndim) const { return std::make_unique(); } @@ -135,9 +136,10 @@ Legion::LogicalPartition Tiling::construct(Legion::LogicalRegion region, return runtime->create_logical_partition(region, index_partition); } -std::unique_ptr Tiling::get_projection(detail::LogicalStore* store) const +std::unique_ptr Tiling::get_projection(detail::LogicalStore* store, + int32_t launch_ndim) const { - return store->create_projection(this); + return store->create_projection(this, launch_ndim); } bool Tiling::has_launch_domain() const { return true; } diff --git a/src/core/partitioning/partition.h b/src/core/partitioning/partition.h index 163eb8c709..49ac59d8e0 100644 --- a/src/core/partitioning/partition.h +++ b/src/core/partitioning/partition.h @@ -54,8 +54,9 @@ struct Partition { public: virtual Legion::LogicalPartition construct(Legion::LogicalRegion region, bool disjoint = false, - bool complete = false) const = 0; - virtual std::unique_ptr get_projection(detail::LogicalStore* store) const = 0; + bool complete = false) const = 0; + virtual std::unique_ptr get_projection(detail::LogicalStore* store, + int32_t launch_ndim) const = 0; public: virtual bool has_launch_domain() const = 0; @@ -83,7 +84,8 @@ class NoPartition : public Partition { virtual Legion::LogicalPartition construct(Legion::LogicalRegion region, bool disjoint, bool complete) const override; - virtual std::unique_ptr get_projection(detail::LogicalStore* store) const override; + virtual std::unique_ptr get_projection(detail::LogicalStore* store, + int32_t launch_ndim) const override; public: virtual bool has_launch_domain() const override; @@ -118,7 +120,8 @@ class Tiling : public Partition { virtual Legion::LogicalPartition construct(Legion::LogicalRegion region, bool disjoint, bool complete) const override; - virtual std::unique_ptr get_projection(detail::LogicalStore* store) const override; + virtual std::unique_ptr get_projection(detail::LogicalStore* store, + int32_t launch_ndim) const override; public: virtual bool has_launch_domain() const override; diff --git a/src/core/partitioning/partitioner.cc b/src/core/partitioning/partitioner.cc index 369fcd5727..644971b97b 100644 --- a/src/core/partitioning/partitioner.cc +++ b/src/core/partitioning/partitioner.cc @@ -28,34 +28,86 @@ namespace legate { //////////////////////////////////////////////////// -// legate::Strategy +// legate::LaunchDomainResolver //////////////////////////////////////////////////// -Strategy::Strategy() {} +class LaunchDomainResolver { + private: + static constexpr int32_t UNSET = -1; -bool Strategy::parallel(const Operation* op) const + public: + void record_launch_domain(const Legion::Domain& launch_domain); + void record_unbound_store(int32_t unbound_dim); + + public: + std::unique_ptr resolve_launch_domain() const; + + private: + bool must_be_sequential_{false}; + bool must_be_1d_{false}; + int32_t unbound_dim_{UNSET}; + std::set launch_domains_; + std::set launch_volumes_; +}; + +void LaunchDomainResolver::record_launch_domain(const Legion::Domain& launch_domain) { - auto finder = launch_domains_.find(op); - return (launch_domains_.end() != finder) && (nullptr != finder->second); + launch_domains_.insert(launch_domain); + launch_volumes_.insert(launch_domain.get_volume()); + if (launch_domains_.size() > 1) must_be_1d_ = true; + if (launch_volumes_.size() > 1) must_be_sequential_ = true; } -bool Strategy::has_launch_domain(const Operation* op) const +void LaunchDomainResolver::record_unbound_store(int32_t unbound_dim) { - return launch_domains_.find(op) != launch_domains_.end(); + if (unbound_dim_ != UNSET && unbound_dim_ != unbound_dim) + must_be_sequential_ = true; + else + unbound_dim_ = unbound_dim; } -Legion::Domain Strategy::launch_domain(const Operation* op) const +std::unique_ptr LaunchDomainResolver::resolve_launch_domain() const { - auto finder = launch_domains_.find(op); - assert(finder != launch_domains_.end()); - return *finder->second; + if (must_be_sequential_) return nullptr; + if (must_be_1d_) { + if (unbound_dim_ != UNSET && unbound_dim_ > 1) + return nullptr; + else { +#ifdef DEBUG_LEGATE + assert(launch_volumes_.size() == 1); +#endif + int64_t volume = *launch_volumes_.begin(); + return std::make_unique(0, volume - 1); + } + } else { +#ifdef DEBUG_LEGATE + assert(launch_domains_.size() == 1); +#endif + auto& launch_domain = *launch_domains_.begin(); + if (unbound_dim_ != UNSET && launch_domain.dim != unbound_dim_) { + int64_t volume = *launch_volumes_.begin(); + return std::make_unique(0, volume - 1); + } + return std::make_unique(launch_domain); + } } -void Strategy::set_single_launch(const Operation* op) { launch_domains_[op] = nullptr; } +//////////////////////////////////////////////////// +// legate::Strategy +//////////////////////////////////////////////////// + +Strategy::Strategy() {} -void Strategy::set_launch_domain(const Operation* op, const Legion::Domain& launch_domain) +bool Strategy::parallel(const Operation* op) const +{ + auto finder = launch_domains_.find(op); + return finder != launch_domains_.end() && finder->second != nullptr; +} + +const Legion::Domain* Strategy::launch_domain(const Operation* op) const { - launch_domains_[op] = std::make_unique(launch_domain); + auto finder = launch_domains_.find(op); + return finder != launch_domains_.end() ? finder->second.get() : nullptr; } void Strategy::insert(const Variable* partition_symbol, std::shared_ptr partition) @@ -100,6 +152,29 @@ const Legion::FieldSpace& Strategy::find_field_space(const Variable* partition_s return finder->second; } +void Strategy::compute_launch_domains() +{ + std::map domain_resolvers; + + for (auto& assignment : assignments_) { + auto& part_symb = assignment.first; + auto& partition = assignment.second; + auto* op = part_symb.operation(); + + if (partition->has_launch_domain()) { + domain_resolvers[op].record_launch_domain(partition->launch_domain()); + continue; + } + + auto store = op->find_store(&part_symb); + + if (store->unbound()) domain_resolvers[op].record_unbound_store(store->dim()); + } + + for (auto& pair : domain_resolvers) + launch_domains_[pair.first] = pair.second.resolve_launch_domain(); +} + //////////////////////////////////////////////////// // legate::Partitioner //////////////////////////////////////////////////// @@ -148,12 +223,6 @@ std::unique_ptr Partitioner::solve() auto* op = part_symb->operation(); auto store = op->find_store(part_symb); auto partition = store->find_or_create_key_partition(); - if (!strategy->has_launch_domain(op)) { - if (partition->has_launch_domain()) - strategy->set_launch_domain(op, partition->launch_domain()); - else - strategy->set_single_launch(op); - } std::vector equiv_class; constraints.find_equivalence_class(part_symb, equiv_class); @@ -161,6 +230,8 @@ std::unique_ptr Partitioner::solve() for (auto symb : equiv_class) strategy->insert(symb, partition); } + strategy->compute_launch_domains(); + return std::move(strategy); } diff --git a/src/core/partitioning/partitioner.h b/src/core/partitioning/partitioner.h index 1df5b94fe8..3a3630d8ad 100644 --- a/src/core/partitioning/partitioner.h +++ b/src/core/partitioning/partitioner.h @@ -28,18 +28,18 @@ class ConstraintGraph; class LogicalStore; class Operation; class Partition; +class Partitioner; class Projection; class Strategy { + friend class Partitioner; + public: Strategy(); public: bool parallel(const Operation* op) const; - bool has_launch_domain(const Operation* op) const; - Legion::Domain launch_domain(const Operation* op) const; - void set_single_launch(const Operation* op); - void set_launch_domain(const Operation* op, const Legion::Domain& launch_domain); + const Legion::Domain* launch_domain(const Operation* op) const; public: void insert(const Variable* partition_symbol, std::shared_ptr partition); @@ -50,6 +50,9 @@ class Strategy { const std::shared_ptr& operator[](const Variable* partition_symbol) const; const Legion::FieldSpace& find_field_space(const Variable* partition_symbol) const; + private: + void compute_launch_domains(); + private: std::map> assignments_{}; std::map field_spaces_{}; diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/launcher.cc index fc2510303a..2bfc7dca61 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -220,7 +220,7 @@ void TaskLauncher::bind_region_fields_to_unbound_stores() #endif auto* store = arg->store(); auto& req = output_requirements_[arg->requirement_index()]; - auto region_field = runtime->import_region_field(req.region, arg->field_id(), store->code()); + auto region_field = runtime->import_region_field(req.parent, arg->field_id(), store->code()); store->set_region_field(std::move(region_field)); } } diff --git a/src/core/runtime/operation.cc b/src/core/runtime/operation.cc index 2a4bc524e7..5fcc0b113b 100644 --- a/src/core/runtime/operation.cc +++ b/src/core/runtime/operation.cc @@ -118,11 +118,13 @@ void Task::launch(Strategy* p_strategy) auto& strategy = *p_strategy; TaskLauncher launcher(library_, task_id_, mapper_id_); + auto launch_domain = strategy.launch_domain(this); + auto launch_ndim = launch_domain != nullptr ? launch_domain->dim : 0; for (auto& pair : inputs_) { auto& store = pair.first; auto& var = pair.second; - auto proj = strategy[var]->get_projection(store); + auto proj = strategy[var]->get_projection(store, launch_ndim); launcher.add_input(store, std::move(proj)); } for (auto& pair : outputs_) { @@ -130,7 +132,7 @@ void Task::launch(Strategy* p_strategy) if (store->unbound()) continue; auto& var = pair.second; auto part = strategy[var]; - auto proj = part->get_projection(store); + auto proj = part->get_projection(store, launch_ndim); launcher.add_output(store, std::move(proj)); store->set_key_partition(part.get()); } @@ -138,7 +140,7 @@ void Task::launch(Strategy* p_strategy) for (auto& pair : reductions_) { auto& store = pair.first; auto& var = pair.second; - auto proj = strategy[var]->get_projection(store); + auto proj = strategy[var]->get_projection(store, launch_ndim); auto redop = reduction_ops_[idx++]; proj->set_reduction_op(redop); launcher.add_reduction(store, std::move(proj)); @@ -155,8 +157,8 @@ void Task::launch(Strategy* p_strategy) } for (auto& scalar : scalars_) launcher.add_scalar(scalar); - if (strategy.parallel(this)) - launcher.execute(strategy.launch_domain(this)); + if (launch_domain != nullptr) + launcher.execute(*launch_domain); else launcher.execute_single(); } diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 4c4f7325fc..627f6c9679 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -858,6 +858,11 @@ Legion::ProjectionID Runtime::get_projection(int32_t src_ndim, const proj::Symbo return proj_id; } +Legion::ProjectionID Runtime::get_delinearizing_projection() +{ + return core_context_->get_projection_id(LEGATE_CORE_DELINEARIZE_PROJ_ID); +} + /*static*/ void Runtime::initialize(int32_t argc, char** argv) { Legion::Runtime::initialize(&argc, &argv, true /*filter legion and realm args*/); diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 8fabb8d9c3..564745cd21 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -159,6 +159,7 @@ class Runtime { public: Legion::ProjectionID get_projection(int32_t src_ndim, const proj::SymbolicPoint& point); + Legion::ProjectionID get_delinearizing_projection(); private: void schedule(std::vector> operations); From 910027f1b2a89913bae8004e67f6b17e2147e881 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 14 Sep 2022 20:49:12 -0700 Subject: [PATCH 0086/1425] API to issue execution fences --- src/core/runtime/runtime.cc | 6 ++++++ src/core/runtime/runtime.h | 3 +++ 2 files changed, 9 insertions(+) diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 627f6c9679..5335e4a067 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -817,6 +817,12 @@ std::shared_ptr Runtime::dispatch( return nullptr; } +void Runtime::issue_execution_fence(bool block /*=false*/) +{ + auto future = legion_runtime_->issue_execution_fence(legion_context_); + if (block) future.wait(); +} + Legion::ProjectionID Runtime::get_projection(int32_t src_ndim, const proj::SymbolicPoint& point) { #ifdef DEBUG_LEGATE diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 564745cd21..a9ee0c985e 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -157,6 +157,9 @@ class Runtime { Legion::IndexTaskLauncher* launcher, std::vector* output_requirements = nullptr); + public: + void issue_execution_fence(bool block = false); + public: Legion::ProjectionID get_projection(int32_t src_ndim, const proj::SymbolicPoint& point); Legion::ProjectionID get_delinearizing_projection(); From 57c0a3e0e64928dabfdfe58f0d27bd952d503193 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 28 Sep 2022 23:22:30 -0700 Subject: [PATCH 0087/1425] Add C++ core files to the cmake build --- legate_core_cpp.cmake | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index f22a11764f..34ae3c6d4b 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -189,6 +189,8 @@ list(APPEND legate_core_SOURCES src/core/comm/comm_cpu.cc src/core/comm/coll.cc src/core/data/allocator.cc + src/core/data/logical_store.cc + src/core/data/logical_store_detail.cc src/core/data/scalar.cc src/core/data/store.cc src/core/data/transform.cc @@ -197,12 +199,21 @@ list(APPEND legate_core_SOURCES src/core/mapping/instance_manager.cc src/core/mapping/mapping.cc src/core/mapping/task.cc + src/core/partitioning/constraint.cc + src/core/partitioning/constraint_graph.cc + src/core/partitioning/partition.cc + src/core/partitioning/partitioner.cc src/core/runtime/context.cc + src/core/runtime/launcher_arg.cc + src/core/runtime/launcher.cc + src/core/runtime/operation.cc src/core/runtime/projection.cc + src/core/runtime/req_analyzer.cc src/core/runtime/runtime.cc src/core/runtime/shard.cc src/core/task/return.cc src/core/task/task.cc + src/core/utilities/buffer_builder.cc src/core/utilities/debug.cc src/core/utilities/deserializer.cc src/core/utilities/machine.cc @@ -333,6 +344,7 @@ install( install( FILES src/core/data/allocator.h src/core/data/buffer.h + src/core/data/logical_store.h src/core/data/scalar.h src/core/data/scalar.inl src/core/data/store.h @@ -347,9 +359,16 @@ install( src/core/mapping/task.inl DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/legate/core/mapping) +install( + FILES src/core/partitioning/constraint.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/legate/core/partitioning) + install( FILES src/core/runtime/context.h + src/core/runtime/operation.h + src/core/runtime/projection.h src/core/runtime/runtime.h + src/core/runtime/runtime.inl DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/legate/core/runtime) install( @@ -366,6 +385,8 @@ install( src/core/utilities/machine.h src/core/utilities/nvtx_help.h src/core/utilities/span.h + src/core/utilities/tuple.inl + src/core/utilities/tuple.h src/core/utilities/type_traits.h src/core/utilities/typedefs.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/legate/core/utilities) From 10a015ce98a625464cc12bdb5d054ddebb24bc2e Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 30 Sep 2022 11:33:50 -0700 Subject: [PATCH 0088/1425] Adjust consensus match frequency based on field sizes (#402) * Perform consensus match more frequently for bigger free fields * Minor cleanup --- legate/core/runtime.py | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/legate/core/runtime.py b/legate/core/runtime.py index fa4fdaad99..b47624378f 100644 --- a/legate/core/runtime.py +++ b/legate/core/runtime.py @@ -200,9 +200,9 @@ def add_free_field( ) -> None: self._freed_fields.append(FreeFieldInfo(manager, region, field_id)) - def issue_field_match(self) -> None: + def issue_field_match(self, credit: int) -> None: # Increment our match counter - self._match_counter += 1 + self._match_counter += credit if self._match_counter < self._match_frequency: return # If the match counter equals our match frequency then do an exchange @@ -342,9 +342,29 @@ def __init__( ) -> None: super().__init__(runtime, shape, field_size) self._field_match_manager = runtime.field_match_manager + self._update_match_credit() + + def _update_match_credit(self) -> None: + if self.shape.fixed: + size = self.shape.volume() * self.field_size + self._match_credit = ( + size + self.runtime.max_field_reuse_size - 1 + if size > self.runtime.max_field_reuse_size + else self.runtime.max_field_reuse_size + ) // self.runtime.max_field_reuse_size + # No need to update the credit as the exact size is known + self._need_to_update_match_credit = False + # If the shape is unknown, we set the credit such that every new + # free field leads to a consensus match, and ask the manager + # to update the credit. + else: + self._match_credit = self.runtime.max_field_reuse_frequency + self._need_to_update_match_credit = True def try_reuse_field(self) -> Optional[tuple[Region, int]]: - self._field_match_manager.issue_field_match() + if self._need_to_update_match_credit: + self._update_match_credit() + self._field_match_manager.issue_field_match(self._match_credit) # First, if we have a free field then we know everyone has one of those if len(self.free_fields) > 0: @@ -915,6 +935,12 @@ def __init__(self, core_library: CoreLib) -> None: ty.uint32, ) ) + self.max_field_reuse_size = int( + self._core_context.get_tunable( + legion.LEGATE_CORE_TUNABLE_FIELD_REUSE_SIZE, + ty.uint64, + ) + ) self._field_manager_class = ( ConsensusMatchingFieldManager if self._num_nodes > 1 or self._args.consensus @@ -1246,12 +1272,7 @@ def find_region_manager(self, region: Region) -> RegionManager: return self.region_managers_by_region[region] def revive_manager(self, region_mgr: RegionManager) -> None: - lru_managers: Deque[RegionManager] = deque() - for to_check in self.lru_managers: - if to_check is not region_mgr: - lru_managers.append(to_check) - assert len(lru_managers) < len(self.lru_managers) - self.lru_managers = lru_managers + self.lru_managers.remove(region_mgr) def free_region_manager( self, shape: Shape, region: Region, unordered: bool = False From 67ce8e55a7d5371887d4884b6099aca94425a1c6 Mon Sep 17 00:00:00 2001 From: Manolis Papadakis Date: Tue, 4 Oct 2022 15:59:48 -0700 Subject: [PATCH 0089/1425] Legion bug WAR: don't instantiate futures on framebuffer (#409) * Legion bug WAR: don't instantiate futures on framebuffer * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Wrap the workaround with a define Co-authored-by: Manolis Papadakis Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- src/core/data/store.cc | 6 +++++- src/core/mapping/base_mapper.cc | 6 +++++- src/legate_defines.h | 3 +++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/core/data/store.cc b/src/core/data/store.cc index 76d4b405fe..a4ca73f4bd 100644 --- a/src/core/data/store.cc +++ b/src/core/data/store.cc @@ -136,9 +136,13 @@ FutureWrapper::FutureWrapper( #ifdef DEBUG_LEGATE assert(!initialize || future_.get_untyped_size() == field_size); #endif - auto proc = Processor::get_executing_processor(); + auto proc = Processor::get_executing_processor(); +#ifdef LEGATE_NO_FUTURES_ON_FB + auto mem_kind = find_memory_kind_for_executing_processor(); +#else auto mem_kind = proc.kind() == Processor::Kind::TOC_PROC ? Memory::Kind::GPU_FB_MEM : Memory::Kind::SYSTEM_MEM; +#endif if (initialize) { auto p_init_value = future_.get_buffer(mem_kind); #ifdef LEGATE_USE_CUDA diff --git a/src/core/mapping/base_mapper.cc b/src/core/mapping/base_mapper.cc index 739cd8f978..f975fe3f86 100644 --- a/src/core/mapping/base_mapper.cc +++ b/src/core/mapping/base_mapper.cc @@ -610,7 +610,11 @@ void BaseMapper::map_task(const MapperContext ctx, if (req_indices.empty()) { // This is a mapping for futures - output.future_locations.push_back(get_target_memory(task.target_proc, mapping.policy.target)); + StoreTarget target = mapping.policy.target; +#ifdef LEGATE_NO_FUTURES_ON_FB + if (target == StoreTarget::FBMEM) target = StoreTarget::ZCMEM; +#endif + output.future_locations.push_back(get_target_memory(task.target_proc, target)); continue; } else if (mapping.for_unbound_stores()) { for (auto req_idx : req_indices) { diff --git a/src/legate_defines.h b/src/legate_defines.h index b7c3dacbae..fa215e8e73 100644 --- a/src/legate_defines.h +++ b/src/legate_defines.h @@ -50,3 +50,6 @@ #define LEGATE_USE_NETWORK #endif #endif + +// TODO: 2022-10-04: Work around a Legion bug, by not instantiating futures on framebuffer. +#define LEGATE_NO_FUTURES_ON_FB From 85cc75aa7ca3270c3a3e9f6f6228a57a374aaea8 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Thu, 6 Oct 2022 20:41:34 -0700 Subject: [PATCH 0090/1425] Revive dead region managers on field allocations (#418) * Make sure we test LRU mechanism in debug mode * Make sure we don't create fields on a dead region manager --- legate/core/runtime.py | 32 ++++++++++++++++++++++---------- src/core/mapping/core_mapper.cc | 2 +- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/legate/core/runtime.py b/legate/core/runtime.py index b47624378f..ab7371ce56 100644 --- a/legate/core/runtime.py +++ b/legate/core/runtime.py @@ -264,15 +264,21 @@ def destroy(self, unordered: bool) -> None: # unordered destructions self._region.destroy(unordered) - def increase_field_count(self) -> bool: + def increase_active_field_count(self) -> bool: revived = self._active_field_count == 0 self._active_field_count += 1 return revived - def decrease_field_count(self) -> bool: + def decrease_active_field_count(self) -> bool: self._active_field_count -= 1 return self._active_field_count == 0 + def increase_field_count(self) -> bool: + fresh = self._alloc_field_count == 0 + self._alloc_field_count += 1 + revived = self.increase_active_field_count() + return not fresh and revived + @property def has_space(self) -> bool: return self._alloc_field_count < LEGATE_MAX_FIELDS @@ -282,13 +288,12 @@ def get_next_field_id(self) -> int: self._next_field_id += 1 return field_id - def allocate_field(self, field_size: Any) -> tuple[Region, int]: + def allocate_field(self, field_size: Any) -> tuple[Region, int, bool]: field_id = self._region.field_space.allocate_field( field_size, self.get_next_field_id() ) - self._alloc_field_count += 1 - self.increase_field_count() - return self._region, field_id + revived = self.increase_field_count() + return self._region, field_id, revived # This class manages the allocation and reuse of fields @@ -316,18 +321,23 @@ def try_reuse_field(self) -> Optional[tuple[Region, int]]: def allocate_field(self) -> tuple[Region, int]: if (result := self.try_reuse_field()) is not None: region_manager = self.runtime.find_region_manager(result[0]) - if region_manager.increase_field_count(): + if region_manager.increase_active_field_count(): self.runtime.revive_manager(region_manager) return result region_manager = self.runtime.find_or_create_region_manager(self.shape) - return region_manager.allocate_field(self.field_size) + region, field_id, revived = region_manager.allocate_field( + self.field_size + ) + if revived: + self.runtime.revive_manager(region_manager) + return region, field_id def free_field( self, region: Region, field_id: int, ordered: bool = False ) -> None: self.free_fields.append((region, field_id)) region_manager = self.runtime.find_region_manager(region) - if region_manager.decrease_field_count(): + if region_manager.decrease_active_field_count(): self.runtime.free_region_manager( self.shape, region, unordered=not ordered ) @@ -1362,7 +1372,9 @@ def import_output_region( self.region_managers_by_region[region] = region_mgr self.find_or_create_field_manager(shape, dtype.size) - region_mgr.increase_field_count() + revived = region_mgr.increase_field_count() + if revived: + self.revive_manager(region_mgr) return RegionField.create(region, field_id, dtype.size, shape) def create_output_region( diff --git a/src/core/mapping/core_mapper.cc b/src/core/mapping/core_mapper.cc index 26e140879e..da3f7414b6 100644 --- a/src/core/mapping/core_mapper.cc +++ b/src/core/mapping/core_mapper.cc @@ -145,7 +145,7 @@ CoreMapper::CoreMapper(MapperRuntime* rt, Machine m, const LibraryContext& c) precise_exception_trace(static_cast(extract_env("LEGATE_PRECISE_EXCEPTION_TRACE", 0, 0))), field_reuse_frac(extract_env("LEGATE_FIELD_REUSE_FRAC", 256, 256)), field_reuse_freq(extract_env("LEGATE_FIELD_REUSE_FREQ", 32, 32)), - max_lru_length(extract_env("LEGATE_MAX_LRU_LENGTH", 5, 0)), + max_lru_length(extract_env("LEGATE_MAX_LRU_LENGTH", 5, 1)), has_socket_mem(false) { // Query to find all our local processors From c1f4de7e93c668ffd2377b7d9c9c10cbd1cca929 Mon Sep 17 00:00:00 2001 From: Manolis Papadakis Date: Fri, 7 Oct 2022 09:10:41 -0700 Subject: [PATCH 0091/1425] Fix typo in driver script (#420) --- legate/driver/logs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/legate/driver/logs.py b/legate/driver/logs.py index 1173e84864..37fd714002 100644 --- a/legate/driver/logs.py +++ b/legate/driver/logs.py @@ -143,7 +143,7 @@ def process(self) -> bool: dflag = "d" if self.config.debugging.dataflow else "" eflag = "e" if self.config.debugging.event else "" if dflag or eflag: - cmd += ("-{dflag}{eflag}",) + cmd += (f"-{dflag}{eflag}",) cmd += tuple(f"legate_{n}.log" for n in range(ranks)) From 728014ab7cf6aa27c99e7b54b6826525c4c1a418 Mon Sep 17 00:00:00 2001 From: Manolis Papadakis Date: Fri, 7 Oct 2022 16:18:31 -0700 Subject: [PATCH 0092/1425] On mapping failure retry after tightening non-RO reqs (#424) port of #423 --- src/core/mapping/base_mapper.cc | 212 ++++++++++++++++++++++---------- src/core/mapping/base_mapper.h | 3 +- 2 files changed, 151 insertions(+), 64 deletions(-) diff --git a/src/core/mapping/base_mapper.cc b/src/core/mapping/base_mapper.cc index f975fe3f86..66c1f4f801 100644 --- a/src/core/mapping/base_mapper.cc +++ b/src/core/mapping/base_mapper.cc @@ -18,6 +18,7 @@ #include #include "legion/legion_mapping.h" +#include "mappers/mapping_utilities.h" #include "core/data/store.h" #include "core/mapping/base_mapper.h" @@ -502,6 +503,10 @@ void BaseMapper::map_task(const MapperContext ctx, const MapTaskInput& input, MapTaskOutput& output) { +#ifdef DEBUG_LEGATE + logger.debug() << "Entering map_task for " << Utilities::to_string(runtime, ctx, task); +#endif + // Should never be mapping the top-level task here assert(task.get_depth() > 0); @@ -601,10 +606,51 @@ void BaseMapper::map_task(const MapperContext ctx, output.chosen_instances.resize(task.regions.size()); - // Map each field separately for each of the logical regions - std::vector needed_acquires; - std::map> instances_to_mappings; - for (uint32_t mapping_idx = 0; mapping_idx < mappings.size(); ++mapping_idx) { + bool can_fail = true; + std::map> instance_to_mappings; + std::map mapping_to_instance; + std::vector handled(mappings.size(), false); + + // See case of failed instance creation below + auto tighten_write_reqs = [&]() { + for (int32_t mapping_idx = 0; mapping_idx < mappings.size(); ++mapping_idx) { + auto& mapping = mappings[mapping_idx]; + PrivilegeMode priv = LEGION_NO_ACCESS; +#ifdef DEBUG_LEGATE + std::stringstream reqs_ss; +#endif + for (auto req_idx : mapping.requirement_indices()) { + const RegionRequirement& req = task.regions[req_idx]; + if (!req.region.exists()) continue; + priv |= req.privilege; +#ifdef DEBUG_LEGATE + reqs_ss << " " << req_idx; +#endif + } + if (!(priv & LEGION_WRITE_PRIV) || mapping.policy.exact) continue; +#ifdef DEBUG_LEGATE + logger.debug() << "Task " << task.get_unique_id() + << ": tightened mapping policy for reqs:" << reqs_ss.str(); +#endif + mapping.policy.exact = true; + if (!handled[mapping_idx]) continue; + handled[mapping_idx] = false; + auto m2i_it = mapping_to_instance.find(mapping_idx); + if (m2i_it == mapping_to_instance.end()) continue; + PhysicalInstance inst = m2i_it->second; + mapping_to_instance.erase(m2i_it); + auto i2m_it = instance_to_mappings.find(inst); + i2m_it->second.erase(mapping_idx); + if (i2m_it->second.empty()) { + runtime->release_instance(ctx, inst); + instance_to_mappings.erase(i2m_it); + } + } + }; + + // Mapping each field separately for each of the logical regions + for (int32_t mapping_idx = 0; mapping_idx < mappings.size(); ++mapping_idx) { + if (handled[mapping_idx]) continue; auto& mapping = mappings[mapping_idx]; auto req_indices = mapping.requirement_indices(); @@ -615,12 +661,14 @@ void BaseMapper::map_task(const MapperContext ctx, if (target == StoreTarget::FBMEM) target = StoreTarget::ZCMEM; #endif output.future_locations.push_back(get_target_memory(task.target_proc, target)); + handled[mapping_idx] = true; continue; - } else if (mapping.for_unbound_stores()) { + } + + if (mapping.for_unbound_stores()) { for (auto req_idx : req_indices) { output.output_targets[req_idx] = get_target_memory(task.target_proc, mapping.policy.target); auto ndim = mapping.stores.front().dim(); - // FIXME: Unbound stores can have more than one dimension later std::vector dimension_ordering; for (int32_t dim = ndim - 1; dim >= 0; --dim) @@ -630,65 +678,79 @@ void BaseMapper::map_task(const MapperContext ctx, output.output_constraints[req_idx].ordering_constraint = OrderingConstraint(dimension_ordering, false); } + handled[mapping_idx] = true; continue; } std::vector> reqs; +#ifdef DEBUG_LEGATE + std::stringstream reqs_ss; +#endif for (auto req_idx : req_indices) { const auto& req = task.regions[req_idx]; if (!req.region.exists()) continue; reqs.push_back(std::cref(req)); +#ifdef DEBUG_LEGATE + reqs_ss << " " << req_idx; +#endif + } + if (reqs.empty()) { + handled[mapping_idx] = true; + continue; } - if (reqs.empty()) continue; - - // Get the reference to our valid instances in case we decide to use them + // Get an instance and acquire it if necessary. If the acquire fails then prune it from the + // mapper's data structures and retry, until we succeed or map_legate_store fails with an out of + // memory error. PhysicalInstance result; - if (map_legate_store(ctx, task, mapping, reqs, task.target_proc, result)) - needed_acquires.push_back(result); - - for (auto req_idx : req_indices) output.chosen_instances[req_idx].push_back(result); - instances_to_mappings[result].insert(mapping_idx); - } - - // Do an acquire on all the instances so we have our result - // Keep doing this until we succed or we get an out of memory error - while (!needed_acquires.empty() && - !runtime->acquire_and_filter_instances(ctx, needed_acquires, true /*filter on acquire*/)) { - assert(!needed_acquires.empty()); - // If we failed to acquire any of the instances we need to prune them - // out of the mapper's data structure so do that first - std::set failed_acquires; - filter_failed_acquires(ctx, needed_acquires, failed_acquires); - - for (auto failed_acquire : failed_acquires) { - auto affected_mappings = instances_to_mappings[failed_acquire]; - instances_to_mappings.erase(failed_acquire); - - for (auto& mapping_idx : affected_mappings) { - auto& mapping = mappings[mapping_idx]; - auto req_indices = mapping.requirement_indices(); - - std::vector> reqs; - for (auto req_idx : req_indices) reqs.push_back(std::cref(task.regions[req_idx])); - - for (auto req_idx : req_indices) { - auto& instances = output.chosen_instances[req_idx]; - uint32_t inst_idx = 0; - for (; inst_idx < instances.size(); ++inst_idx) - if (instances[inst_idx] == failed_acquire) break; - instances.erase(instances.begin() + inst_idx); - } - - PhysicalInstance result; - if (map_legate_store(ctx, task, mapping, reqs, task.target_proc, result)) - needed_acquires.push_back(result); - - for (auto req_idx : req_indices) output.chosen_instances[req_idx].push_back(result); - instances_to_mappings[result].insert(mapping_idx); + while (map_legate_store(ctx, task, mapping, reqs, task.target_proc, result, can_fail)) { + if (result == PhysicalInstance()) break; + if (instance_to_mappings.count(result) > 0 || runtime->acquire_instance(ctx, result)) { +#ifdef DEBUG_LEGATE + logger.debug() << "Task " << task.get_unique_id() << ": acquired instance " << result + << " for reqs:" << reqs_ss.str(); +#endif + break; } +#ifdef DEBUG_LEGATE + logger.debug() << "Task " << task.get_unique_id() << ": failed to acquire instance " << result + << " for reqs:" << reqs_ss.str(); +#endif + AutoLock lock(ctx, local_instances->manager_lock()); + local_instances->erase(result); } + + // If instance creation failed we try mapping all stores again, but request tight instances for + // write requirements. The hope is that these write requirements cover the entire region (i.e. + // they use a complete partition), so the new tight instances will invalidate any pre-existing + // "bloated" instances for the same region, freeing up enough memory so that mapping can succeed + if (result == PhysicalInstance()) { +#ifdef DEBUG_LEGATE + logger.debug() << "Task " << task.get_unique_id() + << ": failed mapping for reqs:" << reqs_ss.str(); +#endif + assert(can_fail); + tighten_write_reqs(); + mapping_idx = -1; + can_fail = false; + continue; + } + + // Success; record the instance for this mapping. +#ifdef DEBUG_LEGATE + logger.debug() << "Task " << task.get_unique_id() + << ": completed mapping for reqs:" << reqs_ss.str(); +#endif + instance_to_mappings[result].insert(mapping_idx); + mapping_to_instance[mapping_idx] = result; + handled[mapping_idx] = true; } + + // Succeeded in mapping all stores, record it on map_task output. + for (const auto& m2i : mapping_to_instance) + for (auto req_idx : mappings[m2i.first].requirement_indices()) + if (task.regions[req_idx].region.exists()) + output.chosen_instances[req_idx].push_back(m2i.second); } void BaseMapper::map_replicate_task(const MapperContext ctx, @@ -747,7 +809,8 @@ bool BaseMapper::map_legate_store(const MapperContext ctx, const StoreMapping& mapping, std::vector> reqs, Processor target_proc, - PhysicalInstance& result) + PhysicalInstance& result, + bool can_fail) { const auto& policy = mapping.policy; std::vector regions; @@ -776,12 +839,29 @@ bool BaseMapper::map_legate_store(const MapperContext ctx, // If we're making a reduction instance, we should just make it now if (redop != 0) { layout_constraints.add_constraint(SpecializedConstraint(REDUCTION_FOLD_SPECIALIZE, redop)); - - if (!runtime->create_physical_instance( - ctx, target_memory, layout_constraints, regions, result, true /*acquire*/)) + size_t footprint = 0; + if (runtime->create_physical_instance(ctx, + target_memory, + layout_constraints, + regions, + result, + true /*acquire*/, + LEGION_GC_DEFAULT_PRIORITY, + false /*tight bounds*/, + &footprint)) { +#ifdef DEBUG_LEGATE + Realm::LoggerMessage msg = logger.debug(); + msg << "Operation " << mappable.get_unique_id() << ": created reduction instance " << result + << " for"; + for (LogicalRegion r : regions) msg << " " << r; + msg << " (size: " << footprint << " bytes, memory: " << target_memory << ")"; +#endif + // We already did the acquire + return false; + } + if (!can_fail) report_failed_mapping(mappable, mapping.requirement_index(), target_memory, redop); - // We already did the acquire - return false; + return true; } auto& fields = layout_constraints.field_constraint.field_set; @@ -797,8 +877,8 @@ bool BaseMapper::map_legate_store(const MapperContext ctx, local_instances->find_instance( regions.front(), fields.front(), target_memory, result, policy)) { #ifdef DEBUG_LEGATE - logger.debug() << get_mapper_name() << " found instance " << result << " for " - << regions.front(); + logger.debug() << "Operation " << mappable.get_unique_id() << ": reused cached instance " + << result << " for " << regions.front(); #endif runtime->enable_reentrant(ctx); // Needs acquire to keep the runtime happy @@ -861,8 +941,12 @@ bool BaseMapper::map_legate_store(const MapperContext ctx, assert(result.exists()); #ifdef DEBUG_LEGATE if (created) { - logger.debug() << get_mapper_name() << " created instance " << result << " for " << *group - << " (size: " << footprint << " bytes, memory: " << target_memory << ")"; + logger.debug() << "Operation " << mappable.get_unique_id() << ": created instance " << result + << " for " << *group << " (size: " << footprint + << " bytes, memory: " << target_memory << ")"; + } else { + logger.debug() << "Operation " << mappable.get_unique_id() << ": found instance " << result + << " for " << *group; } #endif // Only save the result for future use if it is not an external instance @@ -879,8 +963,10 @@ bool BaseMapper::map_legate_store(const MapperContext ctx, runtime->enable_reentrant(ctx); // If we make it here then we failed entirely - auto req_indices = mapping.requirement_indices(); - for (auto req_idx : req_indices) report_failed_mapping(mappable, req_idx, target_memory, redop); + if (!can_fail) { + auto req_indices = mapping.requirement_indices(); + for (auto req_idx : req_indices) report_failed_mapping(mappable, req_idx, target_memory, redop); + } return true; } diff --git a/src/core/mapping/base_mapper.h b/src/core/mapping/base_mapper.h index fac2c73040..d818984115 100644 --- a/src/core/mapping/base_mapper.h +++ b/src/core/mapping/base_mapper.h @@ -268,7 +268,8 @@ class BaseMapper : public Legion::Mapping::Mapper, public LegateMapper { const StoreMapping& mapping, std::vector> reqs, Legion::Processor target_proc, - Legion::Mapping::PhysicalInstance& result); + Legion::Mapping::PhysicalInstance& result, + bool can_fail); bool map_raw_array(const Legion::Mapping::MapperContext ctx, const Legion::Mappable& mappable, unsigned index, From 9da96ee17cb4b49346db5a5361398969f741f780 Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Tue, 11 Oct 2022 15:08:12 -0700 Subject: [PATCH 0093/1425] Release tasks 22.10 (#428) Resolved conflicts with main. It seems that there was a divergence between the main commits and the dev branch commits at some point, so the merge is deceptively large. However, the only change is to hardcode the version of Legion and to resolve conflicts. --- install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.py b/install.py index 6566ca68b9..3e684463de 100755 --- a/install.py +++ b/install.py @@ -742,7 +742,7 @@ def driver(): "--legion-branch", dest="legion_branch", required=False, - default="control_replication", + default="da1b0fb", help="Legion branch to build Legate with.", ) args, unknown = parser.parse_known_args() From 6c718de3c7a986a8a5003d0964dfcc810756bc6b Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Fri, 27 Jan 2023 10:31:48 -0800 Subject: [PATCH 0094/1425] Update the architectures built in conda package (#545) (#546) Co-authored-by: Marcin Zalewski --- conda/conda-build/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/conda-build/build.sh b/conda/conda-build/build.sh index 27b5aead18..317947dc4d 100644 --- a/conda/conda-build/build.sh +++ b/conda/conda-build/build.sh @@ -16,7 +16,7 @@ CMAKE_ARGS+=" if [ -z "$CPU_ONLY" ]; then CMAKE_ARGS+=" -DLegion_USE_CUDA=ON --DCMAKE_CUDA_ARCHITECTURES:LIST=60-real;70-real;75-real;80-real;86 +-DCMAKE_CUDA_ARCHITECTURES:LIST=60-real;70-real;75-real;80-real;90 " fi From 90153d289519437f22393af03ed077b6fad3e0a7 Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Mon, 30 Jan 2023 13:55:09 -0800 Subject: [PATCH 0095/1425] Revert "Update the architectures built in conda package (#545) (#546)" (#550) This reverts commit 6c718de3c7a986a8a5003d0964dfcc810756bc6b. Co-authored-by: Marcin Zalewski --- conda/conda-build/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/conda-build/build.sh b/conda/conda-build/build.sh index 317947dc4d..27b5aead18 100644 --- a/conda/conda-build/build.sh +++ b/conda/conda-build/build.sh @@ -16,7 +16,7 @@ CMAKE_ARGS+=" if [ -z "$CPU_ONLY" ]; then CMAKE_ARGS+=" -DLegion_USE_CUDA=ON --DCMAKE_CUDA_ARCHITECTURES:LIST=60-real;70-real;75-real;80-real;90 +-DCMAKE_CUDA_ARCHITECTURES:LIST=60-real;70-real;75-real;80-real;86 " fi From 3b5f69a372e23d26e574475663212ea8977bd17d Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Mon, 30 Jan 2023 14:14:04 -0800 Subject: [PATCH 0096/1425] Fix the default Legion version (#547) Co-authored-by: Marcin Zalewski --- install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.py b/install.py index e2506a68da..56c508ad96 100755 --- a/install.py +++ b/install.py @@ -743,7 +743,7 @@ def driver(): "--legion-branch", dest="legion_branch", required=False, - default="collective", + default="04cf06a2", help="Legion branch to build Legate with.", ) args, unknown = parser.parse_known_args() From 8cfd8301a9a5aa676519b002ee5b37dedc2f9438 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Thu, 23 Feb 2023 11:22:39 -0800 Subject: [PATCH 0097/1425] Make sure to retrieve tunables after start-up in the standalone mode --- src/core/runtime/runtime.cc | 19 ++++++++++++++++--- src/core/runtime/runtime.h | 4 ++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 6be6689243..0e9e090f9e 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -53,6 +53,8 @@ static const char* const core_library_name = "legate.core"; /*static*/ bool Core::has_socket_mem = false; +/*static*/ bool Core::standalone = false; + /*static*/ LegateMainFnPtr Core::main_fn = nullptr; /*static*/ void Core::parse_config(void) @@ -174,6 +176,15 @@ static void extract_scalar_task( LEGATE_ABORT; } +/*static*/ void Core::retrieve_tunable(Legion::Context legion_context, + Legion::Runtime* legion_runtime, + LibraryContext* context) +{ + auto fut = legion_runtime->select_tunable_value( + legion_context, LEGATE_CORE_TUNABLE_HAS_SOCKET_MEM, context->get_mapper_id(0)); + Core::has_socket_mem = fut.get_result(); +} + void register_legate_core_tasks(Machine machine, Legion::Runtime* runtime, const LibraryContext& context) @@ -252,15 +263,16 @@ extern void register_exception_reduction_op(Legion::Runtime* runtime, register_legate_core_sharding_functors(legion_runtime, *core_lib); - auto fut = legion_runtime->select_tunable_value( - Legion::Runtime::get_context(), LEGATE_CORE_TUNABLE_HAS_SOCKET_MEM, core_lib->get_mapper_id(0)); - Core::has_socket_mem = fut.get_result(); + if (!Core::standalone) + Core::retrieve_tunable(Legion::Runtime::get_context(), legion_runtime, core_lib); } /*static*/ void core_library_bootstrapping_callback(Machine machine, Legion::Runtime* legion_runtime, const std::set& local_procs) { + Core::standalone = true; + core_library_registration_callback(machine, legion_runtime, local_procs); auto runtime = Runtime::get_runtime(); @@ -632,6 +644,7 @@ void Runtime::post_startup_initialization(Legion::Context legion_context) legion_context_ = legion_context; core_context_ = find_library(core_library_name); partition_manager_ = new PartitionManager(this, core_context_); + Core::retrieve_tunable(legion_context_, legion_runtime_, core_context_); } // This function should be moved to the library context diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 011275c7b2..fdb12e4d04 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -42,6 +42,9 @@ class Core { Legion::Runtime* runtime, const char* task_name); static void report_unexpected_exception(const char* task_name, const legate::TaskException& e); + static void retrieve_tunable(Legion::Context legion_context, + Legion::Runtime* legion_runtime, + LibraryContext* context); public: // Configuration settings @@ -50,6 +53,7 @@ class Core { static bool synchronize_stream_view; static bool log_mapping_decisions; static bool has_socket_mem; + static bool standalone; static LegateMainFnPtr main_fn; }; From 350a1d4143c9839af090b7e326ed1b2fe46f04aa Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Thu, 23 Feb 2023 11:23:48 -0800 Subject: [PATCH 0098/1425] Fix launch domain resolver for single-processor cases --- src/core/partitioning/partitioner.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/partitioning/partitioner.cc b/src/core/partitioning/partitioner.cc index 644971b97b..9eec7c4a44 100644 --- a/src/core/partitioning/partitioner.cc +++ b/src/core/partitioning/partitioner.cc @@ -68,7 +68,7 @@ void LaunchDomainResolver::record_unbound_store(int32_t unbound_dim) std::unique_ptr LaunchDomainResolver::resolve_launch_domain() const { - if (must_be_sequential_) return nullptr; + if (must_be_sequential_ || launch_domains_.empty()) return nullptr; if (must_be_1d_) { if (unbound_dim_ != UNSET && unbound_dim_ > 1) return nullptr; From be3b1992ba3ff8af8af8e09f6316f6fad2ab43a8 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 1 Mar 2023 01:03:36 -0800 Subject: [PATCH 0099/1425] Recover a line that was lost during merge --- src/core/task/registrar.cc | 3 ++- src/core/task/registrar.h | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/core/task/registrar.cc b/src/core/task/registrar.cc index 3386d20104..7949aea404 100644 --- a/src/core/task/registrar.cc +++ b/src/core/task/registrar.cc @@ -81,11 +81,12 @@ void TaskRegistrar::record_variant(Legion::TaskID tid, pending_task_variants_.push_back(registrar); } -void TaskRegistrar::register_all_tasks(const LibraryContext& context) +void TaskRegistrar::register_all_tasks(LibraryContext& context) { auto runtime = Legion::Runtime::get_runtime(); // Do all our registrations for (auto& task : pending_task_variants_) { + context.record_task_name(task->task_id, task->task_name); task->task_id = context.get_task_id(task->task_id); // Convert a task local task id to a global id // Attach the task name too for debugging diff --git a/src/core/task/registrar.h b/src/core/task/registrar.h index ad7e3ddb35..73304e2749 100644 --- a/src/core/task/registrar.h +++ b/src/core/task/registrar.h @@ -40,7 +40,7 @@ class TaskRegistrar { const VariantOptions& options); public: - void register_all_tasks(const LibraryContext& context); + void register_all_tasks(LibraryContext& context); private: std::vector pending_task_variants_; From d3cc8afa6c7139c425795805687c6a4fdfe2e218 Mon Sep 17 00:00:00 2001 From: Mark Vaz Date: Tue, 7 Mar 2023 16:19:48 -0800 Subject: [PATCH 0100/1425] Add support for Python 3.11 (#608) * Updating for py311 * Document support for python 3.11 --------- Co-authored-by: Manolis Papadakis --- conda/conda-build/conda_build_config.yaml | 1 + scripts/generate-conda-envs.py | 2 +- setup.py | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/conda/conda-build/conda_build_config.yaml b/conda/conda-build/conda_build_config.yaml index 00fe47139c..81326c39a6 100644 --- a/conda/conda-build/conda_build_config.yaml +++ b/conda/conda-build/conda_build_config.yaml @@ -5,6 +5,7 @@ gpu_enabled: python: - "3.9,!=3.9.7" - 3.10 + - 3.11 numpy_version: - ">=1.22" diff --git a/scripts/generate-conda-envs.py b/scripts/generate-conda-envs.py index 7743914f48..18e35d2b3d 100755 --- a/scripts/generate-conda-envs.py +++ b/scripts/generate-conda-envs.py @@ -220,7 +220,7 @@ def filename(self) -> str: # --- Setup ------------------------------------------------------------------- -PYTHON_VERSIONS = ("3.9", "3.10") +PYTHON_VERSIONS = ("3.9", "3.10", "3.11") CTK_VERSIONS = ( "none", diff --git a/setup.py b/setup.py index f03a05cc46..83912f31f8 100755 --- a/setup.py +++ b/setup.py @@ -35,6 +35,7 @@ "Programming Language :: Python", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ], extras_require={ "test": [ From 8f5dd49ef4f8b173b9be55353428929b5b357ae3 Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Tue, 7 Mar 2023 17:18:46 -0800 Subject: [PATCH 0101/1425] Tutorial editable install fix (#610) (#613) * replace hello => @target@ * fix library paths for editable installs * deconflict user and auto skbuild cmake args * clean up rapids_export calls in project template --------- Co-authored-by: Jeremy Co-authored-by: Mads R. B. Kristensen --- cmake/legate_helper_functions.cmake | 35 ++++++++++++++++++++--------- examples/hello/setup.py | 5 ++++- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/cmake/legate_helper_functions.cmake b/cmake/legate_helper_functions.cmake index 79f2d1c1f7..db269fa787 100644 --- a/cmake/legate_helper_functions.cmake +++ b/cmake/legate_helper_functions.cmake @@ -38,11 +38,16 @@ function(legate_default_cpp_install target) DESTINATION ${lib_dir} EXPORT ${LEGATE_OPT_EXPORT}) + set(final_code_block + "set(${target}_BUILD_LIBDIR ${CMAKE_BINARY_DIR}/legate_${target})" + ) + rapids_export( INSTALL ${target} EXPORT_SET ${LEGATE_OPT_EXPORT} GLOBAL_TARGETS ${target} NAMESPACE legate:: + LANGUAGES ${ENABLED_LANGUAGES} ) # build export targets @@ -51,6 +56,8 @@ function(legate_default_cpp_install target) EXPORT_SET ${LEGATE_OPT_EXPORT} GLOBAL_TARGETS ${target} NAMESPACE legate:: + FINAL_CODE_BLOCK final_code_block + LANGUAGES ${ENABLED_LANGUAGES} ) endfunction() @@ -90,7 +97,7 @@ def get_libpath(): }[platform.system()] def find_lib(libdir): - target = f"libhello{so_ext}*" + target = f"lib@target@{so_ext}*" search_path = Path(libdir) matches = [m for m in search_path.rglob(target)] if matches: @@ -138,14 +145,19 @@ header: str = """ @ONLY ) - # cmake doesn't let you pipe the output of a - # custom command so you have to generate a script - # that writes to a file and THEN run that in a - # custom command + if (DEFINED ${target}_BUILD_LIBDIR) + # this must have been imported from an existing editable build + set(libdir ${${target}_BUILD_LIBDIR}) + else() + # libraries are built in a common spot + set(libdir ${CMAKE_BINARY_DIR}/legate_${target}) + message("libdir to binary dir") + endif() add_custom_target("generate_install_info_py" ALL COMMAND ${CMAKE_COMMAND} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -Dtarget=${target} + -Dlibdir=${libdir} -P ${generate_script} OUTPUT ${install_info_py} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} @@ -229,6 +241,9 @@ function(legate_add_cpp_subdirectory dir) if (NOT ${target}_FOUND) add_subdirectory(${dir} ${CMAKE_BINARY_DIR}/legate_${target}) legate_default_cpp_install(${target} EXPORT ${LEGATE_OPT_EXPORT}) + else() + # Make sure the libdir is visible to other functions + set(${target}_BUILD_LIBDIR "${${target}_BUILD_LIBDIR}" PARENT_SCOPE) endif() else() add_subdirectory(${dir} ${CMAKE_BINARY_DIR}/legate_${target}) @@ -320,9 +335,9 @@ class Mapper : public legate::mapping::LegateMapper { const legate::mapping::MachineQueryInterface* machine_; }; -static const char* const library_name = "hello"; +static const char* const library_name = "@target@"; -Legion::Logger log_hello("hello"); +Legion::Logger log_@target@(library_name); /*static*/ legate::TaskRegistrar& Registry::get_registrar() { @@ -392,16 +407,16 @@ class UserLibrary(Library): return self.name def get_shared_library(self) -> str: - from hello.install_info import libpath + from @target@.install_info import libpath return os.path.join(libpath, f"lib@target@{self.get_library_extension()}") def get_c_header(self) -> str: - from hello.install_info import header + from @target@.install_info import header return header def get_registration_callback(self) -> str: - return "hello_perform_registration" + return "@target@_perform_registration" def get_resource_configuration(self) -> ResourceConfig: assert self.shared_object is not None diff --git a/examples/hello/setup.py b/examples/hello/setup.py index fc918c8438..f919989b5a 100644 --- a/examples/hello/setup.py +++ b/examples/hello/setup.py @@ -28,7 +28,10 @@ f"-Dlegate_core_ROOT:STRING={legate_dir}", ] -os.environ["SKBUILD_CONFIGURE_OPTIONS"] = " ".join(cmake_flags) +env_cmake_args = os.environ.get("CMAKE_ARGS") +if env_cmake_args is not None: + cmake_flags.append(env_cmake_args) +os.environ["CMAKE_ARGS"] = " ".join(cmake_flags) setup( From dba7143d188f78a0613d72afb0f3647139f02eb0 Mon Sep 17 00:00:00 2001 From: Marcin Zalewski Date: Tue, 14 Mar 2023 17:46:40 -0700 Subject: [PATCH 0102/1425] Update Legion SHA (bug fix in Legion) (#622) Co-authored-by: Marcin Zalewski --- cmake/versions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/versions.json b/cmake/versions.json index c5bf2d31ab..f78d94414a 100644 --- a/cmake/versions.json +++ b/cmake/versions.json @@ -7,7 +7,7 @@ }, "Legion": { "git_url" : "https://gitlab.com/StanfordLegion/legion.git", - "git_tag" : "e1f1ef61e29c3160419d0cd528950b2d565c2a0d" + "git_tag" : "b66b612ecd33f28d755e28bf8ae82b6589464d4a" } } } From b18b0fc897e7f7ee239b691533ff84a51584b444 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Thu, 23 Mar 2023 23:36:53 -0700 Subject: [PATCH 0103/1425] Don't take a pointer to the std::vector before resizing it See merge request legate/legate.core.internal!5 --- src/core/utilities/buffer_builder.cc | 5 +++-- src/core/utilities/buffer_builder.h | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/core/utilities/buffer_builder.cc b/src/core/utilities/buffer_builder.cc index 9068921e76..14bcc78b19 100644 --- a/src/core/utilities/buffer_builder.cc +++ b/src/core/utilities/buffer_builder.cc @@ -1,4 +1,4 @@ -/* Copyright 2021 NVIDIA Corporation +/* Copyright 2021-2023 NVIDIA Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,8 +26,9 @@ BufferBuilder::BufferBuilder() void BufferBuilder::pack_buffer(const void* src, size_t size) { - auto tgt = buffer_.data() + buffer_.size(); + auto off = buffer_.size(); buffer_.resize(buffer_.size() + size); + auto tgt = buffer_.data() + off; memcpy(tgt, src, size); } diff --git a/src/core/utilities/buffer_builder.h b/src/core/utilities/buffer_builder.h index 7b10975c1e..4c7bef538c 100644 --- a/src/core/utilities/buffer_builder.h +++ b/src/core/utilities/buffer_builder.h @@ -1,4 +1,4 @@ -/* Copyright 2021 NVIDIA Corporation +/* Copyright 2021-2023 NVIDIA Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 559cae78c8e275f00b2d359307b8c32c74d3346d Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Tue, 28 Mar 2023 20:45:05 -0700 Subject: [PATCH 0104/1425] Manual task launch API * Support for multiple scalar outputs and reductions * Manual task launch API * Factor out tuple to Shape so we can later add deferred shapes * Extend the test to use the manual tiling API * Initial implementation for the manual tiling API * Write a C++ example to which a manual API test will be added See merge request legate/legate.core.internal!6 --- legate_core_cpp.cmake | 1 + src/core/data/logical_store.cc | 19 +- src/core/data/logical_store.h | 22 +- src/core/data/logical_store_detail.cc | 97 ++++++- src/core/data/logical_store_detail.h | 60 ++++- src/core/data/shape.cc | 33 +++ src/core/data/shape.h | 29 +++ src/core/data/store.cc | 16 ++ src/core/data/store.h | 5 + src/core/partitioning/partition.cc | 31 +-- src/core/partitioning/partition.h | 4 +- src/core/partitioning/partitioner.cc | 8 + src/core/partitioning/partitioner.h | 2 + src/core/runtime/launcher.cc | 87 ++++--- src/core/runtime/launcher.h | 13 +- src/core/runtime/operation.cc | 241 ++++++++++++++---- src/core/runtime/operation.h | 112 +++++--- src/core/runtime/runtime.cc | 88 +++++-- src/core/runtime/runtime.h | 42 +-- tests/cpp/.clang-format | 129 ++++++++++ .../integration/manual_simple/CMakeLists.txt | 31 +++ tests/cpp/integration/manual_simple/build.sh | 5 + .../manual_simple/manual_simple.cc | 142 +++++++++++ .../multi_scalar_out/CMakeLists.txt | 31 +++ .../cpp/integration/multi_scalar_out/build.sh | 5 + .../multi_scalar_out/multi_scalar_out.cc | 194 ++++++++++++++ 26 files changed, 1242 insertions(+), 205 deletions(-) create mode 100644 src/core/data/shape.cc create mode 100644 src/core/data/shape.h create mode 100644 tests/cpp/.clang-format create mode 100644 tests/cpp/integration/manual_simple/CMakeLists.txt create mode 100755 tests/cpp/integration/manual_simple/build.sh create mode 100644 tests/cpp/integration/manual_simple/manual_simple.cc create mode 100644 tests/cpp/integration/multi_scalar_out/CMakeLists.txt create mode 100755 tests/cpp/integration/multi_scalar_out/build.sh create mode 100644 tests/cpp/integration/multi_scalar_out/multi_scalar_out.cc diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index b7aa401e70..da23ab33fe 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -196,6 +196,7 @@ list(APPEND legate_core_SOURCES src/core/data/logical_store.cc src/core/data/logical_store_detail.cc src/core/data/scalar.cc + src/core/data/shape.cc src/core/data/store.cc src/core/data/transform.cc src/core/mapping/base_mapper.cc diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index 0464de67c5..8771857172 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -52,7 +52,7 @@ int32_t LogicalStore::dim() const { return impl_->dim(); } LegateTypeCode LogicalStore::code() const { return impl_->code(); } -const tuple& LogicalStore::extents() const { return impl_->extents(); } +const Shape& LogicalStore::extents() const { return impl_->extents(); } size_t LogicalStore::volume() const { return impl_->volume(); } @@ -66,9 +66,26 @@ LogicalStore LogicalStore::project(int32_t dim, int64_t index) const return LogicalStore(impl_->project(dim, index)); } +LogicalStorePartition LogicalStore::partition_by_tiling(std::vector tile_shape) const +{ + return LogicalStorePartition(impl_->partition_by_tiling(Shape(std::move(tile_shape)))); +} + std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) { return impl_->get_physical_store(context); } +LogicalStorePartition::LogicalStorePartition(std::shared_ptr&& impl) + : impl_(std::forward(impl)) +{ +} + +LogicalStore LogicalStorePartition::store() const { return LogicalStore(impl_->store()); } + +std::shared_ptr LogicalStorePartition::partition() const +{ + return impl_->storage_partition()->partition(); +} + } // namespace legate diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index 2d05144a97..bce92ae5af 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -20,6 +20,7 @@ #include "legion.h" +#include "core/data/shape.h" #include "core/data/transform.h" #include "core/utilities/typedefs.h" @@ -27,6 +28,7 @@ namespace legate { class BufferBuilder; class LibraryContext; +class LogicalStorePartition; class Partition; class Projection; class Runtime; @@ -35,12 +37,14 @@ class Store; namespace detail { class LogicalStore; +class LogicalStorePartition; } // namespace detail class LogicalStore { private: friend class Runtime; + friend class LogicalStorePartition; LogicalStore(std::shared_ptr&& impl); public: @@ -54,13 +58,16 @@ class LogicalStore { public: int32_t dim() const; LegateTypeCode code() const; - const tuple& extents() const; + const Shape& extents() const; size_t volume() const; public: LogicalStore promote(int32_t extra_dim, size_t dim_size) const; LogicalStore project(int32_t dim, int64_t index) const; + public: + LogicalStorePartition partition_by_tiling(std::vector tile_shape) const; + public: std::shared_ptr get_physical_store(LibraryContext* context); @@ -71,4 +78,17 @@ class LogicalStore { std::shared_ptr impl_{nullptr}; }; +class LogicalStorePartition { + private: + friend class LogicalStore; + LogicalStorePartition(std::shared_ptr&& impl); + + public: + LogicalStore store() const; + std::shared_ptr partition() const; + + private: + std::shared_ptr impl_{nullptr}; +}; + } // namespace legate diff --git a/src/core/data/logical_store_detail.cc b/src/core/data/logical_store_detail.cc index 23300c42a3..4670fa777e 100644 --- a/src/core/data/logical_store_detail.cc +++ b/src/core/data/logical_store_detail.cc @@ -34,12 +34,13 @@ namespace detail { Storage::Storage(int32_t dim, LegateTypeCode code) : unbound_(true), dim_(dim), code_(code) {} -Storage::Storage(tuple extents, LegateTypeCode code) +Storage::Storage(Shape extents, LegateTypeCode code, bool optimize_scalar) : dim_(extents.size()), extents_(extents), code_(code), volume_(extents.volume()) { + if (optimize_scalar && volume_ == 1) kind_ = Kind::FUTURE; } -Storage::Storage(tuple extents, LegateTypeCode code, const Legion::Future& future) +Storage::Storage(Shape extents, LegateTypeCode code, const Legion::Future& future) : dim_(extents.size()), extents_(extents), code_(code), @@ -74,7 +75,7 @@ void Storage::set_region_field(std::shared_ptr&& region_fiel unbound_ = false; region_field_ = std::move(region_field); - // TODO: this is a blocking operator + // TODO: this is a blocking operation auto domain = region_field_->domain(); auto lo = domain.lo(); auto hi = domain.hi(); @@ -83,6 +84,8 @@ void Storage::set_region_field(std::shared_ptr&& region_fiel extents_ = extents; } +void Storage::set_future(Legion::Future future) { future_ = future; } + RegionField Storage::map(LibraryContext* context) { #ifdef DEBUG_LEGATE @@ -124,6 +127,21 @@ Legion::LogicalPartition Storage::find_or_create_legion_partition(const Partitio region, partition->is_disjoint_for(nullptr), partition->is_complete_for(nullptr)); } +std::shared_ptr Storage::create_partition(std::shared_ptr partition) +{ + return std::make_shared(shared_from_this(), std::move(partition)); +} + +//////////////////////////////////////////////////// +// legate::detail::StoragePartition +//////////////////////////////////////////////////// + +StoragePartition::StoragePartition(std::shared_ptr parent, + std::shared_ptr partition) + : parent_(std::move(parent)), partition_(std::move(partition)) +{ +} + //////////////////////////////////////////////////// // legate::detail::LogicalStore //////////////////////////////////////////////////// @@ -141,7 +159,7 @@ LogicalStore::LogicalStore(std::shared_ptr&& storage) if (!unbound()) extents_ = storage_->extents(); } -LogicalStore::LogicalStore(tuple&& extents, +LogicalStore::LogicalStore(Shape&& extents, const std::shared_ptr& storage, std::shared_ptr&& transform) : store_id_(Runtime::get_runtime()->get_unique_store_id()), @@ -163,7 +181,7 @@ LogicalStore::~LogicalStore() bool LogicalStore::unbound() const { return storage_->unbound(); } -const tuple& LogicalStore::extents() const { return extents_; } +const Shape& LogicalStore::extents() const { return extents_; } size_t LogicalStore::volume() const { return extents_.volume(); } @@ -196,10 +214,21 @@ Legion::Future LogicalStore::get_future() { return storage_->get_future(); } void LogicalStore::set_region_field(std::shared_ptr&& region_field) { +#ifdef DEBUG_LEGATE + assert(!scalar()); +#endif storage_->set_region_field(std::move(region_field)); extents_ = storage_->extents(); } +void LogicalStore::set_future(Legion::Future future) +{ +#ifdef DEBUG_LEGATE + assert(scalar()); +#endif + storage_->set_future(future); +} + std::shared_ptr LogicalStore::promote(int32_t extra_dim, size_t dim_size) const { if (extra_dim < 0 || extra_dim > dim()) { @@ -226,14 +255,45 @@ std::shared_ptr LogicalStore::project(int32_t d, int64_t index) co return std::make_shared(std::move(new_extents), storage_, std::move(transform)); } +std::shared_ptr LogicalStore::partition_by_tiling(Shape tile_shape) +{ + if (tile_shape.size() != extents_.size()) { + log_legate.error("Incompatible tile shape: expected a %zd-tuple, got a %zd-tuple", + extents_.size(), + tile_shape.size()); + LEGATE_ABORT; + } + Shape color_shape(extents_); + // TODO: This better use std::transform + for (size_t idx = 0; idx < tile_shape.size(); ++idx) + color_shape[idx] = (color_shape[idx] + tile_shape[idx] - 1) / tile_shape[idx]; + auto partition = create_tiling(std::move(tile_shape), std::move(color_shape)); + return create_partition(std::move(partition)); +} + std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) { - // TODO: Need to support inline mapping for scalars - assert(storage_->kind() == Storage::Kind::REGION_FIELD); + if (unbound()) { + log_legate.error("Unbound store cannot be inlined mapped"); + LEGATE_ABORT; + } if (nullptr != mapped_) return mapped_; + if (storage_->kind() == Storage::Kind::FUTURE) { + // TODO: future wrappers from inline mappings are read-only for now + auto field_size = type_dispatch(code(), elem_size_fn{}); + auto domain = to_domain(storage_->extents()); + FutureWrapper future(true, field_size, domain, storage_->get_future()); + // Physical stores for future-backed stores shouldn't be cached, as they are not automatically + // remapped to reflect changes by the runtime. + return std::make_shared(dim(), code(), -1, future, transform_); + } + +#ifdef DEBUG_LEGATE + assert(storage_->kind() == Storage::Kind::REGION_FIELD); +#endif auto region_field = storage_->map(context); mapped_ = std::make_shared(dim(), code(), -1, std::move(region_field), transform_); - return std::move(mapped_); + return mapped_; } Legion::ProjectionID LogicalStore::compute_projection(int32_t launch_ndim) const @@ -304,6 +364,18 @@ void LogicalStore::set_key_partition(const Partition* partition) void LogicalStore::reset_key_partition() { storage_->reset_key_partition(); } +std::shared_ptr LogicalStore::create_partition( + std::shared_ptr partition) +{ + if (unbound()) { + log_legate.error("Unbound store cannot be manually partitioned"); + LEGATE_ABORT; + } + // TODO: the partition here should be inverted by the transform + auto storage_partition = storage_->create_partition(partition); + return std::make_shared(std::move(storage_partition), shared_from_this()); +} + void LogicalStore::pack(BufferBuilder& buffer) const { buffer.pack(scalar()); @@ -324,5 +396,14 @@ std::string LogicalStore::to_string() const return ss.str(); } +//////////////////////////////////////////////////// +// legate::detail::LogicalStorePartition +//////////////////////////////////////////////////// +LogicalStorePartition::LogicalStorePartition(std::shared_ptr storage_partition, + std::shared_ptr store) + : storage_partition_(std::move(storage_partition)), store_(std::move(store)) +{ +} + } // namespace detail } // namespace legate diff --git a/src/core/data/logical_store_detail.h b/src/core/data/logical_store_detail.h index ecb1dc20a9..59d5aaa906 100644 --- a/src/core/data/logical_store_detail.h +++ b/src/core/data/logical_store_detail.h @@ -23,7 +23,10 @@ namespace legate { namespace detail { -class Storage { +class StoragePartition; +class LogicalStorePartition; + +class Storage : public std::enable_shared_from_this { public: enum class Kind : int32_t { REGION_FIELD = 0, @@ -33,14 +36,14 @@ class Storage { public: // Create a RegionField-backed storage whose size is unbound. Initialized lazily. Storage(int32_t dim, LegateTypeCode code); - // Create a RegionField-backed storage. Initialized lazily. - Storage(tuple extents, LegateTypeCode code); + // Create a RegionField-backed or a Future-backedstorage. Initialized lazily. + Storage(Shape extents, LegateTypeCode code, bool optimize_scalar); // Create a Future-backed storage. Initialized eagerly. - Storage(tuple extents, LegateTypeCode code, const Legion::Future& future); + Storage(Shape extents, LegateTypeCode code, const Legion::Future& future); public: bool unbound() const { return unbound_; } - const tuple& extents() const { return extents_; } + const Shape& extents() const { return extents_; } size_t volume() const { return volume_; } int32_t dim(); LegateTypeCode code() const { return code_; } @@ -50,6 +53,7 @@ class Storage { LogicalRegionField* get_region_field(); Legion::Future get_future() const; void set_region_field(std::shared_ptr&& region_field); + void set_future(Legion::Future future); public: RegionField map(LibraryContext* context); @@ -60,10 +64,13 @@ class Storage { void reset_key_partition(); Legion::LogicalPartition find_or_create_legion_partition(const Partition* partition); + public: + std::shared_ptr create_partition(std::shared_ptr partition); + private: bool unbound_{false}; int32_t dim_{-1}; - tuple extents_; + Shape extents_; size_t volume_; LegateTypeCode code_{MAX_TYPE_NUMBER}; Kind kind_{Kind::REGION_FIELD}; @@ -74,10 +81,22 @@ class Storage { std::unique_ptr key_partition_{nullptr}; }; -class LogicalStore { +class StoragePartition { + public: + StoragePartition(std::shared_ptr parent, std::shared_ptr partition); + + public: + std::shared_ptr partition() const { return partition_; } + + private: + std::shared_ptr parent_; + std::shared_ptr partition_; +}; + +class LogicalStore : public std::enable_shared_from_this { public: LogicalStore(std::shared_ptr&& storage); - LogicalStore(tuple&& extents, + LogicalStore(Shape&& extents, const std::shared_ptr& storage, std::shared_ptr&& transform); @@ -97,7 +116,7 @@ class LogicalStore { public: bool unbound() const; - const tuple& extents() const; + const Shape& extents() const; size_t volume() const; // Size of the backing storage size_t storage_size() const; @@ -109,11 +128,15 @@ class LogicalStore { LogicalRegionField* get_region_field(); Legion::Future get_future(); void set_region_field(std::shared_ptr&& region_field); + void set_future(Legion::Future future); public: std::shared_ptr promote(int32_t extra_dim, size_t dim_size) const; std::shared_ptr project(int32_t dim, int64_t index) const; + public: + std::shared_ptr partition_by_tiling(Shape tile_shape); + public: std::shared_ptr get_physical_store(LibraryContext* context); @@ -123,6 +146,9 @@ class LogicalStore { void set_key_partition(const Partition* partition); void reset_key_partition(); + private: + std::shared_ptr create_partition(std::shared_ptr partition); + private: Legion::ProjectionID compute_projection(int32_t launch_ndim) const; @@ -134,7 +160,7 @@ class LogicalStore { private: uint64_t store_id_; - tuple extents_; + Shape extents_; std::shared_ptr storage_; std::shared_ptr transform_; @@ -143,5 +169,19 @@ class LogicalStore { std::shared_ptr mapped_{nullptr}; }; +class LogicalStorePartition { + public: + LogicalStorePartition(std::shared_ptr storage_partition, + std::shared_ptr store); + + public: + std::shared_ptr storage_partition() const { return storage_partition_; } + std::shared_ptr store() const { return store_; } + + private: + std::shared_ptr storage_partition_; + std::shared_ptr store_; +}; + } // namespace detail } // namespace legate diff --git a/src/core/data/shape.cc b/src/core/data/shape.cc new file mode 100644 index 0000000000..9b72f3752b --- /dev/null +++ b/src/core/data/shape.cc @@ -0,0 +1,33 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/data/shape.h" + +namespace legate { + +Legion::Domain to_domain(const tuple& shape) +{ + Legion::Domain domain; + auto ndim = static_cast(shape.size()); + domain.dim = ndim; + for (int32_t idx = 0; idx < ndim; ++idx) { + domain.rect_data[idx] = 0; + domain.rect_data[idx + ndim] = shape[idx] - 1; + } + return domain; +} + +} // namespace legate diff --git a/src/core/data/shape.h b/src/core/data/shape.h new file mode 100644 index 0000000000..6bd37175e5 --- /dev/null +++ b/src/core/data/shape.h @@ -0,0 +1,29 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include "core/utilities/tuple.h" + +#include "legion.h" + +namespace legate { + +using Shape = tuple; + +Legion::Domain to_domain(const Shape& shape); + +} // namespace legate diff --git a/src/core/data/store.cc b/src/core/data/store.cc index e63779b32f..ae8014e62d 100644 --- a/src/core/data/store.cc +++ b/src/core/data/store.cc @@ -259,6 +259,22 @@ Store::Store(int32_t dim, { } +Store::Store(int32_t dim, + int32_t code, + int32_t redop_id, + FutureWrapper future, + const std::shared_ptr& transform) + : is_future_(true), + is_unbound_store_(false), + dim_(dim), + code_(code), + redop_id_(redop_id), + future_(future), + transform_(transform), + readable_(true) +{ +} + Store::Store(int32_t dim, int32_t code, int32_t redop_id, diff --git a/src/core/data/store.h b/src/core/data/store.h index ff35df7169..49a9afd182 100644 --- a/src/core/data/store.h +++ b/src/core/data/store.h @@ -287,6 +287,11 @@ class Store { int32_t code, UnboundRegionField&& unbound_field, std::shared_ptr&& transform = nullptr); + Store(int32_t dim, + int32_t code, + int32_t redop_id, + FutureWrapper future, + const std::shared_ptr& transform); Store(int32_t dim, int32_t code, int32_t redop_id, diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index 6ef3428efc..e543618caf 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -61,7 +61,7 @@ Tiling::Tiling(Shape&& tile_shape, Shape&& color_shape, Shape&& offsets) color_shape_(std::forward(color_shape)), offsets_(std::forward(offsets)) { - if (offsets_.empty()) offsets_ = tuple(tile_shape_.size(), 0); + if (offsets_.empty()) offsets_ = Shape(tile_shape_.size(), 0); assert(tile_shape_.size() == color_shape_.size()); assert(tile_shape_.size() == offsets_.size()); } @@ -111,21 +111,14 @@ Legion::LogicalPartition Tiling::construct(Legion::LogicalRegion region, for (int32_t idx = 0; idx < ndim * ndim; ++idx) transform.matrix[idx] = 0; for (int32_t idx = 0; idx < ndim; ++idx) transform.matrix[ndim * idx + idx] = tile_shape_[idx]; - Legion::Domain extent; - extent.dim = ndim; + auto extent = to_domain(tile_shape_); for (int32_t idx = 0; idx < ndim; ++idx) { - extent.rect_data[idx] = offsets_[idx]; - extent.rect_data[idx + ndim] = tile_shape_[idx] - 1 + offsets_[idx]; + extent.rect_data[idx] += offsets_[idx]; + extent.rect_data[idx + ndim] += offsets_[idx]; } - Legion::Domain color_domain; - color_domain.dim = ndim; - for (int32_t idx = 0; idx < ndim; ++idx) { - color_domain.rect_data[idx] = 0; - color_domain.rect_data[idx + ndim] = color_shape_[idx] - 1; - } - - auto color_space = runtime->find_or_create_index_space(color_domain); + auto color_domain = to_domain(color_shape_); + auto color_space = runtime->find_or_create_index_space(color_domain); auto kind = complete ? (disjoint ? LEGION_DISJOINT_COMPLETE_KIND : LEGION_ALIASED_COMPLETE_KIND) : (disjoint ? LEGION_DISJOINT_KIND : LEGION_ALIASED_KIND); @@ -144,17 +137,7 @@ std::unique_ptr Tiling::get_projection(detail::LogicalStore* store, bool Tiling::has_launch_domain() const { return true; } -Legion::Domain Tiling::launch_domain() const -{ - Legion::Domain launch_domain; - int32_t ndim = static_cast(color_shape_.size()); - launch_domain.dim = ndim; - for (int32_t idx = 0; idx < ndim; ++idx) { - launch_domain.rect_data[idx] = 0; - launch_domain.rect_data[idx + ndim] = color_shape_[idx] - 1; - } - return launch_domain; -} +Legion::Domain Tiling::launch_domain() const { return to_domain(color_shape_); } std::unique_ptr Tiling::clone() const { return std::make_unique(*this); } diff --git a/src/core/partitioning/partition.h b/src/core/partitioning/partition.h index 49ac59d8e0..6d6abb8fc8 100644 --- a/src/core/partitioning/partition.h +++ b/src/core/partitioning/partition.h @@ -18,7 +18,7 @@ #include -#include "core/utilities/tuple.h" +#include "core/data/shape.h" #include "legion.h" namespace legate { @@ -31,8 +31,6 @@ class LogicalStore; } // namespace detail -using Shape = tuple; - struct Partition { public: enum class Kind : int32_t { diff --git a/src/core/partitioning/partitioner.cc b/src/core/partitioning/partitioner.cc index 9eec7c4a44..bdf8d6c3f0 100644 --- a/src/core/partitioning/partitioner.cc +++ b/src/core/partitioning/partitioner.cc @@ -110,6 +110,14 @@ const Legion::Domain* Strategy::launch_domain(const Operation* op) const return finder != launch_domains_.end() ? finder->second.get() : nullptr; } +void Strategy::set_launch_shape(const Operation* op, const Shape& shape) +{ +#ifdef DEBUG_LEGATE + assert(launch_domains_.find(op) == launch_domains_.end()); +#endif + launch_domains_.insert({op, std::make_unique(to_domain(shape))}); +} + void Strategy::insert(const Variable* partition_symbol, std::shared_ptr partition) { #ifdef DEBUG_LEGATE diff --git a/src/core/partitioning/partitioner.h b/src/core/partitioning/partitioner.h index 3a3630d8ad..31f6313cfa 100644 --- a/src/core/partitioning/partitioner.h +++ b/src/core/partitioning/partitioner.h @@ -19,6 +19,7 @@ #include #include +#include "core/data/shape.h" #include "core/partitioning/constraint.h" #include "legion.h" @@ -40,6 +41,7 @@ class Strategy { public: bool parallel(const Operation* op) const; const Legion::Domain* launch_domain(const Operation* op) const; + void set_launch_shape(const Operation* op, const Shape& shape); public: void insert(const Variable* partition_symbol, std::shared_ptr partition); diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/launcher.cc index 2bfc7dca61..d14c7ad949 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -95,28 +95,37 @@ void TaskLauncher::add_unbound_output(detail::LogicalStore* store, unbound_stores_.push_back(arg); } -void TaskLauncher::execute(const Legion::Domain& launch_domain) +void TaskLauncher::add_future(const Legion::Future& future) +{ + // FIXME: Futures that are directly added by this function are incompatible with those + // from scalar stores. We need to separate the two sets. + futures_.push_back(future); +} + +void TaskLauncher::add_future_map(const Legion::FutureMap& future_map) +{ + future_maps_.push_back(future_map); +} + +Legion::FutureMap TaskLauncher::execute(const Legion::Domain& launch_domain) { auto legion_launcher = build_index_task(launch_domain); - if (output_requirements_.empty()) - Runtime::get_runtime()->dispatch(legion_launcher); - else { - Runtime::get_runtime()->dispatch(legion_launcher, &output_requirements_); - bind_region_fields_to_unbound_stores(); - } - delete legion_launcher; + + if (output_requirements_.empty()) return Runtime::get_runtime()->dispatch(legion_launcher.get()); + + auto result = Runtime::get_runtime()->dispatch(legion_launcher.get(), &output_requirements_); + bind_region_fields_to_unbound_stores(); + return result; } -void TaskLauncher::execute_single() +Legion::Future TaskLauncher::execute_single() { auto legion_launcher = build_single_task(); - if (output_requirements_.empty()) - Runtime::get_runtime()->dispatch(legion_launcher); - else { - Runtime::get_runtime()->dispatch(legion_launcher, &output_requirements_); - bind_region_fields_to_unbound_stores(); - } - delete legion_launcher; + + if (output_requirements_.empty()) return Runtime::get_runtime()->dispatch(legion_launcher.get()); + auto result = Runtime::get_runtime()->dispatch(legion_launcher.get(), &output_requirements_); + bind_region_fields_to_unbound_stores(); + return result; } void TaskLauncher::add_store(std::vector& args, @@ -151,7 +160,7 @@ void TaskLauncher::pack_args(const std::vector& args) for (auto& arg : args) arg->pack(*buffer_); } -Legion::TaskLauncher* TaskLauncher::build_single_task() +std::unique_ptr TaskLauncher::build_single_task() { // Coalesce region requirements before packing task arguments // as the latter requires requirement indices to be finalized @@ -166,20 +175,21 @@ Legion::TaskLauncher* TaskLauncher::build_single_task() buffer_->pack(false); buffer_->pack(0); - auto single_task = new Legion::TaskLauncher(legion_task_id(), - buffer_->to_legion_buffer(), - Legion::Predicate::TRUE_PRED, - legion_mapper_id(), - tag_); - for (auto& future_ : futures_) single_task->add_future(future_); + auto single_task = std::make_unique(legion_task_id(), + buffer_->to_legion_buffer(), + Legion::Predicate::TRUE_PRED, + legion_mapper_id(), + tag_); + for (auto& future : futures_) single_task->add_future(future); - req_analyzer_->populate_launcher(single_task); + req_analyzer_->populate_launcher(single_task.get()); out_analyzer_->populate_output_requirements(output_requirements_); - return single_task; + return std::move(single_task); } -Legion::IndexTaskLauncher* TaskLauncher::build_index_task(const Legion::Domain& launch_domain) +std::unique_ptr TaskLauncher::build_index_task( + const Legion::Domain& launch_domain) { // Coalesce region requirements before packing task arguments // as the latter requires requirement indices to be finalized @@ -194,20 +204,21 @@ Legion::IndexTaskLauncher* TaskLauncher::build_index_task(const Legion::Domain& buffer_->pack(false); buffer_->pack(0); - auto index_task = new Legion::IndexTaskLauncher(legion_task_id(), - launch_domain, - buffer_->to_legion_buffer(), - Legion::ArgumentMap(), - Legion::Predicate::TRUE_PRED, - false /*must*/, - legion_mapper_id(), - tag_); - for (auto& future_ : futures_) index_task->add_future(future_); - - req_analyzer_->populate_launcher(index_task); + auto index_task = std::make_unique(legion_task_id(), + launch_domain, + buffer_->to_legion_buffer(), + Legion::ArgumentMap(), + Legion::Predicate::TRUE_PRED, + false /*must*/, + legion_mapper_id(), + tag_); + for (auto& future : futures_) index_task->add_future(future); + for (auto& future_map : future_maps_) index_task->point_futures.push_back(future_map); + + req_analyzer_->populate_launcher(index_task.get()); out_analyzer_->populate_output_requirements(output_requirements_); - return index_task; + return std::move(index_task); } void TaskLauncher::bind_region_fields_to_unbound_stores() diff --git a/src/core/runtime/launcher.h b/src/core/runtime/launcher.h index cb4624015a..af8848b597 100644 --- a/src/core/runtime/launcher.h +++ b/src/core/runtime/launcher.h @@ -66,6 +66,10 @@ class TaskLauncher { Legion::FieldSpace field_space, Legion::FieldID field_id); + public: + void add_future(const Legion::Future& future); + void add_future_map(const Legion::FutureMap& future_map); + private: void add_store(std::vector& args, detail::LogicalStore* store, @@ -75,13 +79,13 @@ class TaskLauncher { Legion::RegionFlags flags); public: - void execute(const Legion::Domain& launch_domain); - void execute_single(); + Legion::FutureMap execute(const Legion::Domain& launch_domain); + Legion::Future execute_single(); private: void pack_args(const std::vector& args); - Legion::IndexTaskLauncher* build_index_task(const Legion::Domain& launch_domain); - Legion::TaskLauncher* build_single_task(); + std::unique_ptr build_index_task(const Legion::Domain& launch_domain); + std::unique_ptr build_single_task(); void bind_region_fields_to_unbound_stores(); private: @@ -97,6 +101,7 @@ class TaskLauncher { std::vector scalars_; std::vector futures_; std::vector unbound_stores_; + std::vector future_maps_; private: RequirementAnalyzer* req_analyzer_; diff --git a/src/core/runtime/operation.cc b/src/core/runtime/operation.cc index 5fcc0b113b..19a2de8c29 100644 --- a/src/core/runtime/operation.cc +++ b/src/core/runtime/operation.cc @@ -32,41 +32,15 @@ namespace legate { +//////////////////////////////////////////////////// +// legate::Operation +//////////////////////////////////////////////////// + Operation::Operation(LibraryContext* library, uint64_t unique_id, int64_t mapper_id) : library_(library), unique_id_(unique_id), mapper_id_(mapper_id) { } -Operation::~Operation() {} - -void Operation::add_input(LogicalStore store, const Variable* partition_symbol) -{ - add_store(inputs_, store, partition_symbol); -} - -void Operation::add_output(LogicalStore store, const Variable* partition_symbol) -{ - add_store(outputs_, store, partition_symbol); -} - -void Operation::add_reduction(LogicalStore store, - Legion::ReductionOpID redop, - const Variable* partition_symbol) -{ - add_store(reductions_, store, partition_symbol); - reduction_ops_.push_back(redop); -} - -void Operation::add_store(std::vector& store_args, - LogicalStore& store, - const Variable* partition_symbol) -{ - auto store_impl = store.impl(); - store_args.push_back(StoreArg(store_impl.get(), partition_symbol)); - store_mappings_[*partition_symbol] = store_impl.get(); - all_stores_.insert(std::move(store_impl)); -} - const Variable* Operation::declare_partition() { partition_symbols_.emplace_back(new Variable(this, next_part_id_++)); @@ -82,26 +56,15 @@ detail::LogicalStore* Operation::find_store(const Variable* part_symb) const return finder->second; } -void Operation::add_constraint(std::unique_ptr constraint) -{ - constraints_.push_back(std::move(constraint)); -} - -void Operation::add_to_constraint_graph(ConstraintGraph& constraint_graph) const -{ - for (auto& pair : inputs_) constraint_graph.add_partition_symbol(pair.second); - for (auto& pair : outputs_) constraint_graph.add_partition_symbol(pair.second); - for (auto& pair : reductions_) constraint_graph.add_partition_symbol(pair.second); - for (auto& constraint : constraints_) constraint_graph.add_constraint(constraint.get()); -} +//////////////////////////////////////////////////// +// legate::Task +//////////////////////////////////////////////////// Task::Task(LibraryContext* library, int64_t task_id, uint64_t unique_id, int64_t mapper_id /*=0*/) : Operation(library, unique_id, mapper_id), task_id_(task_id) { } -Task::~Task() {} - void Task::add_scalar_arg(const Scalar& scalar) { scalars_.push_back(scalar); } struct field_size_fn { @@ -114,8 +77,6 @@ struct field_size_fn { void Task::launch(Strategy* p_strategy) { - auto* runtime = Runtime::get_runtime(); - auto& strategy = *p_strategy; TaskLauncher launcher(library_, task_id_, mapper_id_); auto launch_domain = strategy.launch_domain(this); @@ -145,6 +106,8 @@ void Task::launch(Strategy* p_strategy) proj->set_reduction_op(redop); launcher.add_reduction(store, std::move(proj)); } + + auto* runtime = Runtime::get_runtime(); for (auto& pair : outputs_) { auto& store = pair.first; if (!store->unbound()) continue; @@ -157,10 +120,72 @@ void Task::launch(Strategy* p_strategy) } for (auto& scalar : scalars_) launcher.add_scalar(scalar); - if (launch_domain != nullptr) - launcher.execute(*launch_domain); - else - launcher.execute_single(); + if (launch_domain != nullptr) { + auto result = launcher.execute(*launch_domain); + demux_scalar_stores(result, *launch_domain); + } else { + auto result = launcher.execute_single(); + demux_scalar_stores(result); + } +} + +void Task::demux_scalar_stores(const Legion::Future& result) +{ + // TODO: Handle unbound stores + auto num_scalar_outs = scalar_outputs_.size(); + auto num_scalar_reds = scalar_reductions_.size(); + + auto total = num_scalar_outs + num_scalar_reds; + if (0 == total) + return; + else if (1 == total) { + if (1 == num_scalar_outs) { + auto [store, _] = outputs_[scalar_outputs_.front()]; + store->set_future(result); + } else { + assert(1 == num_scalar_reds); + auto [store, _] = reductions_[scalar_reductions_.front()]; + store->set_future(result); + } + } else { + auto* runtime = Runtime::get_runtime(); + uint32_t idx = 0; + for (const auto& out_idx : scalar_outputs_) { + auto [store, _] = outputs_[out_idx]; + store->set_future(runtime->extract_scalar(result, idx++)); + } + for (const auto& red_idx : scalar_reductions_) { + auto [store, _] = reductions_[red_idx]; + store->set_future(runtime->extract_scalar(result, idx++)); + } + } +} + +void Task::demux_scalar_stores(const Legion::FutureMap& result, const Legion::Domain& launch_domain) +{ + // Tasks with scalar outputs shouldn't have been parallelized + assert(scalar_outputs_.empty()); + + // TODO: Handle unbound stores + auto num_scalar_reds = scalar_reductions_.size(); + + auto total = num_scalar_reds; + if (0 == total) return; + + auto* runtime = Runtime::get_runtime(); + if (1 == total) { + assert(1 == num_scalar_reds); + auto red_idx = scalar_reductions_.front(); + auto [store, _] = reductions_[red_idx]; + store->set_future(runtime->reduce_future_map(result, reduction_ops_[red_idx])); + } else { + uint32_t idx = 0; + for (const auto& red_idx : scalar_reductions_) { + auto [store, _] = reductions_[red_idx]; + auto values = runtime->extract_scalar(result, idx++, launch_domain); + store->set_future(runtime->reduce_future_map(values, reduction_ops_[red_idx])); + } + } } std::string Task::to_string() const @@ -170,4 +195,120 @@ std::string Task::to_string() const return ss.str(); } +//////////////////////////////////////////////////// +// legate::AutoTask +//////////////////////////////////////////////////// + +AutoTask::AutoTask(LibraryContext* library, int64_t task_id, uint64_t unique_id, int64_t mapper_id) + : Task(library, task_id, unique_id, mapper_id) +{ +} + +void AutoTask::add_input(LogicalStore store, const Variable* partition_symbol) +{ + add_store(inputs_, store, partition_symbol); +} + +void AutoTask::add_output(LogicalStore store, const Variable* partition_symbol) +{ + if (store.impl()->scalar()) scalar_outputs_.push_back(outputs_.size()); + add_store(outputs_, store, partition_symbol); +} + +void AutoTask::add_reduction(LogicalStore store, + Legion::ReductionOpID redop, + const Variable* partition_symbol) +{ + if (store.impl()->scalar()) scalar_reductions_.push_back(reductions_.size()); + add_store(reductions_, store, partition_symbol); + reduction_ops_.push_back(redop); +} + +void AutoTask::add_store(std::vector& store_args, + LogicalStore& store, + const Variable* partition_symbol) +{ + auto store_impl = store.impl(); + store_args.push_back(StoreArg(store_impl.get(), partition_symbol)); + store_mappings_[*partition_symbol] = store_impl.get(); + all_stores_.insert(std::move(store_impl)); +} + +void AutoTask::add_constraint(std::unique_ptr constraint) +{ + constraints_.push_back(std::move(constraint)); +} + +void AutoTask::add_to_constraint_graph(ConstraintGraph& constraint_graph) const +{ + for (auto& constraint : constraints_) constraint_graph.add_constraint(constraint.get()); + for (auto& [_, symb] : inputs_) constraint_graph.add_partition_symbol(symb); + for (auto& [_, symb] : outputs_) constraint_graph.add_partition_symbol(symb); + for (auto& [_, symb] : reductions_) constraint_graph.add_partition_symbol(symb); +} + +//////////////////////////////////////////////////// +// legate::ManualTask +//////////////////////////////////////////////////// + +ManualTask::~ManualTask() {} + +ManualTask::ManualTask(LibraryContext* library, + int64_t task_id, + const Shape& launch_shape, + uint64_t unique_id, + int64_t mapper_id) + : Task(library, task_id, unique_id, mapper_id), strategy_(std::make_unique()) +{ + strategy_->set_launch_shape(this, launch_shape); +} + +void ManualTask::add_input(LogicalStore store) { add_store(inputs_, store, create_no_partition()); } + +void ManualTask::add_output(LogicalStore store) +{ + if (store.impl()->scalar()) scalar_outputs_.push_back(outputs_.size()); + add_store(outputs_, store, create_no_partition()); +} + +void ManualTask::add_reduction(LogicalStore store, Legion::ReductionOpID redop) +{ + if (store.impl()->scalar()) scalar_reductions_.push_back(reductions_.size()); + add_store(reductions_, store, create_no_partition()); + reduction_ops_.push_back(redop); +} + +void ManualTask::add_input(LogicalStorePartition store_partition) +{ + add_store(inputs_, store_partition.store(), store_partition.partition()); +} + +void ManualTask::add_output(LogicalStorePartition store_partition) +{ + if (store_partition.store().impl()->scalar()) scalar_outputs_.push_back(outputs_.size()); + add_store(outputs_, store_partition.store(), store_partition.partition()); +} + +void ManualTask::add_reduction(LogicalStorePartition store_partition, Legion::ReductionOpID redop) +{ + if (store_partition.store().impl()->scalar()) scalar_reductions_.push_back(reductions_.size()); + add_store(reductions_, store_partition.store(), store_partition.partition()); + reduction_ops_.push_back(redop); +} + +void ManualTask::add_store(std::vector& store_args, + const LogicalStore& store, + std::shared_ptr partition) +{ + auto store_impl = store.impl(); + auto partition_symbol = declare_partition(); + store_args.push_back(StoreArg(store_impl.get(), partition_symbol)); + strategy_->insert(partition_symbol, std::move(partition)); + all_stores_.insert(std::move(store_impl)); +} + +void ManualTask::launch(Strategy*) { Task::launch(strategy_.get()); } + +void ManualTask::add_to_constraint_graph(ConstraintGraph& constraint_graph) const {} + } // namespace legate diff --git a/src/core/runtime/operation.h b/src/core/runtime/operation.h index 006ea5f188..05b6a4d9da 100644 --- a/src/core/runtime/operation.h +++ b/src/core/runtime/operation.h @@ -28,6 +28,7 @@ namespace legate { class Constraint; class ConstraintGraph; class LibraryContext; +class Runtime; class Scalar; class Strategy; @@ -40,36 +41,19 @@ class LogicalStore; class Operation { protected: using StoreArg = std::pair; - - public: Operation(LibraryContext* library, uint64_t unique_id, int64_t mapper_id); public: - virtual ~Operation(); + virtual ~Operation() {} public: - void add_input(LogicalStore store, const Variable* partition_symbol); - void add_output(LogicalStore store, const Variable* partition_symbol); - void add_reduction(LogicalStore store, - Legion::ReductionOpID redop, - const Variable* partition_symbol); - - private: - void add_store(std::vector& store_args, - LogicalStore& store, - const Variable* partition_symbol); + virtual void add_to_constraint_graph(ConstraintGraph& constraint_graph) const = 0; + virtual void launch(Strategy* strategy) = 0; + virtual std::string to_string() const = 0; public: const Variable* declare_partition(); detail::LogicalStore* find_store(const Variable* variable) const; - void add_constraint(std::unique_ptr constraint); - void add_to_constraint_graph(ConstraintGraph& constraint_graph) const; - - public: - virtual void launch(Strategy* strategy) = 0; - - public: - virtual std::string to_string() const = 0; protected: LibraryContext* library_; @@ -83,21 +67,18 @@ class Operation { std::vector reductions_{}; std::vector reduction_ops_{}; - private: + protected: uint32_t next_part_id_{0}; std::vector> partition_symbols_{}; - - private: std::map store_mappings_{}; - std::vector> constraints_{}; }; class Task : public Operation { - public: - Task(LibraryContext* library, int64_t task_id, uint64_t unique_id, int64_t mapper_id = 0); + protected: + Task(LibraryContext* library, int64_t task_id, uint64_t unique_id, int64_t mapper_id); public: - ~Task(); + virtual ~Task() {} public: void add_scalar_arg(const Scalar& scalar); @@ -105,12 +86,83 @@ class Task : public Operation { public: virtual void launch(Strategy* strategy) override; + private: + void demux_scalar_stores(const Legion::Future& result); + void demux_scalar_stores(const Legion::FutureMap& result, const Legion::Domain& launch_domain); + public: - virtual std::string to_string() const override; + std::string to_string() const override; - private: + protected: int64_t task_id_; std::vector scalars_{}; + std::vector scalar_outputs_{}; + std::vector scalar_reductions_{}; +}; + +class AutoTask : public Task { + public: + friend class Runtime; + AutoTask(LibraryContext* library, int64_t task_id, uint64_t unique_id, int64_t mapper_id); + + public: + ~AutoTask() {} + + public: + void add_input(LogicalStore store, const Variable* partition_symbol); + void add_output(LogicalStore store, const Variable* partition_symbol); + void add_reduction(LogicalStore store, + Legion::ReductionOpID redop, + const Variable* partition_symbol); + + private: + void add_store(std::vector& store_args, + LogicalStore& store, + const Variable* partition_symbol); + + public: + void add_constraint(std::unique_ptr constraint); + void add_to_constraint_graph(ConstraintGraph& constraint_graph) const override; + + private: + std::vector> constraints_{}; +}; + +class ManualTask : public Task { + private: + friend class Runtime; + ManualTask(LibraryContext* library, + int64_t task_id, + const Shape& launch_shape, + uint64_t unique_id, + int64_t mapper_id); + + public: + ~ManualTask(); + + public: + void add_input(LogicalStore store); + void add_output(LogicalStore store); + void add_reduction(LogicalStore store, Legion::ReductionOpID redop); + + public: + void add_input(LogicalStorePartition store_partition); + void add_output(LogicalStorePartition store_partition); + void add_reduction(LogicalStorePartition store_partition, Legion::ReductionOpID redop); + + private: + void add_store(std::vector& store_args, + const LogicalStore& store, + std::shared_ptr partition); + + public: + void launch(Strategy* strategy) override; + + public: + void add_to_constraint_graph(ConstraintGraph& constraint_graph) const override; + + private: + std::unique_ptr strategy_; }; } // namespace legate diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index de30da803f..277b633122 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -25,6 +25,7 @@ #include "core/partitioning/partition.h" #include "core/partitioning/partitioner.h" #include "core/runtime/context.h" +#include "core/runtime/launcher.h" #include "core/runtime/projection.h" #include "core/runtime/shard.h" #include "core/task/exception.h" @@ -441,7 +442,7 @@ PartitionManager::PartitionManager(Runtime* runtime, const LibraryContext* conte push_factors(2); } -tuple PartitionManager::compute_launch_shape(const tuple& shape) +Shape PartitionManager::compute_launch_shape(const Shape& shape) { // Easy case if we only have one piece: no parallel launch space if (num_pieces_ == 1) return {}; @@ -563,14 +564,13 @@ tuple PartitionManager::compute_launch_shape(const tuple& shape) std::vector result(shape.size(), 1); for (uint32_t idx = 0; idx < ndim; ++idx) result[temp_dims[idx]] = temp_result[idx]; - return tuple(std::move(result)); + return Shape(std::move(result)); } -tuple PartitionManager::compute_tile_shape(const tuple& extents, - const tuple& launch_shape) +Shape PartitionManager::compute_tile_shape(const Shape& extents, const Shape& launch_shape) { assert(extents.size() == launch_shape.size()); - tuple tile_shape; + Shape tile_shape; for (uint32_t idx = 0; idx < extents.size(); ++idx) { auto x = extents[idx]; auto y = launch_shape[idx]; @@ -646,11 +646,21 @@ void Runtime::post_startup_initialization(Legion::Context legion_context) } // This function should be moved to the library context -std::unique_ptr Runtime::create_task(LibraryContext* library, - int64_t task_id, - int64_t mapper_id /*=0*/) +std::unique_ptr Runtime::create_task(LibraryContext* library, + int64_t task_id, + int64_t mapper_id /*=0*/) { - return std::make_unique(library, task_id, next_unique_id_++, mapper_id); + auto task = new AutoTask(library, task_id, next_unique_id_++, mapper_id); + return std::unique_ptr(task); +} + +std::unique_ptr Runtime::create_task(LibraryContext* library, + int64_t task_id, + const Shape& launch_shape, + int64_t mapper_id /*=0*/) +{ + auto task = new ManualTask(library, task_id, launch_shape, next_unique_id_++, mapper_id); + return std::unique_ptr(task); } void Runtime::submit(std::unique_ptr op) @@ -681,15 +691,17 @@ LogicalStore Runtime::create_store(LegateTypeCode code, int32_t dim) return LogicalStore(std::make_shared(std::move(storage))); } -LogicalStore Runtime::create_store(std::vector extents, LegateTypeCode code) +LogicalStore Runtime::create_store(std::vector extents, + LegateTypeCode code, + bool optimize_scalar /*=false*/) { - auto storage = std::make_shared(extents, code); + auto storage = std::make_shared(extents, code, optimize_scalar); return LogicalStore(std::make_shared(std::move(storage))); } LogicalStore Runtime::create_store(const Scalar& scalar) { - tuple extents{1}; + Shape extents{1}; auto future = create_future(scalar.ptr(), scalar.size()); auto storage = std::make_shared(extents, scalar.code(), future); return LogicalStore(std::make_shared(std::move(storage))); @@ -697,7 +709,7 @@ LogicalStore Runtime::create_store(const Scalar& scalar) uint64_t Runtime::get_unique_store_id() { return next_store_id_++; } -std::shared_ptr Runtime::create_region_field(const tuple& extents, +std::shared_ptr Runtime::create_region_field(const Shape& extents, LegateTypeCode code) { DomainPoint lo, hi; @@ -839,20 +851,54 @@ Domain Runtime::get_index_space_domain(const Legion::IndexSpace& index_space) co return legion_runtime_->get_index_space_domain(legion_context_, index_space); } -std::shared_ptr Runtime::dispatch( - Legion::TaskLauncher* launcher, std::vector* output_requirements) +Legion::Future Runtime::dispatch(Legion::TaskLauncher* launcher, + std::vector* output_requirements) { assert(nullptr != legion_context_); - legion_runtime_->execute_task(legion_context_, *launcher, output_requirements); - return nullptr; + return legion_runtime_->execute_task(legion_context_, *launcher, output_requirements); } -std::shared_ptr Runtime::dispatch( - Legion::IndexTaskLauncher* launcher, std::vector* output_requirements) +Legion::FutureMap Runtime::dispatch(Legion::IndexTaskLauncher* launcher, + std::vector* output_requirements) { assert(nullptr != legion_context_); - legion_runtime_->execute_index_space(legion_context_, *launcher, output_requirements); - return nullptr; + return legion_runtime_->execute_index_space(legion_context_, *launcher, output_requirements); +} + +Legion::Future Runtime::extract_scalar(const Legion::Future& result, uint32_t idx) const +{ + // FIXME: One of two things should happen: + // 1) the variant should be picked based on available processor kinds + // 2) the variant should be picked by the core mapper + TaskLauncher launcher( + core_context_, LEGATE_CORE_EXTRACT_SCALAR_TASK_ID, 0 /*mapper_id*/, LEGATE_CPU_VARIANT); + launcher.add_future(result); + launcher.add_scalar(Scalar(idx)); + return launcher.execute_single(); +} + +Legion::FutureMap Runtime::extract_scalar(const Legion::FutureMap& result, + uint32_t idx, + const Legion::Domain& launch_domain) const +{ + // FIXME: One of two things should happen: + // 1) the variant should be picked based on available processor kinds + // 2) the variant should be picked by the core mapper + TaskLauncher launcher( + core_context_, LEGATE_CORE_EXTRACT_SCALAR_TASK_ID, 0 /*mapper_id*/, LEGATE_CPU_VARIANT); + launcher.add_future_map(result); + launcher.add_scalar(Scalar(idx)); + return launcher.execute(launch_domain); +} + +Legion::Future Runtime::reduce_future_map(const Legion::FutureMap& future_map, + int32_t reduction_op) const +{ + return legion_runtime_->reduce_future_map(legion_context_, + future_map, + reduction_op, + false /*deterministic*/, + core_context_->get_mapper_id(0)); } void Runtime::issue_execution_fence(bool block /*=false*/) diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 29c07a9c1d..33526ec77a 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -20,11 +20,11 @@ #include "legion.h" +#include "core/data/shape.h" #include "core/data/store.h" #include "core/legate_c.h" #include "core/runtime/context.h" #include "core/task/exception.h" -#include "core/utilities/tuple.h" #include "core/utilities/typedefs.h" /** @defgroup runtime Runtime and library contexts @@ -78,15 +78,16 @@ struct Core { static LegateMainFnPtr main_fn; }; +class AutoTask; class FieldManager; class LogicalRegionField; class LogicalStore; +class ManualTask; class Operation; class PartitioningFunctor; class RegionManager; class ResourceConfig; class Runtime; -class Task; class Tiling; class PartitionManager { @@ -94,8 +95,8 @@ class PartitionManager { PartitionManager(Runtime* runtime, const LibraryContext* context); public: - tuple compute_launch_shape(const tuple& shape); - tuple compute_tile_shape(const tuple& extents, const tuple& launch_shape); + Shape compute_launch_shape(const Shape& shape); + Shape compute_tile_shape(const Shape& extents, const Shape& launch_shape); public: Legion::IndexPartition find_index_partition(const Legion::IndexSpace& index_space, @@ -135,19 +136,25 @@ class Runtime { T get_tunable(const LibraryContext* context, int64_t tunable_id, int64_t mapper_id = 0); public: - std::unique_ptr create_task(LibraryContext* library, - int64_t task_id, - int64_t mapper_id = 0); + std::unique_ptr create_task(LibraryContext* library, + int64_t task_id, + int64_t mapper_id = 0); + std::unique_ptr create_task(LibraryContext* library, + int64_t task_id, + const Shape& launch_shape, + int64_t mapper_id = 0); void submit(std::unique_ptr op); public: LogicalStore create_store(LegateTypeCode code, int32_t dim = 1); - LogicalStore create_store(std::vector extents, LegateTypeCode code); + LogicalStore create_store(std::vector extents, + LegateTypeCode code, + bool optimize_scalar = false); LogicalStore create_store(const Scalar& scalar); uint64_t get_unique_store_id(); public: - std::shared_ptr create_region_field(const tuple& extents, + std::shared_ptr create_region_field(const Shape& extents, LegateTypeCode code); std::shared_ptr import_region_field(Legion::LogicalRegion region, Legion::FieldID field_id, @@ -177,12 +184,17 @@ class Runtime { Legion::Domain get_index_space_domain(const Legion::IndexSpace& index_space) const; public: - std::shared_ptr dispatch( - Legion::TaskLauncher* launcher, - std::vector* output_requirements = nullptr); - std::shared_ptr dispatch( - Legion::IndexTaskLauncher* launcher, - std::vector* output_requirements = nullptr); + Legion::Future dispatch(Legion::TaskLauncher* launcher, + std::vector* output_requirements = nullptr); + Legion::FutureMap dispatch(Legion::IndexTaskLauncher* launcher, + std::vector* output_requirements = nullptr); + + public: + Legion::Future extract_scalar(const Legion::Future& result, uint32_t idx) const; + Legion::FutureMap extract_scalar(const Legion::FutureMap& result, + uint32_t idx, + const Legion::Domain& launch_domain) const; + Legion::Future reduce_future_map(const Legion::FutureMap& future_map, int32_t reduction_op) const; public: void issue_execution_fence(bool block = false); diff --git a/tests/cpp/.clang-format b/tests/cpp/.clang-format new file mode 100644 index 0000000000..1ceb0920ba --- /dev/null +++ b/tests/cpp/.clang-format @@ -0,0 +1,129 @@ +Language: Cpp +# BasedOnStyle: Google +AccessModifierOffset: -1 +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: true +AlignConsecutiveDeclarations: false +AlignEscapedNewlines: Left +AlignOperands: true +AlignTrailingComments: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: true +AllowShortCaseLabelsOnASingleLine: true +AllowShortFunctionsOnASingleLine: All +AllowShortIfStatementsOnASingleLine: true +AllowShortLoopsOnASingleLine: true +# This is deprecated +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: Yes +BinPackArguments: false +BinPackParameters: false +BraceWrapping: + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + IndentBraces: false + # disabling the below splits, else, they'll just add to the vertical length of source files! + SplitEmptyFunction: false + SplitEmptyRecord: false + SplitEmptyNamespace: false +BreakBeforeBinaryOperators: None +BreakBeforeBraces: WebKit +BreakBeforeInheritanceComma: false +BreakInheritanceList: BeforeColon +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +ColumnLimit: 100 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: true +# Kept the below 2 to be the same as `IndentWidth` to keep everything uniform +ConstructorInitializerIndentWidth: 2 +ContinuationIndentWidth: 2 +Cpp11BracedListStyle: true +DerivePointerAlignment: false +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: +IncludeBlocks: Preserve +IncludeCategories: + - Regex: '^<.*\.h>' + Priority: 1 + - Regex: '^<.*' + Priority: 2 + - Regex: '.*' + Priority: 3 +IncludeIsMainRegex: '([-_](test|unittest))?$' +IndentCaseLabels: true +IndentPPDirectives: None +IndentWidth: 2 +IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBinPackProtocolList: Never +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 1 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 200 +PointerAlignment: Left +RawStringFormats: + - Language: Cpp + Delimiters: + - cc + - CC + - cpp + - Cpp + - CPP + - 'c++' + - 'C++' + CanonicalDelimiter: '' +# Enabling comment reflow causes doxygen comments to be messed up in their formats! +ReflowComments: true +SortIncludes: true +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: Cpp11 +StatementMacros: +# Be consistent with indent-width, even for people who use tab for indentation! +TabWidth: 2 +UseTab: Never diff --git a/tests/cpp/integration/manual_simple/CMakeLists.txt b/tests/cpp/integration/manual_simple/CMakeLists.txt new file mode 100644 index 0000000000..36b9afad48 --- /dev/null +++ b/tests/cpp/integration/manual_simple/CMakeLists.txt @@ -0,0 +1,31 @@ +#============================================================================= +# Copyright 2023 NVIDIA Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +cmake_minimum_required(VERSION 3.22.1 FATAL_ERROR) + +project(manual_simple VERSION 0.1 LANGUAGES C CXX) + +if (NOT CMAKE_CXX_STANDARD) + set(CMAKE_CXX_STANDARD 17) +endif() + +find_package(legate_core REQUIRED) + +add_executable(manual_simple manual_simple.cc) + +target_link_libraries(manual_simple PRIVATE legate::core) + +install(TARGETS manual_simple DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/cmake-install") diff --git a/tests/cpp/integration/manual_simple/build.sh b/tests/cpp/integration/manual_simple/build.sh new file mode 100755 index 0000000000..e6508cd229 --- /dev/null +++ b/tests/cpp/integration/manual_simple/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +rm -rf build +cmake -B build -S . -D legate_core_ROOT=../../../../build -D CMAKE_BUILD_TYPE=Debug +cmake --build build -j 4 diff --git a/tests/cpp/integration/manual_simple/manual_simple.cc b/tests/cpp/integration/manual_simple/manual_simple.cc new file mode 100644 index 0000000000..67b84de58c --- /dev/null +++ b/tests/cpp/integration/manual_simple/manual_simple.cc @@ -0,0 +1,142 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/mapping/mapping.h" +#include "legate.h" + +namespace example { + +static const char* library_name = "manual"; +static legate::Logger logger(library_name); + +enum TaskIDs { + HELLO = 0, +}; + +struct Registrar { + template + static void record_variant(Args&&... args) + { + get_registrar().record_variant(std::forward(args)...); + } + static legate::TaskRegistrar& get_registrar() + { + static legate::TaskRegistrar registrar; + return registrar; + } +}; + +template +struct BaseTask : public legate::LegateTask { + using Registrar = example::Registrar; +}; + +struct HelloTask : public BaseTask { + static const int32_t TASK_ID = HELLO; + static void cpu_variant(legate::TaskContext& context); +}; + +struct Mapper : public legate::mapping::LegateMapper { + void set_machine(const legate::mapping::MachineQueryInterface* machine) override {} + legate::mapping::TaskTarget task_target( + const legate::mapping::Task& task, + const std::vector& options) override + { + return options.front(); + } + std::vector store_mappings( + const legate::mapping::Task& task, + const std::vector& options) override + { + return {}; + } + legate::Scalar tunable_value(legate::TunableID tunable_id) override { return legate::Scalar(); } +}; + +void register_tasks() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name, legate::ResourceConfig()); + HelloTask::register_variants(); + Registrar::get_registrar().register_all_tasks(*context); + context->register_mapper(std::make_unique()); +} + +/*static*/ void HelloTask::cpu_variant(legate::TaskContext& context) +{ + auto& output = context.outputs()[0]; + auto shape = output.shape<2>(); + + if (shape.empty()) return; + + auto acc = output.write_accessor(shape); + for (legate::PointInRectIterator<2> it(shape); it.valid(); ++it) + acc[*it] = (*it)[0] + (*it)[1] * 1000; +} + +void test_auto_task(legate::LibraryContext* context, legate::LogicalStore store) +{ + auto runtime = legate::Runtime::get_runtime(); + auto task = runtime->create_task(context, HELLO); + auto part = task->declare_partition(); + task->add_output(store, part); + runtime->submit(std::move(task)); +} + +void test_manual_task(legate::LibraryContext* context, legate::LogicalStore store) +{ + auto runtime = legate::Runtime::get_runtime(); + auto task = runtime->create_task(context, HELLO, {3, 3}); + auto part = store.partition_by_tiling({2, 2}); + task->add_output(part); + runtime->submit(std::move(task)); +} + +void print_store(legate::LibraryContext* context, legate::LogicalStore store) +{ + auto runtime = legate::Runtime::get_runtime(); + auto p_store = store.get_physical_store(context); + auto acc = p_store->read_accessor(); + auto shape = p_store->shape<2>(); + std::stringstream ss; + for (legate::PointInRectIterator<2> it(shape); it.valid(); ++it) ss << acc[*it] << " "; + logger.print() << ss.str(); +} + +void legate_main(int32_t argc, char** argv) +{ + legate::Core::perform_registration(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + auto store = runtime->create_store({5, 5}, legate::LegateTypeCode::INT64_LT); + test_auto_task(context, store); + print_store(context, store); + test_manual_task(context, store); + print_store(context, store); +} + +} // namespace example + +int main(int argc, char** argv) +{ + legate::initialize(argc, argv); + + legate::set_main_function(example::legate_main); + + return legate::start(argc, argv); +} diff --git a/tests/cpp/integration/multi_scalar_out/CMakeLists.txt b/tests/cpp/integration/multi_scalar_out/CMakeLists.txt new file mode 100644 index 0000000000..6c7641328d --- /dev/null +++ b/tests/cpp/integration/multi_scalar_out/CMakeLists.txt @@ -0,0 +1,31 @@ +#============================================================================= +# Copyright 2023 NVIDIA Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +cmake_minimum_required(VERSION 3.22.1 FATAL_ERROR) + +project(multi_scalar_out VERSION 0.1 LANGUAGES C CXX) + +if (NOT CMAKE_CXX_STANDARD) + set(CMAKE_CXX_STANDARD 17) +endif() + +find_package(legate_core REQUIRED) + +add_executable(multi_scalar_out multi_scalar_out.cc) + +target_link_libraries(multi_scalar_out PRIVATE legate::core) + +install(TARGETS multi_scalar_out DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/cmake-install") diff --git a/tests/cpp/integration/multi_scalar_out/build.sh b/tests/cpp/integration/multi_scalar_out/build.sh new file mode 100755 index 0000000000..e6508cd229 --- /dev/null +++ b/tests/cpp/integration/multi_scalar_out/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +rm -rf build +cmake -B build -S . -D legate_core_ROOT=../../../../build -D CMAKE_BUILD_TYPE=Debug +cmake --build build -j 4 diff --git a/tests/cpp/integration/multi_scalar_out/multi_scalar_out.cc b/tests/cpp/integration/multi_scalar_out/multi_scalar_out.cc new file mode 100644 index 0000000000..fb0719ba08 --- /dev/null +++ b/tests/cpp/integration/multi_scalar_out/multi_scalar_out.cc @@ -0,0 +1,194 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/mapping/mapping.h" +#include "legate.h" + +namespace example { + +static const char* library_name = "multi_scalar"; +static legate::Logger logger(library_name); + +enum TaskIDs { + WRITER = 0, + REDUCER = 1, +}; + +struct Registrar { + template + static void record_variant(Args&&... args) + { + get_registrar().record_variant(std::forward(args)...); + } + static legate::TaskRegistrar& get_registrar() + { + static legate::TaskRegistrar registrar; + return registrar; + } +}; + +template +struct BaseTask : public legate::LegateTask { + using Registrar = example::Registrar; +}; + +struct WriterTask : public BaseTask { + static const int32_t TASK_ID = WRITER; + static void cpu_variant(legate::TaskContext& context); +}; + +struct ReducerTask : public BaseTask { + static const int32_t TASK_ID = REDUCER; + static void cpu_variant(legate::TaskContext& context); +}; + +struct Mapper : public legate::mapping::LegateMapper { + void set_machine(const legate::mapping::MachineQueryInterface* machine) override {} + legate::mapping::TaskTarget task_target( + const legate::mapping::Task& task, + const std::vector& options) override + { + return options.front(); + } + std::vector store_mappings( + const legate::mapping::Task& task, + const std::vector& options) override + { + return {}; + } + legate::Scalar tunable_value(legate::TunableID tunable_id) override { return legate::Scalar(); } +}; + +void register_tasks() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name, legate::ResourceConfig()); + WriterTask::register_variants(); + ReducerTask::register_variants(); + Registrar::get_registrar().register_all_tasks(*context); + context->register_mapper(std::make_unique()); +} + +/*static*/ void WriterTask::cpu_variant(legate::TaskContext& context) +{ + auto& output1 = context.outputs()[0]; + auto& output2 = context.outputs()[1]; + + auto acc1 = output1.write_accessor(); + auto acc2 = output2.write_accessor(); + + acc1[{0, 0}] = 10; + acc2[{0, 0, 0}] = 20; +} + +/*static*/ void ReducerTask::cpu_variant(legate::TaskContext& context) +{ + auto& red1 = context.reductions()[0]; + auto& red2 = context.reductions()[1]; + + auto acc1 = red1.reduce_accessor, true, 2>(); + auto acc2 = red2.reduce_accessor, true, 3>(); + + acc1[{0, 0}].reduce(10); + acc2[{0, 0, 0}].reduce(2); +} + +void test_writer_auto(legate::LibraryContext* context, + legate::LogicalStore scalar1, + legate::LogicalStore scalar2) +{ + auto runtime = legate::Runtime::get_runtime(); + auto task = runtime->create_task(context, WRITER); + auto part1 = task->declare_partition(); + auto part2 = task->declare_partition(); + task->add_output(scalar1, part1); + task->add_output(scalar2, part2); + runtime->submit(std::move(task)); +} + +void test_reducer_auto(legate::LibraryContext* context, + legate::LogicalStore scalar1, + legate::LogicalStore scalar2, + legate::LogicalStore store) +{ + auto runtime = legate::Runtime::get_runtime(); + auto task = runtime->create_task(context, REDUCER); + auto part1 = task->declare_partition(); + auto part2 = task->declare_partition(); + auto part3 = task->declare_partition(); + auto redop1 = LEGION_REDOP_BASE + LEGION_TYPE_TOTAL * LEGION_REDOP_KIND_SUM + scalar1.code(); + auto redop2 = LEGION_REDOP_BASE + LEGION_TYPE_TOTAL * LEGION_REDOP_KIND_PROD + scalar2.code(); + task->add_reduction(scalar1, redop1, part1); + task->add_reduction(scalar2, redop2, part2); + task->add_output(store, part3); + runtime->submit(std::move(task)); +} + +void test_reducer_manual(legate::LibraryContext* context, + legate::LogicalStore scalar1, + legate::LogicalStore scalar2) +{ + auto runtime = legate::Runtime::get_runtime(); + auto task = runtime->create_task(context, REDUCER, legate::Shape({2})); + auto redop1 = LEGION_REDOP_BASE + LEGION_TYPE_TOTAL * LEGION_REDOP_KIND_SUM + scalar1.code(); + auto redop2 = LEGION_REDOP_BASE + LEGION_TYPE_TOTAL * LEGION_REDOP_KIND_PROD + scalar2.code(); + task->add_reduction(scalar1, redop1); + task->add_reduction(scalar2, redop2); + runtime->submit(std::move(task)); +} + +void print_stores(legate::LibraryContext* context, + legate::LogicalStore scalar1, + legate::LogicalStore scalar2) +{ + auto runtime = legate::Runtime::get_runtime(); + auto p_scalar1 = scalar1.get_physical_store(context); + auto p_scalar2 = scalar2.get_physical_store(context); + auto acc1 = p_scalar1->read_accessor(); + auto acc2 = p_scalar2->read_accessor(); + std::stringstream ss; + ss << static_cast(acc1[{0, 0}]) << " " << acc2[{0, 0, 0}]; + logger.print() << ss.str(); +} + +void legate_main(int32_t argc, char** argv) +{ + legate::Core::perform_registration(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + auto scalar1 = runtime->create_store({1, 1}, legate::LegateTypeCode::INT8_LT, true); + auto scalar2 = runtime->create_store({1, 1, 1}, legate::LegateTypeCode::INT32_LT, true); + auto store = runtime->create_store({10}, legate::LegateTypeCode::INT64_LT); + test_writer_auto(context, scalar1, scalar2); + print_stores(context, scalar1, scalar2); + test_reducer_auto(context, scalar1, scalar2, store); + print_stores(context, scalar1, scalar2); + test_reducer_manual(context, scalar1, scalar2); + print_stores(context, scalar1, scalar2); +} + +} // namespace example + +int main(int argc, char** argv) +{ + legate::initialize(argc, argv); + + legate::set_main_function(example::legate_main); + + return legate::start(argc, argv); +} From d65adc8d140a7ed7b2311fd073534cc973ab43f7 Mon Sep 17 00:00:00 2001 From: Vladislav Zhurba Date: Fri, 31 Mar 2023 14:33:02 -0700 Subject: [PATCH 0105/1425] Write C++ version of store unit tests * Remove TODO comment from store methods These method names are good and will be expanded as needed in the future. * Run pre-commit * Formatting and spelling * Write C++ version of store unit tests Adds placeholders for missing transformations: - slice - transpose - delinearize See merge request legate/legate.core.internal!7 --- src/core/data/logical_store.cc | 19 ++++ src/core/data/logical_store.h | 6 ++ src/core/data/logical_store_detail.cc | 21 ++++ src/core/data/logical_store_detail.h | 6 ++ tests/cpp/unit/CMakeLists.txt | 31 ++++++ tests/cpp/unit/build.sh | 5 + tests/cpp/unit/test_store.cc | 143 ++++++++++++++++++++++++++ 7 files changed, 231 insertions(+) create mode 100644 tests/cpp/unit/CMakeLists.txt create mode 100755 tests/cpp/unit/build.sh create mode 100644 tests/cpp/unit/test_store.cc diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index 8771857172..d3c641f707 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -56,6 +56,10 @@ const Shape& LogicalStore::extents() const { return impl_->extents(); } size_t LogicalStore::volume() const { return impl_->volume(); } +bool LogicalStore::unbound() const { return impl_->unbound(); } + +bool LogicalStore::transformed() const { return impl_->transformed(); } + LogicalStore LogicalStore::promote(int32_t extra_dim, size_t dim_size) const { return LogicalStore(impl_->promote(extra_dim, dim_size)); @@ -71,6 +75,21 @@ LogicalStorePartition LogicalStore::partition_by_tiling(std::vector tile return LogicalStorePartition(impl_->partition_by_tiling(Shape(std::move(tile_shape)))); } +LogicalStore LogicalStore::slice(int32_t dim, std::slice sl) const +{ + return LogicalStore(impl_->slice(dim, sl)); +} + +LogicalStore LogicalStore::transpose(std::vector&& axes) const +{ + return LogicalStore(impl_->transpose(std::forward(axes))); +} + +LogicalStore LogicalStore::delinearize(int32_t dim, std::vector&& sizes) const +{ + return LogicalStore(impl_->delinearize(dim, std::forward(sizes))); +} + std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) { return impl_->get_physical_store(context); diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index bce92ae5af..80775ce5a1 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -17,6 +17,7 @@ #pragma once #include +#include #include "legion.h" @@ -60,10 +61,15 @@ class LogicalStore { LegateTypeCode code() const; const Shape& extents() const; size_t volume() const; + bool unbound() const; + bool transformed() const; public: LogicalStore promote(int32_t extra_dim, size_t dim_size) const; LogicalStore project(int32_t dim, int64_t index) const; + LogicalStore slice(int32_t dim, std::slice sl) const; + LogicalStore transpose(std::vector&& axes) const; + LogicalStore delinearize(int32_t dim, std::vector&& sizes) const; public: LogicalStorePartition partition_by_tiling(std::vector tile_shape) const; diff --git a/src/core/data/logical_store_detail.cc b/src/core/data/logical_store_detail.cc index 4670fa777e..b360ab777c 100644 --- a/src/core/data/logical_store_detail.cc +++ b/src/core/data/logical_store_detail.cc @@ -208,6 +208,8 @@ bool LogicalStore::scalar() const { return storage_->kind() == Storage::Kind::FU LegateTypeCode LogicalStore::code() const { return storage_->code(); } +bool LogicalStore::transformed() const { return !transform_->identity(); } + LogicalRegionField* LogicalStore::get_region_field() { return storage_->get_region_field(); } Legion::Future LogicalStore::get_future() { return storage_->get_future(); } @@ -271,6 +273,25 @@ std::shared_ptr LogicalStore::partition_by_tiling(Shape t return create_partition(std::move(partition)); } +std::shared_ptr LogicalStore::slice(int32_t dim, std::slice sl) const +{ + log_legate.error("Slice not implemented"); + return nullptr; +} + +std::shared_ptr LogicalStore::transpose(std::vector&& axes) const +{ + log_legate.error("Transpose not implemented"); + return nullptr; +} + +std::shared_ptr LogicalStore::delinearize(int32_t dim, + std::vector&& sizes) const +{ + log_legate.error("Delinearize not implemented"); + return nullptr; +} + std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) { if (unbound()) { diff --git a/src/core/data/logical_store_detail.h b/src/core/data/logical_store_detail.h index 59d5aaa906..9a8a40d27c 100644 --- a/src/core/data/logical_store_detail.h +++ b/src/core/data/logical_store_detail.h @@ -16,6 +16,8 @@ #pragma once +#include + #include "core/data/logical_region_field.h" #include "core/partitioning/partition.h" #include "core/runtime/runtime.h" @@ -123,6 +125,7 @@ class LogicalStore : public std::enable_shared_from_this { int32_t dim() const; bool scalar() const; LegateTypeCode code() const; + bool transformed() const; public: LogicalRegionField* get_region_field(); @@ -133,6 +136,9 @@ class LogicalStore : public std::enable_shared_from_this { public: std::shared_ptr promote(int32_t extra_dim, size_t dim_size) const; std::shared_ptr project(int32_t dim, int64_t index) const; + std::shared_ptr slice(int32_t dim, std::slice sl) const; + std::shared_ptr transpose(std::vector&& axes) const; + std::shared_ptr delinearize(int32_t dim, std::vector&& sizes) const; public: std::shared_ptr partition_by_tiling(Shape tile_shape); diff --git a/tests/cpp/unit/CMakeLists.txt b/tests/cpp/unit/CMakeLists.txt new file mode 100644 index 0000000000..8c3ca2e035 --- /dev/null +++ b/tests/cpp/unit/CMakeLists.txt @@ -0,0 +1,31 @@ +#============================================================================= +# Copyright 2023 NVIDIA Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +cmake_minimum_required(VERSION 3.22.1 FATAL_ERROR) + +project(test_store VERSION 0.1 LANGUAGES C CXX) + +if (NOT CMAKE_CXX_STANDARD) + set(CMAKE_CXX_STANDARD 17) +endif() + +find_package(legate_core REQUIRED) + +add_executable(test_store test_store.cc) + +target_link_libraries(test_store PRIVATE legate::core) + +install(TARGETS test_store DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/cmake-install") diff --git a/tests/cpp/unit/build.sh b/tests/cpp/unit/build.sh new file mode 100755 index 0000000000..b4effaa819 --- /dev/null +++ b/tests/cpp/unit/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +rm -rf build +cmake -B build -S . -D legate_core_ROOT=../../../build -D CMAKE_BUILD_TYPE=Debug +cmake --build build -j 4 diff --git a/tests/cpp/unit/test_store.cc b/tests/cpp/unit/test_store.cc new file mode 100644 index 0000000000..a210e271d5 --- /dev/null +++ b/tests/cpp/unit/test_store.cc @@ -0,0 +1,143 @@ +#include +#include +#include +#include + +#include "legate.h" + +namespace unit { + +void test_store_creation() +{ + // Bound + { + auto runtime = legate::Runtime::get_runtime(); + auto store = runtime->create_store({4, 4}, legate::LegateTypeCode::INT64_LT); + assert(!store.unbound()); + assert(store.dim() == 2); + assert((store.extents() == std::vector{4, 4})); + assert(store.code() == legate::LegateTypeCode::INT64_LT); + assert(!store.transformed()); + } + + // Unbound + { + auto runtime = legate::Runtime::get_runtime(); + auto store = runtime->create_store(legate::LegateTypeCode::INT64_LT); + assert(store.unbound()); + assert(store.dim() == 1); + assert(store.code() == legate::LegateTypeCode::INT64_LT); + assert(!store.transformed()); + // TODO: How can we do negative test cases? + // with pytest.raises(ValueError): + // store.shape + } +} + +void test_store_valid_transform() +{ + // Bound + auto runtime = legate::Runtime::get_runtime(); + auto store = runtime->create_store({4, 3}, legate::LegateTypeCode::INT64_LT); + + auto promoted = store.promote(0, 5); + assert((promoted.extents() == std::vector{5, 4, 3})); + assert(promoted.transformed()); + + auto projected = store.project(0, 1); + assert((projected.extents() == std::vector{ + 3, + })); + assert(projected.transformed()); + + auto sliced = store.slice(1, std::slice(1, 3, 1)); + // TODO: Enable once implemented + // assert((sliced.extents() == std::vector{4, 2})); + // assert(sliced.transformed()); + + auto transposed = store.transpose({1, 0}); + // TODO: Enable once implemented + // assert((transposed.extents() == std::vector{3, 4})); + // assert(transposed.transformed()); + + auto delinearized = store.delinearize(0, {2, 2}); + // TODO: Enable once implemented + // assert((delinearized.extents() == std::vector{2, 2, 3})); + // assert(delinearized.transformed()); +} + +void test_store_invalid_transform() +{ + // Bound + { + auto runtime = legate::Runtime::get_runtime(); + auto store = runtime->create_store({4, 3}, legate::LegateTypeCode::INT64_LT); + + // with pytest.raises(ValueError): + // store.promote(3, 5) + + // with pytest.raises(ValueError): + // store.promote(-3, 5) + + // with pytest.raises(ValueError): + // store.project(2, 1) + + // with pytest.raises(ValueError): + // store.project(-3, 1) + + // with pytest.raises(ValueError): + // store.project(0, 4) + + // with pytest.raises(ValueError): + // store.slice(2, slice(1, 3)) + + // with pytest.raises(NotImplementedError): + // store.slice(1, slice(1, 3, 2)) + + // with pytest.raises(ValueError): + // store.slice(1, slice(1, 4)) + + // with pytest.raises(ValueError): + // store.transpose((2,)) + + // with pytest.raises(ValueError): + // store.transpose((0, 0)) + + // with pytest.raises(ValueError): + // store.transpose((2, 0)) + + // with pytest.raises(ValueError): + // store.delinearize(2, (2, 3)) + + // with pytest.raises(ValueError): + // store.delinearize(0, (2, 3)) + } + + // Unbound + { + auto runtime = legate::Runtime::get_runtime(); + auto store = runtime->create_store(legate::LegateTypeCode::INT64_LT); + // with pytest.raises(ValueError): + // store.promote(1, 1) + } +} + +void legate_main(int32_t argc, char** argv) +{ + auto runtime = legate::Runtime::get_runtime(); + + test_store_creation(); + test_store_valid_transform(); + test_store_invalid_transform(); +} + +} // namespace unit + +int main(int argc, char** argv) +{ + legate::initialize(argc, argv); + + legate::set_main_function(unit::legate_main); + + return legate::start(argc, argv); +} From 909e54dafa98921d168224545a8697829941feb4 Mon Sep 17 00:00:00 2001 From: Vladislav Zhurba Date: Wed, 5 Apr 2023 12:30:19 -0700 Subject: [PATCH 0106/1425] Use gtest for integration and unit tests * Use gtest for integration and unit tests See merge request legate/legate.core.internal!8 --- .../manual_simple => }/CMakeLists.txt | 25 ++- tests/cpp/build.sh | 5 + .../{manual_simple => }/manual_simple.cc | 18 +-- tests/cpp/integration/manual_simple/build.sh | 5 - .../multi_scalar_out.cc | 18 +-- .../multi_scalar_out/CMakeLists.txt | 31 ---- .../cpp/integration/multi_scalar_out/build.sh | 5 - tests/cpp/run.sh | 4 + tests/cpp/unit/CMakeLists.txt | 31 ---- tests/cpp/unit/build.sh | 5 - tests/cpp/unit/store.cc | 124 +++++++++++++++ tests/cpp/unit/test_store.cc | 143 ------------------ 12 files changed, 172 insertions(+), 242 deletions(-) rename tests/cpp/{integration/manual_simple => }/CMakeLists.txt (56%) create mode 100755 tests/cpp/build.sh rename tests/cpp/integration/{manual_simple => }/manual_simple.cc (93%) delete mode 100755 tests/cpp/integration/manual_simple/build.sh rename tests/cpp/integration/{multi_scalar_out => }/multi_scalar_out.cc (95%) delete mode 100644 tests/cpp/integration/multi_scalar_out/CMakeLists.txt delete mode 100755 tests/cpp/integration/multi_scalar_out/build.sh create mode 100755 tests/cpp/run.sh delete mode 100644 tests/cpp/unit/CMakeLists.txt delete mode 100755 tests/cpp/unit/build.sh create mode 100644 tests/cpp/unit/store.cc delete mode 100644 tests/cpp/unit/test_store.cc diff --git a/tests/cpp/integration/manual_simple/CMakeLists.txt b/tests/cpp/CMakeLists.txt similarity index 56% rename from tests/cpp/integration/manual_simple/CMakeLists.txt rename to tests/cpp/CMakeLists.txt index 36b9afad48..bf39b9f8db 100644 --- a/tests/cpp/integration/manual_simple/CMakeLists.txt +++ b/tests/cpp/CMakeLists.txt @@ -16,16 +16,33 @@ cmake_minimum_required(VERSION 3.22.1 FATAL_ERROR) -project(manual_simple VERSION 0.1 LANGUAGES C CXX) +project(cpp_tests VERSION 0.1 LANGUAGES C CXX) if (NOT CMAKE_CXX_STANDARD) set(CMAKE_CXX_STANDARD 17) endif() +include(FetchContent) +FetchContent_Declare( + googletest + URL https://github.com/google/googletest/archive/refs/tags/v1.13.0.zip +) + +set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) +FetchContent_MakeAvailable(googletest) + find_package(legate_core REQUIRED) -add_executable(manual_simple manual_simple.cc) +enable_testing() + +file(GLOB integration_SRC ${PROJECT_SOURCE_DIR}/integration/*.cc) +file(GLOB unit_SRC ${PROJECT_SOURCE_DIR}/unit/*.cc) + +add_executable(cpp_tests ${integration_SRC} ${unit_SRC}) + +target_link_libraries(cpp_tests PRIVATE legate::core PRIVATE GTest::gtest_main) -target_link_libraries(manual_simple PRIVATE legate::core) +include(GoogleTest) +gtest_discover_tests(cpp_tests) -install(TARGETS manual_simple DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/cmake-install") +install(TARGETS cpp_tests DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/cmake-install") diff --git a/tests/cpp/build.sh b/tests/cpp/build.sh new file mode 100755 index 0000000000..dd0fb9706a --- /dev/null +++ b/tests/cpp/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +rm -rf build +cmake -B build -S . -D legate_core_ROOT=../../build -D CMAKE_BUILD_TYPE=Debug +cmake --build build -j 4 diff --git a/tests/cpp/integration/manual_simple/manual_simple.cc b/tests/cpp/integration/manual_simple.cc similarity index 93% rename from tests/cpp/integration/manual_simple/manual_simple.cc rename to tests/cpp/integration/manual_simple.cc index 67b84de58c..f6ede7f9aa 100644 --- a/tests/cpp/integration/manual_simple/manual_simple.cc +++ b/tests/cpp/integration/manual_simple.cc @@ -14,10 +14,12 @@ * */ +#include + #include "core/mapping/mapping.h" #include "legate.h" -namespace example { +namespace manualsimple { static const char* library_name = "manual"; static legate::Logger logger(library_name); @@ -41,7 +43,7 @@ struct Registrar { template struct BaseTask : public legate::LegateTask { - using Registrar = example::Registrar; + using Registrar = manualsimple::Registrar; }; struct HelloTask : public BaseTask { @@ -130,13 +132,11 @@ void legate_main(int32_t argc, char** argv) print_store(context, store); } -} // namespace example +} // namespace manualsimple -int main(int argc, char** argv) +TEST(Integration, ManualSimple) { - legate::initialize(argc, argv); - - legate::set_main_function(example::legate_main); - - return legate::start(argc, argv); + legate::initialize(0, NULL); + legate::set_main_function(manualsimple::legate_main); + legate::start(0, NULL); } diff --git a/tests/cpp/integration/manual_simple/build.sh b/tests/cpp/integration/manual_simple/build.sh deleted file mode 100755 index e6508cd229..0000000000 --- a/tests/cpp/integration/manual_simple/build.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -rm -rf build -cmake -B build -S . -D legate_core_ROOT=../../../../build -D CMAKE_BUILD_TYPE=Debug -cmake --build build -j 4 diff --git a/tests/cpp/integration/multi_scalar_out/multi_scalar_out.cc b/tests/cpp/integration/multi_scalar_out.cc similarity index 95% rename from tests/cpp/integration/multi_scalar_out/multi_scalar_out.cc rename to tests/cpp/integration/multi_scalar_out.cc index fb0719ba08..92a1b6b9a9 100644 --- a/tests/cpp/integration/multi_scalar_out/multi_scalar_out.cc +++ b/tests/cpp/integration/multi_scalar_out.cc @@ -14,10 +14,12 @@ * */ +#include + #include "core/mapping/mapping.h" #include "legate.h" -namespace example { +namespace multiscalarout { static const char* library_name = "multi_scalar"; static legate::Logger logger(library_name); @@ -42,7 +44,7 @@ struct Registrar { template struct BaseTask : public legate::LegateTask { - using Registrar = example::Registrar; + using Registrar = multiscalarout::Registrar; }; struct WriterTask : public BaseTask { @@ -182,13 +184,11 @@ void legate_main(int32_t argc, char** argv) print_stores(context, scalar1, scalar2); } -} // namespace example +} // namespace multiscalarout -int main(int argc, char** argv) +TEST(Integration, ManualScalarOut) { - legate::initialize(argc, argv); - - legate::set_main_function(example::legate_main); - - return legate::start(argc, argv); + legate::initialize(0, NULL); + legate::set_main_function(multiscalarout::legate_main); + legate::start(0, NULL); } diff --git a/tests/cpp/integration/multi_scalar_out/CMakeLists.txt b/tests/cpp/integration/multi_scalar_out/CMakeLists.txt deleted file mode 100644 index 6c7641328d..0000000000 --- a/tests/cpp/integration/multi_scalar_out/CMakeLists.txt +++ /dev/null @@ -1,31 +0,0 @@ -#============================================================================= -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= - -cmake_minimum_required(VERSION 3.22.1 FATAL_ERROR) - -project(multi_scalar_out VERSION 0.1 LANGUAGES C CXX) - -if (NOT CMAKE_CXX_STANDARD) - set(CMAKE_CXX_STANDARD 17) -endif() - -find_package(legate_core REQUIRED) - -add_executable(multi_scalar_out multi_scalar_out.cc) - -target_link_libraries(multi_scalar_out PRIVATE legate::core) - -install(TARGETS multi_scalar_out DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/cmake-install") diff --git a/tests/cpp/integration/multi_scalar_out/build.sh b/tests/cpp/integration/multi_scalar_out/build.sh deleted file mode 100755 index e6508cd229..0000000000 --- a/tests/cpp/integration/multi_scalar_out/build.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -rm -rf build -cmake -B build -S . -D legate_core_ROOT=../../../../build -D CMAKE_BUILD_TYPE=Debug -cmake --build build -j 4 diff --git a/tests/cpp/run.sh b/tests/cpp/run.sh new file mode 100755 index 0000000000..1717b0a8ed --- /dev/null +++ b/tests/cpp/run.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +cd build +ctest --output-on-failure diff --git a/tests/cpp/unit/CMakeLists.txt b/tests/cpp/unit/CMakeLists.txt deleted file mode 100644 index 8c3ca2e035..0000000000 --- a/tests/cpp/unit/CMakeLists.txt +++ /dev/null @@ -1,31 +0,0 @@ -#============================================================================= -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= - -cmake_minimum_required(VERSION 3.22.1 FATAL_ERROR) - -project(test_store VERSION 0.1 LANGUAGES C CXX) - -if (NOT CMAKE_CXX_STANDARD) - set(CMAKE_CXX_STANDARD 17) -endif() - -find_package(legate_core REQUIRED) - -add_executable(test_store test_store.cc) - -target_link_libraries(test_store PRIVATE legate::core) - -install(TARGETS test_store DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/cmake-install") diff --git a/tests/cpp/unit/build.sh b/tests/cpp/unit/build.sh deleted file mode 100755 index b4effaa819..0000000000 --- a/tests/cpp/unit/build.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -rm -rf build -cmake -B build -S . -D legate_core_ROOT=../../../build -D CMAKE_BUILD_TYPE=Debug -cmake --build build -j 4 diff --git a/tests/cpp/unit/store.cc b/tests/cpp/unit/store.cc new file mode 100644 index 0000000000..e718290e07 --- /dev/null +++ b/tests/cpp/unit/store.cc @@ -0,0 +1,124 @@ +#include +#include +#include +#include +#include + +#include "legate.h" + +namespace unit { + +void store_creation(int32_t argc, char** argv) +{ + // Bound + { + auto runtime = legate::Runtime::get_runtime(); + auto store = runtime->create_store({4, 4}, legate::LegateTypeCode::INT64_LT); + EXPECT_FALSE(store.unbound()); + EXPECT_EQ(store.dim(), 2); + EXPECT_EQ(store.extents(), (std::vector{4, 4})); + EXPECT_EQ(store.code(), legate::LegateTypeCode::INT64_LT); + EXPECT_FALSE(store.transformed()); + } + + // Unbound + { + auto runtime = legate::Runtime::get_runtime(); + auto store = runtime->create_store(legate::LegateTypeCode::INT64_LT); + EXPECT_TRUE(store.unbound()); + EXPECT_EQ(store.dim(), 1); + EXPECT_EQ(store.code(), legate::LegateTypeCode::INT64_LT); + EXPECT_FALSE(store.transformed()); + EXPECT_THROW(store.extents(), std::invalid_argument); + } +} + +void store_valid_transform(int32_t argc, char** argv) +{ + // Bound + auto runtime = legate::Runtime::get_runtime(); + auto store = runtime->create_store({4, 3}, legate::LegateTypeCode::INT64_LT); + + auto promoted = store.promote(0, 5); + EXPECT_EQ(promoted.extents(), (std::vector{5, 4, 3})); + EXPECT_TRUE(promoted.transformed()); + + auto projected = store.project(0, 1); + EXPECT_EQ(projected.extents(), + (std::vector{ + 3, + })); + EXPECT_TRUE(projected.transformed()); + + auto sliced = store.slice(1, std::slice(1, 3, 1)); + EXPECT_EQ(sliced.extents(), (std::vector{4, 2})); + EXPECT_TRUE(sliced.transformed()); + + auto transposed = store.transpose({1, 0}); + EXPECT_EQ(transposed.extents(), (std::vector{3, 4})); + EXPECT_TRUE(transposed.transformed()); + + auto delinearized = store.delinearize(0, {2, 2}); + EXPECT_EQ(delinearized.extents(), (std::vector{2, 2, 3})); + EXPECT_TRUE(delinearized.transformed()); +} + +void store_invalid_transform(int32_t argc, char** argv) +{ + // Bound + { + auto runtime = legate::Runtime::get_runtime(); + auto store = runtime->create_store({4, 3}, legate::LegateTypeCode::INT64_LT); + + EXPECT_THROW(store.promote(3, 5), std::invalid_argument); + EXPECT_THROW(store.promote(-3, 5), std::invalid_argument); + + EXPECT_THROW(store.project(2, 1), std::invalid_argument); + EXPECT_THROW(store.project(-3, 1), std::invalid_argument); + EXPECT_THROW(store.project(0, 4), std::invalid_argument); + + EXPECT_THROW(store.slice(2, std::slice(1, 3, 1)), std::invalid_argument); + EXPECT_THROW(store.slice(1, std::slice(1, 3, 2)), std::invalid_argument); + EXPECT_THROW(store.slice(1, std::slice(1, 4, 1)), std::invalid_argument); + + EXPECT_THROW(store.transpose({ + 2, + }), + std::invalid_argument); + EXPECT_THROW(store.transpose({0, 0}), std::invalid_argument); + EXPECT_THROW(store.transpose({2, 0}), std::invalid_argument); + + EXPECT_THROW(store.delinearize(2, {2, 3}), std::invalid_argument); + EXPECT_THROW(store.delinearize(0, {2, 3}), std::invalid_argument); + } + + // Unbound + { + auto runtime = legate::Runtime::get_runtime(); + auto store = runtime->create_store(legate::LegateTypeCode::INT64_LT); + EXPECT_THROW(store.promote(1, 1), std::invalid_argument); + } +} + +} // namespace unit + +TEST(Store, Creation) +{ + legate::initialize(0, NULL); + legate::set_main_function(unit::store_creation); + legate::start(0, NULL); +} + +TEST(Store, Transform) +{ + legate::initialize(0, NULL); + legate::set_main_function(unit::store_valid_transform); + legate::start(0, NULL); +} + +TEST(Store, InvalidTransform) +{ + legate::initialize(0, NULL); + legate::set_main_function(unit::store_invalid_transform); + legate::start(0, NULL); +} diff --git a/tests/cpp/unit/test_store.cc b/tests/cpp/unit/test_store.cc deleted file mode 100644 index a210e271d5..0000000000 --- a/tests/cpp/unit/test_store.cc +++ /dev/null @@ -1,143 +0,0 @@ -#include -#include -#include -#include - -#include "legate.h" - -namespace unit { - -void test_store_creation() -{ - // Bound - { - auto runtime = legate::Runtime::get_runtime(); - auto store = runtime->create_store({4, 4}, legate::LegateTypeCode::INT64_LT); - assert(!store.unbound()); - assert(store.dim() == 2); - assert((store.extents() == std::vector{4, 4})); - assert(store.code() == legate::LegateTypeCode::INT64_LT); - assert(!store.transformed()); - } - - // Unbound - { - auto runtime = legate::Runtime::get_runtime(); - auto store = runtime->create_store(legate::LegateTypeCode::INT64_LT); - assert(store.unbound()); - assert(store.dim() == 1); - assert(store.code() == legate::LegateTypeCode::INT64_LT); - assert(!store.transformed()); - // TODO: How can we do negative test cases? - // with pytest.raises(ValueError): - // store.shape - } -} - -void test_store_valid_transform() -{ - // Bound - auto runtime = legate::Runtime::get_runtime(); - auto store = runtime->create_store({4, 3}, legate::LegateTypeCode::INT64_LT); - - auto promoted = store.promote(0, 5); - assert((promoted.extents() == std::vector{5, 4, 3})); - assert(promoted.transformed()); - - auto projected = store.project(0, 1); - assert((projected.extents() == std::vector{ - 3, - })); - assert(projected.transformed()); - - auto sliced = store.slice(1, std::slice(1, 3, 1)); - // TODO: Enable once implemented - // assert((sliced.extents() == std::vector{4, 2})); - // assert(sliced.transformed()); - - auto transposed = store.transpose({1, 0}); - // TODO: Enable once implemented - // assert((transposed.extents() == std::vector{3, 4})); - // assert(transposed.transformed()); - - auto delinearized = store.delinearize(0, {2, 2}); - // TODO: Enable once implemented - // assert((delinearized.extents() == std::vector{2, 2, 3})); - // assert(delinearized.transformed()); -} - -void test_store_invalid_transform() -{ - // Bound - { - auto runtime = legate::Runtime::get_runtime(); - auto store = runtime->create_store({4, 3}, legate::LegateTypeCode::INT64_LT); - - // with pytest.raises(ValueError): - // store.promote(3, 5) - - // with pytest.raises(ValueError): - // store.promote(-3, 5) - - // with pytest.raises(ValueError): - // store.project(2, 1) - - // with pytest.raises(ValueError): - // store.project(-3, 1) - - // with pytest.raises(ValueError): - // store.project(0, 4) - - // with pytest.raises(ValueError): - // store.slice(2, slice(1, 3)) - - // with pytest.raises(NotImplementedError): - // store.slice(1, slice(1, 3, 2)) - - // with pytest.raises(ValueError): - // store.slice(1, slice(1, 4)) - - // with pytest.raises(ValueError): - // store.transpose((2,)) - - // with pytest.raises(ValueError): - // store.transpose((0, 0)) - - // with pytest.raises(ValueError): - // store.transpose((2, 0)) - - // with pytest.raises(ValueError): - // store.delinearize(2, (2, 3)) - - // with pytest.raises(ValueError): - // store.delinearize(0, (2, 3)) - } - - // Unbound - { - auto runtime = legate::Runtime::get_runtime(); - auto store = runtime->create_store(legate::LegateTypeCode::INT64_LT); - // with pytest.raises(ValueError): - // store.promote(1, 1) - } -} - -void legate_main(int32_t argc, char** argv) -{ - auto runtime = legate::Runtime::get_runtime(); - - test_store_creation(); - test_store_valid_transform(); - test_store_invalid_transform(); -} - -} // namespace unit - -int main(int argc, char** argv) -{ - legate::initialize(argc, argv); - - legate::set_main_function(unit::legate_main); - - return legate::start(argc, argv); -} From a5633b9fe4b9e959eb5c33420db1e8e181ba5413 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 7 Apr 2023 11:57:06 -0700 Subject: [PATCH 0107/1425] Should install shape.h * Should install shape.h See merge request legate/legate.core.internal!10 --- legate_core_cpp.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index da23ab33fe..e6ef1ae30d 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -409,6 +409,7 @@ install( src/core/data/logical_store.h src/core/data/scalar.h src/core/data/scalar.inl + src/core/data/shape.h src/core/data/store.h src/core/data/store.inl src/core/data/transform.h From 1b49d5835a335f3c0b1dfd1f6d5fef9a1f336a23 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Tue, 11 Apr 2023 16:10:41 -0700 Subject: [PATCH 0108/1425] Rename scalar to has_scalar_storage to clarify what it does * Rename scalar to has_scalar_storage to clarify what it does See merge request legate/legate.core.internal!11 --- src/core/data/logical_store_detail.cc | 12 ++++++------ src/core/data/logical_store_detail.h | 2 +- src/core/runtime/launcher.cc | 2 +- src/core/runtime/operation.cc | 14 ++++++++------ 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/core/data/logical_store_detail.cc b/src/core/data/logical_store_detail.cc index b360ab777c..c410549cac 100644 --- a/src/core/data/logical_store_detail.cc +++ b/src/core/data/logical_store_detail.cc @@ -204,7 +204,7 @@ int32_t LogicalStore::dim() const return unbound() ? storage_->dim() : static_cast(extents_.size()); } -bool LogicalStore::scalar() const { return storage_->kind() == Storage::Kind::FUTURE; } +bool LogicalStore::has_scalar_storage() const { return storage_->kind() == Storage::Kind::FUTURE; } LegateTypeCode LogicalStore::code() const { return storage_->code(); } @@ -217,7 +217,7 @@ Legion::Future LogicalStore::get_future() { return storage_->get_future(); } void LogicalStore::set_region_field(std::shared_ptr&& region_field) { #ifdef DEBUG_LEGATE - assert(!scalar()); + assert(!has_scalar_storage()); #endif storage_->set_region_field(std::move(region_field)); extents_ = storage_->extents(); @@ -226,7 +226,7 @@ void LogicalStore::set_region_field(std::shared_ptr&& region void LogicalStore::set_future(Legion::Future future) { #ifdef DEBUG_LEGATE - assert(scalar()); + assert(has_scalar_storage()); #endif storage_->set_future(future); } @@ -338,7 +338,7 @@ Legion::ProjectionID LogicalStore::compute_projection(int32_t launch_ndim) const std::unique_ptr LogicalStore::create_projection(const Partition* partition, int32_t launch_ndim) { - if (scalar()) return std::make_unique(); + if (has_scalar_storage()) return std::make_unique(); // We're about to create a legion partition for this store, so the store should have its region // created. @@ -366,7 +366,7 @@ std::shared_ptr LogicalStore::find_or_create_key_partition() { if (key_partition_ != nullptr) return key_partition_; - if (scalar()) { + if (has_scalar_storage()) { key_partition_ = create_no_partition(); return key_partition_; } @@ -399,7 +399,7 @@ std::shared_ptr LogicalStore::create_partition( void LogicalStore::pack(BufferBuilder& buffer) const { - buffer.pack(scalar()); + buffer.pack(has_scalar_storage()); buffer.pack(unbound()); buffer.pack(dim()); buffer.pack(code()); diff --git a/src/core/data/logical_store_detail.h b/src/core/data/logical_store_detail.h index 9a8a40d27c..e1b2c2771d 100644 --- a/src/core/data/logical_store_detail.h +++ b/src/core/data/logical_store_detail.h @@ -123,7 +123,7 @@ class LogicalStore : public std::enable_shared_from_this { // Size of the backing storage size_t storage_size() const; int32_t dim() const; - bool scalar() const; + bool has_scalar_storage() const; LegateTypeCode code() const; bool transformed() const; diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/launcher.cc index d14c7ad949..03b70c73c8 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -137,7 +137,7 @@ void TaskLauncher::add_store(std::vector& args, { auto redop = proj->redop; - if (store->scalar()) { + if (store->has_scalar_storage()) { auto has_storage = privilege != WRITE_ONLY; auto read_only = privilege == READ_ONLY; if (has_storage) futures_.push_back(store->get_future()); diff --git a/src/core/runtime/operation.cc b/src/core/runtime/operation.cc index 19a2de8c29..8e5b7f38c1 100644 --- a/src/core/runtime/operation.cc +++ b/src/core/runtime/operation.cc @@ -211,7 +211,7 @@ void AutoTask::add_input(LogicalStore store, const Variable* partition_symbol) void AutoTask::add_output(LogicalStore store, const Variable* partition_symbol) { - if (store.impl()->scalar()) scalar_outputs_.push_back(outputs_.size()); + if (store.impl()->has_scalar_storage()) scalar_outputs_.push_back(outputs_.size()); add_store(outputs_, store, partition_symbol); } @@ -219,7 +219,7 @@ void AutoTask::add_reduction(LogicalStore store, Legion::ReductionOpID redop, const Variable* partition_symbol) { - if (store.impl()->scalar()) scalar_reductions_.push_back(reductions_.size()); + if (store.impl()->has_scalar_storage()) scalar_reductions_.push_back(reductions_.size()); add_store(reductions_, store, partition_symbol); reduction_ops_.push_back(redop); } @@ -267,13 +267,13 @@ void ManualTask::add_input(LogicalStore store) { add_store(inputs_, store, creat void ManualTask::add_output(LogicalStore store) { - if (store.impl()->scalar()) scalar_outputs_.push_back(outputs_.size()); + if (store.impl()->has_scalar_storage()) scalar_outputs_.push_back(outputs_.size()); add_store(outputs_, store, create_no_partition()); } void ManualTask::add_reduction(LogicalStore store, Legion::ReductionOpID redop) { - if (store.impl()->scalar()) scalar_reductions_.push_back(reductions_.size()); + if (store.impl()->has_scalar_storage()) scalar_reductions_.push_back(reductions_.size()); add_store(reductions_, store, create_no_partition()); reduction_ops_.push_back(redop); } @@ -285,13 +285,15 @@ void ManualTask::add_input(LogicalStorePartition store_partition) void ManualTask::add_output(LogicalStorePartition store_partition) { - if (store_partition.store().impl()->scalar()) scalar_outputs_.push_back(outputs_.size()); + if (store_partition.store().impl()->has_scalar_storage()) + scalar_outputs_.push_back(outputs_.size()); add_store(outputs_, store_partition.store(), store_partition.partition()); } void ManualTask::add_reduction(LogicalStorePartition store_partition, Legion::ReductionOpID redop) { - if (store_partition.store().impl()->scalar()) scalar_reductions_.push_back(reductions_.size()); + if (store_partition.store().impl()->has_scalar_storage()) + scalar_reductions_.push_back(reductions_.size()); add_store(reductions_, store_partition.store(), store_partition.partition()); reduction_ops_.push_back(redop); } From 12203204c00bce5f5d246a5c760c3daed5b8d53a Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 14 Apr 2023 22:45:45 -0700 Subject: [PATCH 0109/1425] Post-merge clean-up * Post-merge clean-up See merge request legate/legate.core.internal!12 --- src/core/data/logical_store_detail.cc | 2 +- src/core/partitioning/constraint.cc | 4 +- src/core/partitioning/partition.cc | 2 +- src/core/runtime/launcher.cc | 7 +--- src/core/runtime/launcher.h | 3 +- src/core/runtime/operation.cc | 23 ++++++----- src/core/runtime/operation.h | 10 ++--- src/core/runtime/runtime.cc | 17 +++----- src/core/runtime/runtime.h | 7 +--- src/core/task/task.cc | 2 +- tests/cpp/integration/manual_simple.cc | 43 ++------------------- tests/cpp/integration/multi_scalar_out.cc | 47 +++-------------------- 12 files changed, 39 insertions(+), 128 deletions(-) diff --git a/src/core/data/logical_store_detail.cc b/src/core/data/logical_store_detail.cc index c410549cac..5b229f36c5 100644 --- a/src/core/data/logical_store_detail.cc +++ b/src/core/data/logical_store_detail.cc @@ -414,7 +414,7 @@ std::string LogicalStore::to_string() const ss << ", transform: " << *transform_ << "}"; else ss << "}"; - return ss.str(); + return std::move(ss).str(); } //////////////////////////////////////////////////// diff --git a/src/core/partitioning/constraint.cc b/src/core/partitioning/constraint.cc index 7846db8e55..fcb7a4760e 100644 --- a/src/core/partitioning/constraint.cc +++ b/src/core/partitioning/constraint.cc @@ -47,7 +47,7 @@ std::string Variable::to_string() const { std::stringstream ss; ss << "X" << id_ << "{" << op_->to_string() << "}"; - return ss.str(); + return std::move(ss).str(); } Alignment::Alignment(std::unique_ptr&& lhs, std::unique_ptr&& rhs) @@ -65,7 +65,7 @@ std::string Alignment::to_string() const { std::stringstream ss; ss << "Align(" << lhs_->to_string() << ", " << rhs_->to_string() << ")"; - return ss.str(); + return std::move(ss).str(); } std::unique_ptr align(const Variable* lhs, const Variable* rhs) diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index e543618caf..f6895a1c48 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -146,7 +146,7 @@ std::string Tiling::to_string() const std::stringstream ss; ss << "Tiling(tile:" << tile_shape_ << ",colors:" << color_shape_ << ",offset:" << offsets_ << ")"; - return ss.str(); + return std::move(ss).str(); } std::unique_ptr create_no_partition() { return std::make_unique(); } diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/launcher.cc index ab14328b5d..d274361b9f 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -28,11 +28,8 @@ namespace legate { -TaskLauncher::TaskLauncher(LibraryContext* library, - int64_t task_id, - int64_t mapper_id /*= 0*/, - int64_t tag /*= 0*/) - : library_(library), task_id_(task_id), mapper_id_(mapper_id), tag_(tag) +TaskLauncher::TaskLauncher(LibraryContext* library, int64_t task_id, int64_t tag /*= 0*/) + : library_(library), task_id_(task_id), tag_(tag) { req_analyzer_ = new RequirementAnalyzer(); out_analyzer_ = new OutputRequirementAnalyzer(); diff --git a/src/core/runtime/launcher.h b/src/core/runtime/launcher.h index af8848b597..76ae5c5f6c 100644 --- a/src/core/runtime/launcher.h +++ b/src/core/runtime/launcher.h @@ -40,7 +40,7 @@ class LogicalStore; class TaskLauncher { public: - TaskLauncher(LibraryContext* library, int64_t task_id, int64_t mapper_id = 0, int64_t tag = 0); + TaskLauncher(LibraryContext* library, int64_t task_id, int64_t tag = 0); ~TaskLauncher(); public: @@ -91,7 +91,6 @@ class TaskLauncher { private: LibraryContext* library_; int64_t task_id_; - int64_t mapper_id_; int64_t tag_; private: diff --git a/src/core/runtime/operation.cc b/src/core/runtime/operation.cc index 8e5b7f38c1..8a71a63936 100644 --- a/src/core/runtime/operation.cc +++ b/src/core/runtime/operation.cc @@ -36,8 +36,8 @@ namespace legate { // legate::Operation //////////////////////////////////////////////////// -Operation::Operation(LibraryContext* library, uint64_t unique_id, int64_t mapper_id) - : library_(library), unique_id_(unique_id), mapper_id_(mapper_id) +Operation::Operation(LibraryContext* library, uint64_t unique_id) + : library_(library), unique_id_(unique_id) { } @@ -60,8 +60,8 @@ detail::LogicalStore* Operation::find_store(const Variable* part_symb) const // legate::Task //////////////////////////////////////////////////// -Task::Task(LibraryContext* library, int64_t task_id, uint64_t unique_id, int64_t mapper_id /*=0*/) - : Operation(library, unique_id, mapper_id), task_id_(task_id) +Task::Task(LibraryContext* library, int64_t task_id, uint64_t unique_id) + : Operation(library, unique_id), task_id_(task_id) { } @@ -78,7 +78,7 @@ struct field_size_fn { void Task::launch(Strategy* p_strategy) { auto& strategy = *p_strategy; - TaskLauncher launcher(library_, task_id_, mapper_id_); + TaskLauncher launcher(library_, task_id_); auto launch_domain = strategy.launch_domain(this); auto launch_ndim = launch_domain != nullptr ? launch_domain->dim : 0; @@ -191,16 +191,16 @@ void Task::demux_scalar_stores(const Legion::FutureMap& result, const Legion::Do std::string Task::to_string() const { std::stringstream ss; - ss << library_->get_task_name(task_id_) << ":" << unique_id_; - return ss.str(); + ss << library_->find_task(task_id_)->name() << ":" << unique_id_; + return std::move(ss).str(); } //////////////////////////////////////////////////// // legate::AutoTask //////////////////////////////////////////////////// -AutoTask::AutoTask(LibraryContext* library, int64_t task_id, uint64_t unique_id, int64_t mapper_id) - : Task(library, task_id, unique_id, mapper_id) +AutoTask::AutoTask(LibraryContext* library, int64_t task_id, uint64_t unique_id) + : Task(library, task_id, unique_id) { } @@ -256,9 +256,8 @@ ManualTask::~ManualTask() {} ManualTask::ManualTask(LibraryContext* library, int64_t task_id, const Shape& launch_shape, - uint64_t unique_id, - int64_t mapper_id) - : Task(library, task_id, unique_id, mapper_id), strategy_(std::make_unique()) + uint64_t unique_id) + : Task(library, task_id, unique_id), strategy_(std::make_unique()) { strategy_->set_launch_shape(this, launch_shape); } diff --git a/src/core/runtime/operation.h b/src/core/runtime/operation.h index 05b6a4d9da..c62c4a4a1e 100644 --- a/src/core/runtime/operation.h +++ b/src/core/runtime/operation.h @@ -41,7 +41,7 @@ class LogicalStore; class Operation { protected: using StoreArg = std::pair; - Operation(LibraryContext* library, uint64_t unique_id, int64_t mapper_id); + Operation(LibraryContext* library, uint64_t unique_id); public: virtual ~Operation() {} @@ -58,7 +58,6 @@ class Operation { protected: LibraryContext* library_; uint64_t unique_id_; - int64_t mapper_id_; protected: std::set> all_stores_{}; @@ -75,7 +74,7 @@ class Operation { class Task : public Operation { protected: - Task(LibraryContext* library, int64_t task_id, uint64_t unique_id, int64_t mapper_id); + Task(LibraryContext* library, int64_t task_id, uint64_t unique_id); public: virtual ~Task() {} @@ -103,7 +102,7 @@ class Task : public Operation { class AutoTask : public Task { public: friend class Runtime; - AutoTask(LibraryContext* library, int64_t task_id, uint64_t unique_id, int64_t mapper_id); + AutoTask(LibraryContext* library, int64_t task_id, uint64_t unique_id); public: ~AutoTask() {} @@ -134,8 +133,7 @@ class ManualTask : public Task { ManualTask(LibraryContext* library, int64_t task_id, const Shape& launch_shape, - uint64_t unique_id, - int64_t mapper_id); + uint64_t unique_id); public: ~ManualTask(); diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 7b7ad37409..dc68e1670d 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -647,20 +647,17 @@ void Runtime::post_startup_initialization(Legion::Context legion_context) } // This function should be moved to the library context -std::unique_ptr Runtime::create_task(LibraryContext* library, - int64_t task_id, - int64_t mapper_id /*=0*/) +std::unique_ptr Runtime::create_task(LibraryContext* library, int64_t task_id) { - auto task = new AutoTask(library, task_id, next_unique_id_++, mapper_id); + auto task = new AutoTask(library, task_id, next_unique_id_++); return std::unique_ptr(task); } std::unique_ptr Runtime::create_task(LibraryContext* library, int64_t task_id, - const Shape& launch_shape, - int64_t mapper_id /*=0*/) + const Shape& launch_shape) { - auto task = new ManualTask(library, task_id, launch_shape, next_unique_id_++, mapper_id); + auto task = new ManualTask(library, task_id, launch_shape, next_unique_id_++); return std::unique_ptr(task); } @@ -871,8 +868,7 @@ Legion::Future Runtime::extract_scalar(const Legion::Future& result, uint32_t id // FIXME: One of two things should happen: // 1) the variant should be picked based on available processor kinds // 2) the variant should be picked by the core mapper - TaskLauncher launcher( - core_context_, LEGATE_CORE_EXTRACT_SCALAR_TASK_ID, 0 /*mapper_id*/, LEGATE_CPU_VARIANT); + TaskLauncher launcher(core_context_, LEGATE_CORE_EXTRACT_SCALAR_TASK_ID, LEGATE_CPU_VARIANT); launcher.add_future(result); launcher.add_scalar(Scalar(idx)); return launcher.execute_single(); @@ -885,8 +881,7 @@ Legion::FutureMap Runtime::extract_scalar(const Legion::FutureMap& result, // FIXME: One of two things should happen: // 1) the variant should be picked based on available processor kinds // 2) the variant should be picked by the core mapper - TaskLauncher launcher( - core_context_, LEGATE_CORE_EXTRACT_SCALAR_TASK_ID, 0 /*mapper_id*/, LEGATE_CPU_VARIANT); + TaskLauncher launcher(core_context_, LEGATE_CORE_EXTRACT_SCALAR_TASK_ID, LEGATE_CPU_VARIANT); launcher.add_future_map(result); launcher.add_scalar(Scalar(idx)); return launcher.execute(launch_domain); diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index cf50bcfc51..e19c43a807 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -144,13 +144,10 @@ class Runtime { T get_tunable(const LibraryContext* context, int64_t tunable_id); public: - std::unique_ptr create_task(LibraryContext* library, - int64_t task_id, - int64_t mapper_id = 0); + std::unique_ptr create_task(LibraryContext* library, int64_t task_id); std::unique_ptr create_task(LibraryContext* library, int64_t task_id, - const Shape& launch_shape, - int64_t mapper_id = 0); + const Shape& launch_shape); void submit(std::unique_ptr op); public: diff --git a/src/core/task/task.cc b/src/core/task/task.cc index 898fdfc064..a8f6a58736 100644 --- a/src/core/task/task.cc +++ b/src/core/task/task.cc @@ -62,7 +62,7 @@ void task_wrapper(VariantImpl variant_impl, std::stringstream ss; ss << task_name; if (!task->get_provenance_string().empty()) ss << " @ " + task->get_provenance_string(); - std::string msg = ss.str(); + std::string msg = std::move(ss).str(); nvtx::Range auto_range(msg.c_str()); #endif diff --git a/tests/cpp/integration/manual_simple.cc b/tests/cpp/integration/manual_simple.cc index f6ede7f9aa..54bf6eadb0 100644 --- a/tests/cpp/integration/manual_simple.cc +++ b/tests/cpp/integration/manual_simple.cc @@ -28,53 +28,16 @@ enum TaskIDs { HELLO = 0, }; -struct Registrar { - template - static void record_variant(Args&&... args) - { - get_registrar().record_variant(std::forward(args)...); - } - static legate::TaskRegistrar& get_registrar() - { - static legate::TaskRegistrar registrar; - return registrar; - } -}; - -template -struct BaseTask : public legate::LegateTask { - using Registrar = manualsimple::Registrar; -}; - -struct HelloTask : public BaseTask { +struct HelloTask : public legate::LegateTask { static const int32_t TASK_ID = HELLO; static void cpu_variant(legate::TaskContext& context); }; -struct Mapper : public legate::mapping::LegateMapper { - void set_machine(const legate::mapping::MachineQueryInterface* machine) override {} - legate::mapping::TaskTarget task_target( - const legate::mapping::Task& task, - const std::vector& options) override - { - return options.front(); - } - std::vector store_mappings( - const legate::mapping::Task& task, - const std::vector& options) override - { - return {}; - } - legate::Scalar tunable_value(legate::TunableID tunable_id) override { return legate::Scalar(); } -}; - void register_tasks() { auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->create_library(library_name, legate::ResourceConfig()); - HelloTask::register_variants(); - Registrar::get_registrar().register_all_tasks(*context); - context->register_mapper(std::make_unique()); + auto context = runtime->create_library(library_name); + HelloTask::register_variants(*context); } /*static*/ void HelloTask::cpu_variant(legate::TaskContext& context) diff --git a/tests/cpp/integration/multi_scalar_out.cc b/tests/cpp/integration/multi_scalar_out.cc index 92a1b6b9a9..fdc4df8a32 100644 --- a/tests/cpp/integration/multi_scalar_out.cc +++ b/tests/cpp/integration/multi_scalar_out.cc @@ -29,59 +29,22 @@ enum TaskIDs { REDUCER = 1, }; -struct Registrar { - template - static void record_variant(Args&&... args) - { - get_registrar().record_variant(std::forward(args)...); - } - static legate::TaskRegistrar& get_registrar() - { - static legate::TaskRegistrar registrar; - return registrar; - } -}; - -template -struct BaseTask : public legate::LegateTask { - using Registrar = multiscalarout::Registrar; -}; - -struct WriterTask : public BaseTask { +struct WriterTask : public legate::LegateTask { static const int32_t TASK_ID = WRITER; static void cpu_variant(legate::TaskContext& context); }; -struct ReducerTask : public BaseTask { +struct ReducerTask : public legate::LegateTask { static const int32_t TASK_ID = REDUCER; static void cpu_variant(legate::TaskContext& context); }; -struct Mapper : public legate::mapping::LegateMapper { - void set_machine(const legate::mapping::MachineQueryInterface* machine) override {} - legate::mapping::TaskTarget task_target( - const legate::mapping::Task& task, - const std::vector& options) override - { - return options.front(); - } - std::vector store_mappings( - const legate::mapping::Task& task, - const std::vector& options) override - { - return {}; - } - legate::Scalar tunable_value(legate::TunableID tunable_id) override { return legate::Scalar(); } -}; - void register_tasks() { auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->create_library(library_name, legate::ResourceConfig()); - WriterTask::register_variants(); - ReducerTask::register_variants(); - Registrar::get_registrar().register_all_tasks(*context); - context->register_mapper(std::make_unique()); + auto context = runtime->create_library(library_name); + WriterTask::register_variants(*context); + ReducerTask::register_variants(*context); } /*static*/ void WriterTask::cpu_variant(legate::TaskContext& context) From 7c97a704256cee1555ce337d5e6136bc98f7ad7c Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Sun, 7 May 2023 22:21:23 -0700 Subject: [PATCH 0110/1425] Post merge fixup * Preliminary work for machine serdez to make the launcher work correctly * Make the C++ control code compatible with the new type system See merge request legate/legate.core.internal!17 --- legate/core/_lib/types.pyx | 1 + legate/core/machine.py | 4 +- pyproject.toml | 2 +- src/core/cuda/stream_pool.cu | 1 + src/core/data/logical_store.cc | 2 +- src/core/data/logical_store.h | 2 +- src/core/data/logical_store_detail.cc | 45 +++--- src/core/data/logical_store_detail.h | 12 +- src/core/data/scalar.inl | 2 +- src/core/data/store.cc | 15 +- src/core/data/store.h | 8 +- src/core/mapping/machine.cc | 28 ++++ src/core/mapping/machine.h | 11 +- src/core/runtime/launcher.cc | 32 ++++- src/core/runtime/launcher.h | 2 + src/core/runtime/launcher_arg.cc | 2 +- src/core/runtime/runtime.cc | 77 ++++++---- src/core/runtime/runtime.h | 23 ++- src/core/type/type_info.cc | 100 +++++++++++++ src/core/type/type_info.h | 166 ++++++++++++++++++++++ src/core/utilities/deserializer.inl | 8 +- tests/cpp/integration/manual_simple.cc | 4 +- tests/cpp/integration/multi_scalar_out.cc | 18 +-- tests/cpp/unit/store.cc | 14 +- 24 files changed, 461 insertions(+), 118 deletions(-) diff --git a/legate/core/_lib/types.pyx b/legate/core/_lib/types.pyx index d897ae22ce..2e321afa04 100644 --- a/legate/core/_lib/types.pyx +++ b/legate/core/_lib/types.pyx @@ -275,3 +275,4 @@ cdef class StructDtype(Dtype): buf.pack_32bit_uint(num_fields) for field_idx in range(num_fields): self.field_type(field_idx).serialize(buf) + buf.pack_bool(self.aligned) diff --git a/legate/core/machine.py b/legate/core/machine.py index f7d05c93b0..3f61930dd6 100644 --- a/legate/core/machine.py +++ b/legate/core/machine.py @@ -281,10 +281,10 @@ def __repr__(self) -> str: return f"Machine({desc})" def pack(self, buf: BufferBuilder) -> None: - buf.pack_32bit_uint(self._preferred_kind) + buf.pack_32bit_int(self._preferred_kind) buf.pack_32bit_uint(len(self._proc_ranges)) for kind, proc_range in self._proc_ranges.items(): - buf.pack_32bit_uint(kind) + buf.pack_32bit_int(kind) proc_range.pack(buf) def __enter__(self) -> None: diff --git a/pyproject.toml b/pyproject.toml index c9f0e8cb3b..7157e1e73c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,7 +89,7 @@ strict_equality = true warn_unused_configs = true -exclude = ['tests/examples', 'tests/integration'] +exclude = ['tests/examples', 'tests/integration', 'tests/cpp'] [[tool.mypy.overrides]] # ignore certain auto-generated and utility files diff --git a/src/core/cuda/stream_pool.cu b/src/core/cuda/stream_pool.cu index af63e0dadc..88457c4f91 100644 --- a/src/core/cuda/stream_pool.cu +++ b/src/core/cuda/stream_pool.cu @@ -16,6 +16,7 @@ #include "core/cuda/cuda_help.h" #include "core/cuda/stream_pool.h" +#include "core/mapping/machine.h" #include "core/runtime/runtime.h" namespace legate { diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index 7d4e8b3982..0b7400aaa7 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -50,7 +50,7 @@ LogicalStore::LogicalStore(std::shared_ptr&& impl) int32_t LogicalStore::dim() const { return impl_->dim(); } -Type::Code LogicalStore::code() const { return impl_->code(); } +const Type& LogicalStore::type() const { return impl_->type(); } const Shape& LogicalStore::extents() const { return impl_->extents(); } diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index 07ed41d617..97dd0192ac 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -59,7 +59,7 @@ class LogicalStore { public: int32_t dim() const; - Type::Code code() const; + const Type& type() const; const Shape& extents() const; size_t volume() const; bool unbound() const; diff --git a/src/core/data/logical_store_detail.cc b/src/core/data/logical_store_detail.cc index daa7ddd04f..d02002360d 100644 --- a/src/core/data/logical_store_detail.cc +++ b/src/core/data/logical_store_detail.cc @@ -32,18 +32,21 @@ namespace detail { // legate::detail::Storage //////////////////////////////////////////////////// -Storage::Storage(int32_t dim, Type::Code code) : unbound_(true), dim_(dim), code_(code) {} +Storage::Storage(int32_t dim, std::unique_ptr type) + : unbound_(true), dim_(dim), type_(std::move(type)) +{ +} -Storage::Storage(Shape extents, Type::Code code, bool optimize_scalar) - : dim_(extents.size()), extents_(extents), code_(code), volume_(extents.volume()) +Storage::Storage(Shape extents, std::unique_ptr type, bool optimize_scalar) + : dim_(extents.size()), extents_(extents), type_(std::move(type)), volume_(extents.volume()) { if (optimize_scalar && volume_ == 1) kind_ = Kind::FUTURE; } -Storage::Storage(Shape extents, Type::Code code, const Legion::Future& future) +Storage::Storage(Shape extents, std::unique_ptr type, const Legion::Future& future) : dim_(extents.size()), extents_(extents), - code_(code), + type_(std::move(type)), kind_(Kind::FUTURE), future_(future), volume_(extents.volume()) @@ -58,7 +61,7 @@ LogicalRegionField* Storage::get_region_field() assert(Kind::REGION_FIELD == kind_); #endif if (nullptr == region_field_) - region_field_ = Runtime::get_runtime()->create_region_field(extents_, code_); + region_field_ = Runtime::get_runtime()->create_region_field(extents_, type_->size()); return region_field_.get(); } @@ -185,19 +188,7 @@ const Shape& LogicalStore::extents() const { return extents_; } size_t LogicalStore::volume() const { return extents_.volume(); } -struct elem_size_fn { - template - size_t operator()() - { - return sizeof(legate_type_of); - } -}; - -size_t LogicalStore::storage_size() const -{ - auto elem_size = type_dispatch(code(), elem_size_fn{}); - return storage_->volume() * elem_size; -} +size_t LogicalStore::storage_size() const { return storage_->volume() * type().size(); } int32_t LogicalStore::dim() const { @@ -206,7 +197,7 @@ int32_t LogicalStore::dim() const bool LogicalStore::has_scalar_storage() const { return storage_->kind() == Storage::Kind::FUTURE; } -Type::Code LogicalStore::code() const { return storage_->code(); } +const Type& LogicalStore::type() const { return storage_->type(); } bool LogicalStore::transformed() const { return !transform_->identity(); } @@ -301,22 +292,18 @@ std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) if (nullptr != mapped_) return mapped_; if (storage_->kind() == Storage::Kind::FUTURE) { // TODO: future wrappers from inline mappings are read-only for now - auto field_size = type_dispatch(code(), elem_size_fn{}); - auto domain = to_domain(storage_->extents()); - FutureWrapper future(true, field_size, domain, storage_->get_future()); + auto domain = to_domain(storage_->extents()); + FutureWrapper future(true, type().size(), domain, storage_->get_future()); // Physical stores for future-backed stores shouldn't be cached, as they are not automatically // remapped to reflect changes by the runtime. - // FIXME: Need to catch up the type system change - return nullptr; // std::make_shared(dim(), code(), -1, future, transform_); + return std::make_shared(dim(), type().clone(), -1, future, transform_); } #ifdef DEBUG_LEGATE assert(storage_->kind() == Storage::Kind::REGION_FIELD); #endif auto region_field = storage_->map(context); - // FIXME: Need to catch up the type system change - mapped_ = - nullptr; // std::make_shared(dim(), code(), -1, std::move(region_field), transform_); + mapped_ = std::make_shared(dim(), type().clone(), -1, std::move(region_field), transform_); return mapped_; } @@ -405,7 +392,7 @@ void LogicalStore::pack(BufferBuilder& buffer) const buffer.pack(has_scalar_storage()); buffer.pack(unbound()); buffer.pack(dim()); - buffer.pack(static_cast(code())); + type().pack(buffer); transform_->pack(buffer); } diff --git a/src/core/data/logical_store_detail.h b/src/core/data/logical_store_detail.h index ed69ea270e..ec1de634fa 100644 --- a/src/core/data/logical_store_detail.h +++ b/src/core/data/logical_store_detail.h @@ -37,18 +37,18 @@ class Storage : public std::enable_shared_from_this { public: // Create a RegionField-backed storage whose size is unbound. Initialized lazily. - Storage(int32_t dim, Type::Code code); + Storage(int32_t dim, std::unique_ptr type); // Create a RegionField-backed or a Future-backedstorage. Initialized lazily. - Storage(Shape extents, Type::Code code, bool optimize_scalar); + Storage(Shape extents, std::unique_ptr type, bool optimize_scalar); // Create a Future-backed storage. Initialized eagerly. - Storage(Shape extents, Type::Code code, const Legion::Future& future); + Storage(Shape extents, std::unique_ptr type, const Legion::Future& future); public: bool unbound() const { return unbound_; } const Shape& extents() const { return extents_; } size_t volume() const { return volume_; } int32_t dim(); - Type::Code code() const { return code_; } + const Type& type() const { return *type_; } Kind kind() const { return kind_; } public: @@ -74,7 +74,7 @@ class Storage : public std::enable_shared_from_this { int32_t dim_{-1}; Shape extents_; size_t volume_; - Type::Code code_{Type::Code::INVALID}; + std::unique_ptr type_{nullptr}; Kind kind_{Kind::REGION_FIELD}; std::shared_ptr region_field_{nullptr}; Legion::Future future_{}; @@ -124,7 +124,7 @@ class LogicalStore : public std::enable_shared_from_this { size_t storage_size() const; int32_t dim() const; bool has_scalar_storage() const; - Type::Code code() const; + const Type& type() const; bool transformed() const; public: diff --git a/src/core/data/scalar.inl b/src/core/data/scalar.inl index 4e9d6df7b0..7d61d93063 100644 --- a/src/core/data/scalar.inl +++ b/src/core/data/scalar.inl @@ -26,7 +26,7 @@ Scalar::Scalar(T value) : own_(true), type_(primitive_type(legate_type_code_of Scalar::Scalar(const std::vector& values) - : own_(true), type_(fixed_array_type(legate_type_code_of, values.size())) + : own_(true), type_(fixed_array_type(primitive_type(legate_type_code_of), values.size())) { auto size = type_->size(); auto buffer = malloc(size); diff --git a/src/core/data/store.cc b/src/core/data/store.cc index 01850e445d..e8bc01e0aa 100644 --- a/src/core/data/store.cc +++ b/src/core/data/store.cc @@ -131,15 +131,12 @@ void UnboundRegionField::update_num_elements(size_t num_elements) } FutureWrapper::FutureWrapper(bool read_only, - int32_t field_size, + uint32_t field_size, Domain domain, Legion::Future future, bool initialize /*= false*/) : read_only_(read_only), field_size_(field_size), domain_(domain), future_(future) { -#ifdef DEBUG_LEGATE - assert(field_size > 0); -#endif if (!read_only) { #ifdef DEBUG_LEGATE assert(!initialize || future_.get_untyped_size() == field_size); @@ -260,15 +257,14 @@ Store::Store(int32_t dim, } Store::Store(int32_t dim, - int32_t code, + std::unique_ptr type, int32_t redop_id, FutureWrapper future, const std::shared_ptr& transform) : is_future_(true), is_unbound_store_(false), dim_(dim), - // FIXME: Need to catch up the type system change - // code_(code), + type_(std::move(type)), redop_id_(redop_id), future_(future), transform_(transform), @@ -277,15 +273,14 @@ Store::Store(int32_t dim, } Store::Store(int32_t dim, - int32_t code, + std::unique_ptr type, int32_t redop_id, RegionField&& region_field, const std::shared_ptr& transform) : is_future_(false), is_unbound_store_(false), dim_(dim), - // FIXME: Need to catch up the type system change - // code_(code), + type_(std::move(type)), redop_id_(redop_id), region_field_(std::forward(region_field)), transform_(transform) diff --git a/src/core/data/store.h b/src/core/data/store.h index 8129a2b9f8..a59fe8717e 100644 --- a/src/core/data/store.h +++ b/src/core/data/store.h @@ -212,7 +212,7 @@ class FutureWrapper { public: FutureWrapper() {} FutureWrapper(bool read_only, - int32_t field_size, + uint32_t field_size, Domain domain, Legion::Future future, bool initialize = false); @@ -261,7 +261,7 @@ class FutureWrapper { private: bool read_only_{true}; - size_t field_size_{0}; + uint32_t field_size_{0}; Domain domain_{}; Legion::Future future_{}; Legion::UntypedDeferredValue buffer_{}; @@ -289,12 +289,12 @@ class Store { UnboundRegionField&& unbound_field, std::shared_ptr&& transform = nullptr); Store(int32_t dim, - int32_t code, + std::unique_ptr type, int32_t redop_id, FutureWrapper future, const std::shared_ptr& transform); Store(int32_t dim, - int32_t code, + std::unique_ptr type, int32_t redop_id, RegionField&& region_field, const std::shared_ptr& transform); diff --git a/src/core/mapping/machine.cc b/src/core/mapping/machine.cc index 1d86586926..4c47b3c85a 100644 --- a/src/core/mapping/machine.cc +++ b/src/core/mapping/machine.cc @@ -15,6 +15,7 @@ */ #include "core/mapping/machine.h" +#include "core/utilities/buffer_builder.h" namespace legate { namespace mapping { @@ -90,6 +91,33 @@ std::string ProcessorRange::to_string() const return ss.str(); } +void ProcessorRange::pack(BufferBuilder& buffer) const +{ + buffer.pack(low); + buffer.pack(high); + buffer.pack(per_node_count); +} + +MachineDesc::MachineDesc(const std::map& ranges) + : processor_ranges(ranges) +{ + for (auto& [target, processor_range] : processor_ranges) + if (!processor_range.empty()) { + preferred_target = target; + break; + } +} + +void MachineDesc::pack(BufferBuilder& buffer) const +{ + buffer.pack(static_cast(preferred_target)); + buffer.pack(processor_ranges.size()); + for (auto& [target, processor_range] : processor_ranges) { + buffer.pack(static_cast(target)); + processor_range.pack(buffer); + } +} + ProcessorRange MachineDesc::processor_range() const { auto finder = processor_ranges.find(preferred_target); diff --git a/src/core/mapping/machine.h b/src/core/mapping/machine.h index 6fde283057..34580b3657 100644 --- a/src/core/mapping/machine.h +++ b/src/core/mapping/machine.h @@ -38,19 +38,26 @@ struct ProcessorRange { bool empty() const; std::string to_string() const; + void pack(BufferBuilder& buffer) const; + uint32_t low{0}; uint32_t high{0}; uint32_t per_node_count{1}; }; struct MachineDesc { - TaskTarget preferred_target; - std::map processor_ranges; + MachineDesc() {} + MachineDesc(const std::map& processor_ranges); + + TaskTarget preferred_target{TaskTarget::CPU}; + std::map processor_ranges{}; ProcessorRange processor_range() const; std::vector valid_targets() const; std::vector valid_targets_except(std::set&& to_exclude) const; std::string to_string() const; + + void pack(BufferBuilder& buffer) const; }; std::ostream& operator<<(std::ostream& stream, const MachineDesc& info); diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/launcher.cc index d274361b9f..71df2b74b4 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -34,6 +34,7 @@ TaskLauncher::TaskLauncher(LibraryContext* library, int64_t task_id, int64_t tag req_analyzer_ = new RequirementAnalyzer(); out_analyzer_ = new OutputRequirementAnalyzer(); buffer_ = new BufferBuilder(); + mapper_arg_ = new BufferBuilder(); } TaskLauncher::~TaskLauncher() @@ -45,6 +46,7 @@ TaskLauncher::~TaskLauncher() delete req_analyzer_; delete buffer_; + delete mapper_arg_; } int64_t TaskLauncher::legion_task_id() const { return library_->get_task_id(task_id_); } @@ -157,6 +159,13 @@ void TaskLauncher::pack_args(const std::vector& args) for (auto& arg : args) arg->pack(*buffer_); } +void TaskLauncher::pack_mapper_arg() +{ + Runtime::get_runtime()->get_machine().pack(*mapper_arg_); + // TODO: Generate the right sharding functor id + mapper_arg_->pack(0); +} + std::unique_ptr TaskLauncher::build_single_task() { // Coalesce region requirements before packing task arguments @@ -168,15 +177,21 @@ std::unique_ptr TaskLauncher::build_single_task() pack_args(outputs_); pack_args(reductions_); pack_args(scalars_); + // can_raise_exception buffer_->pack(false); + // insert_barrier buffer_->pack(false); + // # communicators buffer_->pack(0); + pack_mapper_arg(); + auto single_task = std::make_unique(legion_task_id(), buffer_->to_legion_buffer(), Legion::Predicate::TRUE_PRED, legion_mapper_id(), - tag_); + tag_, + mapper_arg_->to_legion_buffer()); for (auto& future : futures_) single_task->add_future(future); req_analyzer_->populate_launcher(single_task.get()); @@ -197,10 +212,15 @@ std::unique_ptr TaskLauncher::build_index_task( pack_args(outputs_); pack_args(reductions_); pack_args(scalars_); + // can_raise_exception buffer_->pack(false); + // insert_barrier buffer_->pack(false); + // # communicators buffer_->pack(0); + pack_mapper_arg(); + auto index_task = std::make_unique(legion_task_id(), launch_domain, buffer_->to_legion_buffer(), @@ -208,7 +228,8 @@ std::unique_ptr TaskLauncher::build_index_task( Legion::Predicate::TRUE_PRED, false /*must*/, legion_mapper_id(), - tag_); + tag_, + mapper_arg_->to_legion_buffer()); for (auto& future : futures_) index_task->add_future(future); for (auto& future_map : future_maps_) index_task->point_futures.push_back(future_map); @@ -226,9 +247,10 @@ void TaskLauncher::bind_region_fields_to_unbound_stores() #ifdef DEBUG_LEGATE assert(arg->requirement_index() != -1U); #endif - auto* store = arg->store(); - auto& req = output_requirements_[arg->requirement_index()]; - auto region_field = runtime->import_region_field(req.parent, arg->field_id(), store->code()); + auto* store = arg->store(); + auto& req = output_requirements_[arg->requirement_index()]; + auto region_field = + runtime->import_region_field(req.parent, arg->field_id(), store->type().size()); store->set_region_field(std::move(region_field)); } } diff --git a/src/core/runtime/launcher.h b/src/core/runtime/launcher.h index 76ae5c5f6c..9f601e120d 100644 --- a/src/core/runtime/launcher.h +++ b/src/core/runtime/launcher.h @@ -84,6 +84,7 @@ class TaskLauncher { private: void pack_args(const std::vector& args); + void pack_mapper_arg(); std::unique_ptr build_index_task(const Legion::Domain& launch_domain); std::unique_ptr build_single_task(); void bind_region_fields_to_unbound_stores(); @@ -106,6 +107,7 @@ class TaskLauncher { RequirementAnalyzer* req_analyzer_; OutputRequirementAnalyzer* out_analyzer_; BufferBuilder* buffer_; + BufferBuilder* mapper_arg_; std::vector output_requirements_; }; diff --git a/src/core/runtime/launcher_arg.cc b/src/core/runtime/launcher_arg.cc index 8f45a748ce..fd71f7d5f8 100644 --- a/src/core/runtime/launcher_arg.cc +++ b/src/core/runtime/launcher_arg.cc @@ -88,7 +88,7 @@ void FutureStoreArg::pack(BufferBuilder& buffer) const buffer.pack(redop_); buffer.pack(read_only_); buffer.pack(has_storage_); - buffer.pack(static_cast(store_->code())); + buffer.pack(store_->type().size()); buffer.pack(store_->extents().data()); } diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index db654c1e16..9bd9656975 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -23,6 +23,7 @@ #include "core/data/logical_store_detail.h" #include "core/mapping/core_mapper.h" #include "core/mapping/default_mapper.h" +#include "core/mapping/machine.h" #include "core/partitioning/partition.h" #include "core/partitioning/partitioner.h" #include "core/runtime/context.h" @@ -314,8 +315,10 @@ void register_builtin_reduction_ops() register_legate_core_sharding_functors(legion_runtime, core_lib); - if (!Core::standalone) + if (!Core::standalone) { Core::retrieve_tunable(Legion::Runtime::get_context(), legion_runtime, core_lib); + Runtime::get_runtime()->initialize_toplevel_machine(); + } } /*static*/ void core_library_bootstrapping_callback(Legion::Machine machine, @@ -393,7 +396,7 @@ void RegionManager::import_region(const Legion::LogicalRegion& region) class FieldManager { public: - FieldManager(Runtime* runtime, const Domain& shape, Type::Code code); + FieldManager(Runtime* runtime, const Domain& shape, uint32_t field_size); public: std::shared_ptr allocate_field(); @@ -403,26 +406,15 @@ class FieldManager { private: Runtime* runtime_; Domain shape_; - Type::Code code_; - size_t field_size_; + uint32_t field_size_; private: using FreeField = std::pair; std::deque free_fields_; }; -struct field_size_fn { - template - size_t operator()() - { - return sizeof(legate_type_of); - } -}; - -static size_t get_field_size(Type::Code code) { return type_dispatch(code, field_size_fn{}); } - -FieldManager::FieldManager(Runtime* runtime, const Domain& shape, Type::Code code) - : runtime_(runtime), shape_(shape), code_(code), field_size_(get_field_size(code)) +FieldManager::FieldManager(Runtime* runtime, const Domain& shape, uint32_t field_size) + : runtime_(runtime), shape_(shape), field_size_(field_size) { } @@ -753,6 +745,7 @@ void Runtime::post_startup_initialization(Legion::Context legion_context) core_context_ = find_library(core_library_name); partition_manager_ = new PartitionManager(this, core_context_); Core::retrieve_tunable(legion_context_, legion_runtime_, core_context_); + initialize_toplevel_machine(); } // This function should be moved to the library context @@ -792,17 +785,17 @@ void Runtime::schedule(std::vector> operations) for (auto& op : operations) op->launch(strategy.get()); } -LogicalStore Runtime::create_store(Type::Code code, int32_t dim) +LogicalStore Runtime::create_store(std::unique_ptr type, int32_t dim) { - auto storage = std::make_shared(dim, code); + auto storage = std::make_shared(dim, std::move(type)); return LogicalStore(std::make_shared(std::move(storage))); } LogicalStore Runtime::create_store(std::vector extents, - Type::Code code, + std::unique_ptr type, bool optimize_scalar /*=false*/) { - auto storage = std::make_shared(extents, code, optimize_scalar); + auto storage = std::make_shared(extents, std::move(type), optimize_scalar); return LogicalStore(std::make_shared(std::move(storage))); } @@ -810,14 +803,14 @@ LogicalStore Runtime::create_store(const Scalar& scalar) { Shape extents{1}; auto future = create_future(scalar.ptr(), scalar.size()); - auto storage = std::make_shared(extents, scalar.type().code, future); + auto storage = std::make_shared(extents, scalar.type().clone(), future); return LogicalStore(std::make_shared(std::move(storage))); } uint64_t Runtime::get_unique_store_id() { return next_store_id_++; } std::shared_ptr Runtime::create_region_field(const Shape& extents, - Type::Code code) + uint32_t field_size) { DomainPoint lo, hi; hi.dim = lo.dim = static_cast(extents.size()); @@ -826,18 +819,18 @@ std::shared_ptr Runtime::create_region_field(const Shape& ex for (int32_t dim = 0; dim < lo.dim; ++dim) hi[dim] = extents[dim] - 1; Domain shape(lo, hi); - auto fld_mgr = runtime_->find_or_create_field_manager(shape, code); + auto fld_mgr = runtime_->find_or_create_field_manager(shape, field_size); return fld_mgr->allocate_field(); } std::shared_ptr Runtime::import_region_field(Legion::LogicalRegion region, Legion::FieldID field_id, - Type::Code code) + uint32_t field_size) { // TODO: This is a blocking operation. We should instead use index sapces as keys to field // managers auto shape = legion_runtime_->get_index_space_domain(legion_context_, region.get_index_space()); - auto fld_mgr = runtime_->find_or_create_field_manager(shape, code); + auto fld_mgr = runtime_->find_or_create_field_manager(shape, field_size); return fld_mgr->import_field(region, field_id); } @@ -881,14 +874,14 @@ RegionManager* Runtime::find_or_create_region_manager(const Domain& shape) } } -FieldManager* Runtime::find_or_create_field_manager(const Domain& shape, Type::Code code) +FieldManager* Runtime::find_or_create_field_manager(const Domain& shape, uint32_t field_size) { - auto key = std::make_pair(shape, code); + auto key = FieldManagerKey(shape, field_size); auto finder = field_managers_.find(key); if (finder != field_managers_.end()) return finder->second; else { - auto fld_mgr = new FieldManager(this, shape, code); + auto fld_mgr = new FieldManager(this, shape, field_size); field_managers_[key] = fld_mgr; return fld_mgr; } @@ -1012,6 +1005,34 @@ void Runtime::issue_execution_fence(bool block /*=false*/) if (block) future.wait(); } +void Runtime::initialize_toplevel_machine() +{ + auto mapper_id = core_context_->get_mapper_id(); + auto num_nodes = get_tunable(mapper_id, LEGATE_CORE_TUNABLE_NUM_NODES); + + auto num_gpus = get_tunable(mapper_id, LEGATE_CORE_TUNABLE_TOTAL_GPUS); + auto num_omps = get_tunable(mapper_id, LEGATE_CORE_TUNABLE_TOTAL_OMPS); + auto num_cpus = get_tunable(mapper_id, LEGATE_CORE_TUNABLE_TOTAL_CPUS); + + auto create_range = [&num_nodes](int32_t num_procs) { + auto per_node_count = num_procs / num_nodes; + return mapping::ProcessorRange(0, num_procs, per_node_count); + }; + + auto machine = new mapping::MachineDesc({{mapping::TaskTarget::GPU, create_range(num_gpus)}, + {mapping::TaskTarget::OMP, create_range(num_omps)}, + {mapping::TaskTarget::CPU, create_range(num_cpus)}}); + machine_.reset(machine); +} + +const mapping::MachineDesc& Runtime::get_machine() const +{ +#ifdef DEBUG_LEGATE + assert(machine_ != nullptr); +#endif + return *machine_; +} + Legion::ProjectionID Runtime::get_projection(int32_t src_ndim, const proj::SymbolicPoint& point) { #ifdef DEBUG_LEGATE diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 820c9b1361..ec294e099e 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -28,6 +28,7 @@ #include "core/legate_c.h" #include "core/runtime/resource.h" #include "core/task/exception.h" +#include "core/type/type_info.h" #include "core/utilities/typedefs.h" /** @defgroup runtime Runtime and library contexts @@ -39,6 +40,7 @@ class LibraryContext; namespace mapping { +class MachineDesc; class Mapper; } // namespace mapping @@ -166,24 +168,25 @@ class Runtime { void submit(std::unique_ptr op); public: - LogicalStore create_store(Type::Code code, int32_t dim = 1); + LogicalStore create_store(std::unique_ptr type, int32_t dim = 1); LogicalStore create_store(std::vector extents, - Type::Code code, + std::unique_ptr type, bool optimize_scalar = false); LogicalStore create_store(const Scalar& scalar); uint64_t get_unique_store_id(); public: - std::shared_ptr create_region_field(const Shape& extents, Type::Code code); + std::shared_ptr create_region_field(const Shape& extents, + uint32_t field_size); std::shared_ptr import_region_field(Legion::LogicalRegion region, Legion::FieldID field_id, - Type::Code code); + uint32_t field_size); RegionField map_region_field(LibraryContext* context, const LogicalRegionField* region_field); void unmap_physical_region(Legion::PhysicalRegion pr); public: RegionManager* find_or_create_region_manager(const Legion::Domain& shape); - FieldManager* find_or_create_field_manager(const Legion::Domain& shape, Type::Code code); + FieldManager* find_or_create_field_manager(const Legion::Domain& shape, uint32_t field_size); PartitionManager* partition_manager() const; public: @@ -218,6 +221,10 @@ class Runtime { public: void issue_execution_fence(bool block = false); + public: + void initialize_toplevel_machine(); + const mapping::MachineDesc& get_machine() const; + public: Legion::ProjectionID get_projection(int32_t src_ndim, const proj::SymbolicPoint& point); Legion::ProjectionID get_delinearizing_projection(); @@ -242,7 +249,8 @@ class Runtime { LibraryContext* core_context_{nullptr}; private: - std::map, FieldManager*> field_managers_; + using FieldManagerKey = std::pair; + std::map field_managers_; std::map region_managers_; PartitionManager* partition_manager_{nullptr}; @@ -273,6 +281,9 @@ class Runtime { private: uint32_t next_type_uid_; std::map, int32_t> reduction_ops_{}; + + private: + std::unique_ptr machine_{nullptr}; }; void initialize(int32_t argc, char** argv); diff --git a/src/core/type/type_info.cc b/src/core/type/type_info.cc index f60237722e..8a7cf11da2 100644 --- a/src/core/type/type_info.cc +++ b/src/core/type/type_info.cc @@ -20,6 +20,7 @@ #include "core/runtime/runtime.h" #include "core/type/type_info.h" #include "core/type/type_traits.h" +#include "core/utilities/buffer_builder.h" namespace legate { @@ -75,6 +76,13 @@ int32_t Type::find_reduction_operator(int32_t op_kind) const return Runtime::get_runtime()->find_reduction_operator(uid(), op_kind); } +int32_t Type::find_reduction_operator(ReductionOpKind op_kind) const +{ + return find_reduction_operator(static_cast(op_kind)); +} + +bool Type::operator==(const Type& other) const { return equal(other); } + PrimitiveType::PrimitiveType(Code code) : Type(code), size_(SIZEOF.at(code)) {} int32_t PrimitiveType::uid() const { return static_cast(code); } @@ -83,6 +91,13 @@ std::unique_ptr PrimitiveType::clone() const { return std::make_unique(static_cast(code)); +} + +bool PrimitiveType::equal(const Type& other) const { return code == other.code; } + ExtensionType::ExtensionType(int32_t uid, Type::Code code) : Type(code), uid_(uid) {} FixedArrayType::FixedArrayType(int32_t uid, @@ -108,6 +123,28 @@ std::string FixedArrayType::to_string() const return std::move(ss).str(); } +void FixedArrayType::pack(BufferBuilder& buffer) const +{ + buffer.pack(static_cast(code)); + buffer.pack(uid_); + buffer.pack(N_); + element_type_->pack(buffer); +} + +bool FixedArrayType::equal(const Type& other) const +{ + if (code != other.code) return false; + auto& casted = static_cast(other); + +#ifdef DEBUG_LEGATE + // Do a structural check in debug mode + return uid_ == casted.uid_ && N_ == casted.N_ && element_type_ == casted.element_type_; +#else + // Each type is uniquely identified by the uid, so it's sufficient to compare between uids + return uid_ == casted.uid_; +#endif +} + StructType::StructType(int32_t uid, std::vector>&& field_types, bool align) noexcept(false) @@ -161,6 +198,34 @@ std::string StructType::to_string() const return std::move(ss).str(); } +void StructType::pack(BufferBuilder& buffer) const +{ + buffer.pack(static_cast(code)); + buffer.pack(uid_); + buffer.pack(field_types_.size()); + for (auto& field_type : field_types_) field_type->pack(buffer); + buffer.pack(aligned_); +} + +bool StructType::equal(const Type& other) const +{ + if (code != other.code) return false; + auto& casted = static_cast(other); + +#ifdef DEBUG_LEGATE + // Do a structural check in debug mode + if (uid_ != casted.uid_) return false; + uint32_t nf = num_fields(); + if (nf != casted.num_fields()) return false; + for (uint32_t idx = 0; idx < nf; ++idx) + if (field_type(idx) != casted.field_type(idx)) return false; + return true; +#else + // Each type is uniquely identified by the uid, so it's sufficient to compare between uids + return uid_ == casted.uid_; +#endif +} + const Type& StructType::field_type(uint32_t field_idx) const { return *field_types_.at(field_idx); } StringType::StringType() : Type(Type::Code::STRING) {} @@ -171,6 +236,13 @@ std::unique_ptr StringType::clone() const { return string_type(); } std::string StringType::to_string() const { return "string"; } +void StringType::pack(BufferBuilder& buffer) const +{ + buffer.pack(static_cast(code)); +} + +bool StringType::equal(const Type& other) const { return code == other.code; } + std::unique_ptr primitive_type(Type::Code code) { return std::make_unique(code); @@ -205,4 +277,32 @@ std::ostream& operator<<(std::ostream& ostream, const Type& type) return ostream; } +std::unique_ptr bool_() { return primitive_type(Type::Code::BOOL); } + +std::unique_ptr int8() { return primitive_type(Type::Code::INT8); } + +std::unique_ptr int16() { return primitive_type(Type::Code::INT16); } + +std::unique_ptr int32() { return primitive_type(Type::Code::INT32); } + +std::unique_ptr int64() { return primitive_type(Type::Code::INT64); } + +std::unique_ptr uint8() { return primitive_type(Type::Code::UINT8); } + +std::unique_ptr uint16() { return primitive_type(Type::Code::UINT16); } + +std::unique_ptr uint32() { return primitive_type(Type::Code::UINT32); } + +std::unique_ptr uint64() { return primitive_type(Type::Code::UINT64); } + +std::unique_ptr float16() { return primitive_type(Type::Code::FLOAT16); } + +std::unique_ptr float32() { return primitive_type(Type::Code::FLOAT32); } + +std::unique_ptr float64() { return primitive_type(Type::Code::FLOAT64); } + +std::unique_ptr complex64() { return primitive_type(Type::Code::COMPLEX64); } + +std::unique_ptr complex128() { return primitive_type(Type::Code::COMPLEX128); } + } // namespace legate diff --git a/src/core/type/type_info.h b/src/core/type/type_info.h index a91fef8428..09d393a89f 100644 --- a/src/core/type/type_info.h +++ b/src/core/type/type_info.h @@ -26,6 +26,8 @@ namespace legate { +class BufferBuilder; + /** * @ingroup types * @brief Enum for reduction operator kinds @@ -76,6 +78,8 @@ class Type { protected: Type(Code code); + virtual bool equal(const Type& other) const = 0; + public: virtual ~Type() {} @@ -122,6 +126,13 @@ class Type { */ virtual std::string to_string() const = 0; + /** + * @brief Serializes the type into a buffer + * + * @param buffer A BufferBuilder object to serialize the type into + */ + virtual void pack(BufferBuilder& buffer) const = 0; + /** * @brief Records a reduction operator. * @@ -144,6 +155,31 @@ class Type { */ int32_t find_reduction_operator(int32_t op_kind) const; + /** + * @brief Finds the global operator ID for a given reduction operator kind. + * + * Raises an exception if no reduction operator has been registered for the kind. + * + * @param op_kind Reduction operator kind + * + * @return Global reduction operator ID + */ + int32_t find_reduction_operator(ReductionOpKind op_kind) const; + + /** + * @brief Equality check between types + * + * Note that type checks are name-based; two isomorphic fixed-size array types are considered + * different if their uids are different (the same applies to struct types). + * + * @param other Type to compare + * + * @return true Types are equal + * @return false Types are different + */ + bool operator==(const Type& other) const; + bool operator!=(const Type& other) const { return !operator==(other); } + const Code code; }; @@ -165,6 +201,10 @@ class PrimitiveType : public Type { bool variable_size() const override { return false; } std::unique_ptr clone() const override; std::string to_string() const override; + void pack(BufferBuilder& buffer) const override; + + private: + bool equal(const Type& other) const override; private: const uint32_t size_; @@ -183,6 +223,10 @@ class StringType : public Type { int32_t uid() const override; std::unique_ptr clone() const override; std::string to_string() const override; + void pack(BufferBuilder& buffer) const override; + + private: + bool equal(const Type& other) const override; }; /** @@ -217,6 +261,8 @@ class FixedArrayType : public ExtensionType { bool variable_size() const override { return false; } std::unique_ptr clone() const override; std::string to_string() const override; + void pack(BufferBuilder& buffer) const override; + /** * @brief Returns the number of elements * @@ -230,6 +276,9 @@ class FixedArrayType : public ExtensionType { */ const Type& element_type() const { return *element_type_; } + private: + bool equal(const Type& other) const override; + private: const std::unique_ptr element_type_; const uint32_t N_; @@ -258,6 +307,8 @@ class StructType : public ExtensionType { bool variable_size() const override { return false; } std::unique_ptr clone() const override; std::string to_string() const override; + void pack(BufferBuilder& buffer) const override; + /** * @brief Returns the number of fields * @@ -280,6 +331,9 @@ class StructType : public ExtensionType { */ bool aligned() const { return aligned_; } + private: + bool equal(const Type& other) const override; + private: bool aligned_; uint32_t size_; @@ -335,4 +389,116 @@ std::ostream& operator<<(std::ostream&, const Type::Code&); std::ostream& operator<<(std::ostream&, const Type&); +/** + * @ingroup types + * @brief Creates a boolean type + * + * @return Type object + */ +std::unique_ptr bool_(); + +/** + * @ingroup types + * @brief Creates a 8-bit signed integer type + * + * @return Type object + */ +std::unique_ptr int8(); + +/** + * @ingroup types + * @brief Creates a 16-bit signed integer type + * + * @return Type object + */ +std::unique_ptr int16(); + +/** + * @ingroup types + * @brief Creates a 32-bit signed integer type + * + * @return Type object + */ +std::unique_ptr int32(); + +/** + * @ingroup types + * @brief Creates a 64-bit signed integer type + * + * @return Type object + */ +std::unique_ptr int64(); + +/** + * @ingroup types + * @brief Creates a 8-bit unsigned integer type + * + * @return Type object + */ +std::unique_ptr uint8(); + +/** + * @ingroup types + * @brief Creates a 16-bit unsigned integer type + * + * @return Type object + */ +std::unique_ptr uint16(); + +/** + * @ingroup types + * @brief Creates a 32-bit unsigned integer type + * + * @return Type object + */ +std::unique_ptr uint32(); + +/** + * @ingroup types + * @brief Creates a 64-bit unsigned integer type + * + * @return Type object + */ +std::unique_ptr uint64(); + +/** + * @ingroup types + * @brief Creates a half-precision floating point type + * + * @return Type object + */ +std::unique_ptr float16(); + +/** + * @ingroup types + * @brief Creates a single-precision floating point type + * + * @return Type object + */ +std::unique_ptr float32(); + +/** + * @ingroup types + * @brief Creates a double-precision floating point type + * + * @return Type object + */ +std::unique_ptr float64(); + +/** + * @ingroup types + * @brief Creates a single-precision complex number type + * + * @return Type object + */ +std::unique_ptr complex64(); + +/** + * @ingroup types + * @brief Creates a double-precision complex number type + * + * @return Type object + */ +std::unique_ptr complex128(); + } // namespace legate diff --git a/src/core/utilities/deserializer.inl b/src/core/utilities/deserializer.inl index 1f31f7841e..25563fa6b7 100644 --- a/src/core/utilities/deserializer.inl +++ b/src/core/utilities/deserializer.inl @@ -109,20 +109,22 @@ std::unique_ptr BaseDeserializer::unpack_type() auto code = static_cast(unpack()); switch (code) { case Type::Code::FIXED_ARRAY: { - auto uid = unpack(); + auto uid = unpack(); auto N = unpack(); auto type = unpack_type(); return std::make_unique(uid, std::move(type), N); } case Type::Code::STRUCT: { - auto uid = unpack(); + auto uid = unpack(); auto num_fields = unpack(); std::vector> field_types; field_types.reserve(num_fields); for (uint32_t idx = 0; idx < num_fields; ++idx) field_types.emplace_back(unpack_type()); - return std::make_unique(uid, std::move(field_types)); + auto align = unpack(); + + return std::make_unique(uid, std::move(field_types), align); } case Type::Code::BOOL: case Type::Code::INT8: diff --git a/tests/cpp/integration/manual_simple.cc b/tests/cpp/integration/manual_simple.cc index 54bf6eadb0..9850a0a786 100644 --- a/tests/cpp/integration/manual_simple.cc +++ b/tests/cpp/integration/manual_simple.cc @@ -37,7 +37,7 @@ void register_tasks() { auto runtime = legate::Runtime::get_runtime(); auto context = runtime->create_library(library_name); - HelloTask::register_variants(*context); + HelloTask::register_variants(context); } /*static*/ void HelloTask::cpu_variant(legate::TaskContext& context) @@ -88,7 +88,7 @@ void legate_main(int32_t argc, char** argv) auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(library_name); - auto store = runtime->create_store({5, 5}, legate::LegateTypeCode::INT64_LT); + auto store = runtime->create_store({5, 5}, legate::int64()); test_auto_task(context, store); print_store(context, store); test_manual_task(context, store); diff --git a/tests/cpp/integration/multi_scalar_out.cc b/tests/cpp/integration/multi_scalar_out.cc index fdc4df8a32..5c229068f4 100644 --- a/tests/cpp/integration/multi_scalar_out.cc +++ b/tests/cpp/integration/multi_scalar_out.cc @@ -43,8 +43,8 @@ void register_tasks() { auto runtime = legate::Runtime::get_runtime(); auto context = runtime->create_library(library_name); - WriterTask::register_variants(*context); - ReducerTask::register_variants(*context); + WriterTask::register_variants(context); + ReducerTask::register_variants(context); } /*static*/ void WriterTask::cpu_variant(legate::TaskContext& context) @@ -94,8 +94,8 @@ void test_reducer_auto(legate::LibraryContext* context, auto part1 = task->declare_partition(); auto part2 = task->declare_partition(); auto part3 = task->declare_partition(); - auto redop1 = LEGION_REDOP_BASE + LEGION_TYPE_TOTAL * LEGION_REDOP_KIND_SUM + scalar1.code(); - auto redop2 = LEGION_REDOP_BASE + LEGION_TYPE_TOTAL * LEGION_REDOP_KIND_PROD + scalar2.code(); + auto redop1 = scalar1.type().find_reduction_operator(legate::ReductionOpKind::ADD); + auto redop2 = scalar1.type().find_reduction_operator(legate::ReductionOpKind::MUL); task->add_reduction(scalar1, redop1, part1); task->add_reduction(scalar2, redop2, part2); task->add_output(store, part3); @@ -108,8 +108,8 @@ void test_reducer_manual(legate::LibraryContext* context, { auto runtime = legate::Runtime::get_runtime(); auto task = runtime->create_task(context, REDUCER, legate::Shape({2})); - auto redop1 = LEGION_REDOP_BASE + LEGION_TYPE_TOTAL * LEGION_REDOP_KIND_SUM + scalar1.code(); - auto redop2 = LEGION_REDOP_BASE + LEGION_TYPE_TOTAL * LEGION_REDOP_KIND_PROD + scalar2.code(); + auto redop1 = scalar1.type().find_reduction_operator(legate::ReductionOpKind::ADD); + auto redop2 = scalar2.type().find_reduction_operator(legate::ReductionOpKind::MUL); task->add_reduction(scalar1, redop1); task->add_reduction(scalar2, redop2); runtime->submit(std::move(task)); @@ -136,9 +136,9 @@ void legate_main(int32_t argc, char** argv) auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(library_name); - auto scalar1 = runtime->create_store({1, 1}, legate::LegateTypeCode::INT8_LT, true); - auto scalar2 = runtime->create_store({1, 1, 1}, legate::LegateTypeCode::INT32_LT, true); - auto store = runtime->create_store({10}, legate::LegateTypeCode::INT64_LT); + auto scalar1 = runtime->create_store({1, 1}, legate::int8(), true); + auto scalar2 = runtime->create_store({1, 1, 1}, legate::int32(), true); + auto store = runtime->create_store({10}, legate::int64()); test_writer_auto(context, scalar1, scalar2); print_stores(context, scalar1, scalar2); test_reducer_auto(context, scalar1, scalar2, store); diff --git a/tests/cpp/unit/store.cc b/tests/cpp/unit/store.cc index e718290e07..234c970a14 100644 --- a/tests/cpp/unit/store.cc +++ b/tests/cpp/unit/store.cc @@ -13,21 +13,21 @@ void store_creation(int32_t argc, char** argv) // Bound { auto runtime = legate::Runtime::get_runtime(); - auto store = runtime->create_store({4, 4}, legate::LegateTypeCode::INT64_LT); + auto store = runtime->create_store({4, 4}, legate::int64()); EXPECT_FALSE(store.unbound()); EXPECT_EQ(store.dim(), 2); EXPECT_EQ(store.extents(), (std::vector{4, 4})); - EXPECT_EQ(store.code(), legate::LegateTypeCode::INT64_LT); + EXPECT_EQ(store.type(), *legate::int64()); EXPECT_FALSE(store.transformed()); } // Unbound { auto runtime = legate::Runtime::get_runtime(); - auto store = runtime->create_store(legate::LegateTypeCode::INT64_LT); + auto store = runtime->create_store(legate::int64()); EXPECT_TRUE(store.unbound()); EXPECT_EQ(store.dim(), 1); - EXPECT_EQ(store.code(), legate::LegateTypeCode::INT64_LT); + EXPECT_EQ(store.type(), *legate::int64()); EXPECT_FALSE(store.transformed()); EXPECT_THROW(store.extents(), std::invalid_argument); } @@ -37,7 +37,7 @@ void store_valid_transform(int32_t argc, char** argv) { // Bound auto runtime = legate::Runtime::get_runtime(); - auto store = runtime->create_store({4, 3}, legate::LegateTypeCode::INT64_LT); + auto store = runtime->create_store({4, 3}, legate::int64()); auto promoted = store.promote(0, 5); EXPECT_EQ(promoted.extents(), (std::vector{5, 4, 3})); @@ -68,7 +68,7 @@ void store_invalid_transform(int32_t argc, char** argv) // Bound { auto runtime = legate::Runtime::get_runtime(); - auto store = runtime->create_store({4, 3}, legate::LegateTypeCode::INT64_LT); + auto store = runtime->create_store({4, 3}, legate::int64()); EXPECT_THROW(store.promote(3, 5), std::invalid_argument); EXPECT_THROW(store.promote(-3, 5), std::invalid_argument); @@ -95,7 +95,7 @@ void store_invalid_transform(int32_t argc, char** argv) // Unbound { auto runtime = legate::Runtime::get_runtime(); - auto store = runtime->create_store(legate::LegateTypeCode::INT64_LT); + auto store = runtime->create_store(legate::int64()); EXPECT_THROW(store.promote(1, 1), std::invalid_argument); } } From 2810d3fff64134abb57295d71653f9d93a109303 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Sun, 7 May 2023 22:37:22 -0700 Subject: [PATCH 0111/1425] Initialize the runtime correctly in non-standalone mode * Initialize the runtime correctly in non-standalone mode See merge request legate/legate.core.internal!18 --- src/core/runtime/runtime.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 9bd9656975..3e67705ac8 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -315,10 +315,8 @@ void register_builtin_reduction_ops() register_legate_core_sharding_functors(legion_runtime, core_lib); - if (!Core::standalone) { - Core::retrieve_tunable(Legion::Runtime::get_context(), legion_runtime, core_lib); - Runtime::get_runtime()->initialize_toplevel_machine(); - } + if (!Core::standalone) + Runtime::get_runtime()->post_startup_initialization(Legion::Runtime::get_context()); } /*static*/ void core_library_bootstrapping_callback(Legion::Machine machine, From c4f316800d6ccf45f933ac3b514d2c70030f4c11 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Sun, 7 May 2023 23:23:05 -0700 Subject: [PATCH 0112/1425] Pack method for Scalar and use the right field sizes for unbound stores * Pack method for Scalar and use the right field sizes for unbound stores See merge request legate/legate.core.internal!19 --- src/core/data/scalar.cc | 8 +++++++- src/core/data/scalar.h | 10 ++++++++++ src/core/runtime/launcher_arg.cc | 8 +------- src/core/runtime/operation.cc | 3 +-- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/core/data/scalar.cc b/src/core/data/scalar.cc index a0246d00f4..640f2c80c3 100644 --- a/src/core/data/scalar.cc +++ b/src/core/data/scalar.cc @@ -15,7 +15,7 @@ */ #include "core/data/scalar.h" -#include "core/utilities/dispatch.h" +#include "core/utilities/buffer_builder.h" namespace legate { @@ -59,4 +59,10 @@ size_t Scalar::size() const return type_->size(); } +void Scalar::pack(BufferBuilder& buffer) const +{ + type_->pack(buffer); + buffer.pack_buffer(data_, size()); +} + } // namespace legate diff --git a/src/core/data/scalar.h b/src/core/data/scalar.h index f06cc41a61..1a22714e94 100644 --- a/src/core/data/scalar.h +++ b/src/core/data/scalar.h @@ -27,6 +27,8 @@ namespace legate { +class BufferBuilder; + /** * @ingroup data * @brief A type-erased container for scalars and tuples of scalars. @@ -120,6 +122,14 @@ class Scalar { */ const void* ptr() const { return data_; } + public: + /** + * @brief Serializes the scalar into a buffer + * + * @param buffer A BufferBuilder into which the scalar needs to be serialized + */ + void pack(BufferBuilder& buffer) const; + private: bool own_{false}; std::unique_ptr type_{nullptr}; diff --git a/src/core/runtime/launcher_arg.cc b/src/core/runtime/launcher_arg.cc index fd71f7d5f8..c0e10fcc5d 100644 --- a/src/core/runtime/launcher_arg.cc +++ b/src/core/runtime/launcher_arg.cc @@ -21,13 +21,7 @@ namespace legate { -void UntypedScalarArg::pack(BufferBuilder& buffer) const -{ - // FIXME: Need to catch up the type system change - // buffer.pack(scalar_.is_tuple()); - buffer.pack(static_cast(scalar_.type().code)); - buffer.pack_buffer(scalar_.ptr(), scalar_.size()); -} +void UntypedScalarArg::pack(BufferBuilder& buffer) const { scalar_.pack(buffer); } RegionFieldArg::RegionFieldArg(RequirementAnalyzer* analyzer, detail::LogicalStore* store, diff --git a/src/core/runtime/operation.cc b/src/core/runtime/operation.cc index 050f6165a2..8b401687d0 100644 --- a/src/core/runtime/operation.cc +++ b/src/core/runtime/operation.cc @@ -106,8 +106,7 @@ void Task::launch(Strategy* p_strategy) auto& var = pair.second; auto field_space = strategy.find_field_space(var); // TODO: We should reuse field ids here - // FIXME: Need to catch up the type system change - auto field_size = 0; // store->type().size(); + auto field_size = store->type().size(); auto field_id = runtime->allocate_field(field_space, field_size); launcher.add_unbound_output(store, field_space, field_id); } From 1685c99337ff0eb082057a7f6d8888aa9637a505 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Mon, 8 May 2023 11:57:09 -0700 Subject: [PATCH 0113/1425] Handle unbound stores correctly in manual tasks * Handle unbound stores correctly in manual tasks See merge request legate/legate.core.internal!20 --- src/core/runtime/operation.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/core/runtime/operation.cc b/src/core/runtime/operation.cc index 8b401687d0..8366636cd4 100644 --- a/src/core/runtime/operation.cc +++ b/src/core/runtime/operation.cc @@ -296,8 +296,12 @@ void ManualTask::add_store(std::vector& store_args, auto store_impl = store.impl(); auto partition_symbol = declare_partition(); store_args.push_back(StoreArg(store_impl.get(), partition_symbol)); - strategy_->insert(partition_symbol, std::move(partition)); all_stores_.insert(std::move(store_impl)); + if (store.unbound()) { + auto field_space = Runtime::get_runtime()->create_field_space(); + strategy_->insert(partition_symbol, std::move(partition), field_space); + } else + strategy_->insert(partition_symbol, std::move(partition)); } void ManualTask::launch(Strategy*) { Task::launch(strategy_.get()); } From a24cc775ee6b5de0a30bab8c88d7f67ebff96b44 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Mon, 8 May 2023 22:19:00 -0700 Subject: [PATCH 0114/1425] Allow the runtime to create more than LEGION_MAX_FIELDS stores for one field size * A stress test for the region manager * Allow the runtime to create more than LEGION_MAX_FIELDS stores for a field size See merge request legate/legate.core.internal!21 --- src/core/runtime/runtime.cc | 59 +++++++---- src/core/runtime/runtime.h | 3 + tests/cpp/.clang-format | 129 ------------------------ tests/cpp/integration/region_manager.cc | 66 ++++++++++++ 4 files changed, 110 insertions(+), 147 deletions(-) delete mode 100644 tests/cpp/.clang-format create mode 100644 tests/cpp/integration/region_manager.cc diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 3e67705ac8..a7c3d28b61 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -341,12 +341,29 @@ void register_builtin_reduction_ops() //////////////////////////////////////////////////// class RegionManager { + private: + struct ManagerEntry { + static constexpr Legion::FieldID FIELD_ID_BASE = 10000; + static constexpr int32_t MAX_NUM_FIELDS = LEGION_MAX_FIELDS - LEGION_DEFAULT_LOCAL_FIELDS; + + ManagerEntry(const Legion::LogicalRegion& _region) + : region(_region), next_field_id(FIELD_ID_BASE) + { + } + bool has_space() const { return next_field_id - FIELD_ID_BASE < MAX_NUM_FIELDS; } + Legion::FieldID get_next_field_id() { return next_field_id++; } + + Legion::LogicalRegion region; + Legion::FieldID next_field_id; + }; + public: RegionManager(Runtime* runtime, const Domain& shape); private: - Legion::LogicalRegion active_region() const; - void create_region(); + const ManagerEntry& active_entry() const { return entries_.back(); } + ManagerEntry& active_entry() { return entries_.back(); } + void push_entry(); public: bool has_space() const; @@ -356,7 +373,7 @@ class RegionManager { private: Runtime* runtime_; Domain shape_; - std::vector regions_{}; + std::vector entries_{}; }; RegionManager::RegionManager(Runtime* runtime, const Domain& shape) @@ -364,28 +381,27 @@ RegionManager::RegionManager(Runtime* runtime, const Domain& shape) { } -Legion::LogicalRegion RegionManager::active_region() const { return regions_.back(); } - -void RegionManager::create_region() +void RegionManager::push_entry() { auto is = runtime_->find_or_create_index_space(shape_); auto fs = runtime_->create_field_space(); - regions_.push_back(runtime_->create_region(is, fs)); + entries_.emplace_back(runtime_->create_region(is, fs)); } -bool RegionManager::has_space() const { return regions_.size() > 0; } +bool RegionManager::has_space() const { return !entries_.empty() && active_entry().has_space(); } std::pair RegionManager::allocate_field(size_t field_size) { - if (!has_space()) create_region(); - auto lr = active_region(); - auto fid = runtime_->allocate_field(lr.get_field_space(), field_size); - return std::make_pair(lr, fid); + if (!has_space()) push_entry(); + auto& entry = active_entry(); + auto fid = + runtime_->allocate_field(entry.region.get_field_space(), entry.get_next_field_id(), field_size); + return std::make_pair(entry.region, fid); } void RegionManager::import_region(const Legion::LogicalRegion& region) { - regions_.push_back(region); + entries_.emplace_back(region); } //////////////////////////////////////////////////// @@ -425,11 +441,9 @@ std::shared_ptr FieldManager::allocate_field() free_fields_.pop_front(); rf = new LogicalRegionField(field.first, field.second); } else { - auto rgn_mgr = runtime_->find_or_create_region_manager(shape_); - Legion::LogicalRegion lr; - Legion::FieldID fid; - std::tie(lr, fid) = rgn_mgr->allocate_field(field_size_); - rf = new LogicalRegionField(lr, fid); + auto rgn_mgr = runtime_->find_or_create_region_manager(shape_); + auto [lr, fid] = rgn_mgr->allocate_field(field_size_); + rf = new LogicalRegionField(lr, fid); log_legate.debug("Field %u created in field manager %p", fid, this); } assert(rf != nullptr); @@ -943,6 +957,15 @@ Legion::FieldID Runtime::allocate_field(const Legion::FieldSpace& field_space, s return allocator.allocate_field(field_size); } +Legion::FieldID Runtime::allocate_field(const Legion::FieldSpace& field_space, + Legion::FieldID field_id, + size_t field_size) +{ + assert(nullptr != legion_context_); + auto allocator = legion_runtime_->create_field_allocator(legion_context_, field_space); + return allocator.allocate_field(field_size, field_id); +} + Domain Runtime::get_index_space_domain(const Legion::IndexSpace& index_space) const { assert(nullptr != legion_context_); diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index ec294e099e..50964b1e9e 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -203,6 +203,9 @@ class Runtime { const Legion::IndexPartition& index_partition); Legion::Future create_future(const void* data, size_t datalen) const; Legion::FieldID allocate_field(const Legion::FieldSpace& field_space, size_t field_size); + Legion::FieldID allocate_field(const Legion::FieldSpace& field_space, + Legion::FieldID field_id, + size_t field_size); Legion::Domain get_index_space_domain(const Legion::IndexSpace& index_space) const; public: diff --git a/tests/cpp/.clang-format b/tests/cpp/.clang-format deleted file mode 100644 index 1ceb0920ba..0000000000 --- a/tests/cpp/.clang-format +++ /dev/null @@ -1,129 +0,0 @@ -Language: Cpp -# BasedOnStyle: Google -AccessModifierOffset: -1 -AlignAfterOpenBracket: Align -AlignConsecutiveAssignments: true -AlignConsecutiveDeclarations: false -AlignEscapedNewlines: Left -AlignOperands: true -AlignTrailingComments: true -AllowAllParametersOfDeclarationOnNextLine: true -AllowShortBlocksOnASingleLine: true -AllowShortCaseLabelsOnASingleLine: true -AllowShortFunctionsOnASingleLine: All -AllowShortIfStatementsOnASingleLine: true -AllowShortLoopsOnASingleLine: true -# This is deprecated -AlwaysBreakAfterDefinitionReturnType: None -AlwaysBreakAfterReturnType: None -AlwaysBreakBeforeMultilineStrings: true -AlwaysBreakTemplateDeclarations: Yes -BinPackArguments: false -BinPackParameters: false -BraceWrapping: - AfterClass: false - AfterControlStatement: false - AfterEnum: false - AfterFunction: false - AfterNamespace: false - AfterObjCDeclaration: false - AfterStruct: false - AfterUnion: false - AfterExternBlock: false - BeforeCatch: false - BeforeElse: false - IndentBraces: false - # disabling the below splits, else, they'll just add to the vertical length of source files! - SplitEmptyFunction: false - SplitEmptyRecord: false - SplitEmptyNamespace: false -BreakBeforeBinaryOperators: None -BreakBeforeBraces: WebKit -BreakBeforeInheritanceComma: false -BreakInheritanceList: BeforeColon -BreakBeforeTernaryOperators: true -BreakConstructorInitializersBeforeComma: false -BreakConstructorInitializers: BeforeColon -BreakAfterJavaFieldAnnotations: false -BreakStringLiterals: true -ColumnLimit: 100 -CommentPragmas: '^ IWYU pragma:' -CompactNamespaces: false -ConstructorInitializerAllOnOneLineOrOnePerLine: true -# Kept the below 2 to be the same as `IndentWidth` to keep everything uniform -ConstructorInitializerIndentWidth: 2 -ContinuationIndentWidth: 2 -Cpp11BracedListStyle: true -DerivePointerAlignment: false -DisableFormat: false -ExperimentalAutoDetectBinPacking: false -FixNamespaceComments: true -ForEachMacros: -IncludeBlocks: Preserve -IncludeCategories: - - Regex: '^<.*\.h>' - Priority: 1 - - Regex: '^<.*' - Priority: 2 - - Regex: '.*' - Priority: 3 -IncludeIsMainRegex: '([-_](test|unittest))?$' -IndentCaseLabels: true -IndentPPDirectives: None -IndentWidth: 2 -IndentWrappedFunctionNames: false -JavaScriptQuotes: Leave -JavaScriptWrapImports: true -KeepEmptyLinesAtTheStartOfBlocks: false -MacroBlockBegin: '' -MacroBlockEnd: '' -MaxEmptyLinesToKeep: 1 -NamespaceIndentation: None -ObjCBinPackProtocolList: Never -ObjCBlockIndentWidth: 2 -ObjCSpaceAfterProperty: false -ObjCSpaceBeforeProtocolList: true -PenaltyBreakAssignment: 2 -PenaltyBreakBeforeFirstCallParameter: 1 -PenaltyBreakComment: 300 -PenaltyBreakFirstLessLess: 120 -PenaltyBreakString: 1000 -PenaltyBreakTemplateDeclaration: 10 -PenaltyExcessCharacter: 1000000 -PenaltyReturnTypeOnItsOwnLine: 200 -PointerAlignment: Left -RawStringFormats: - - Language: Cpp - Delimiters: - - cc - - CC - - cpp - - Cpp - - CPP - - 'c++' - - 'C++' - CanonicalDelimiter: '' -# Enabling comment reflow causes doxygen comments to be messed up in their formats! -ReflowComments: true -SortIncludes: true -SortUsingDeclarations: true -SpaceAfterCStyleCast: false -SpaceAfterTemplateKeyword: true -SpaceBeforeAssignmentOperators: true -SpaceBeforeCpp11BracedList: false -SpaceBeforeCtorInitializerColon: true -SpaceBeforeInheritanceColon: true -SpaceBeforeParens: ControlStatements -SpaceBeforeRangeBasedForLoopColon: true -SpaceInEmptyParentheses: false -SpacesBeforeTrailingComments: 2 -SpacesInAngles: false -SpacesInContainerLiterals: true -SpacesInCStyleCastParentheses: false -SpacesInParentheses: false -SpacesInSquareBrackets: false -Standard: Cpp11 -StatementMacros: -# Be consistent with indent-width, even for people who use tab for indentation! -TabWidth: 2 -UseTab: Never diff --git a/tests/cpp/integration/region_manager.cc b/tests/cpp/integration/region_manager.cc new file mode 100644 index 0000000000..d8bbeedef9 --- /dev/null +++ b/tests/cpp/integration/region_manager.cc @@ -0,0 +1,66 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "legate.h" + +namespace region_manager { + +namespace { + +static const char* library_name = "test_region_manager"; + +struct TesterTask : public legate::LegateTask { + static const int32_t TASK_ID = 0; + static void cpu_variant(legate::TaskContext& context) {} +}; + +void prepare() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + TesterTask::register_variants(context); +} + +} // namespace + +void test_region_manager(int32_t argc, char** argv) +{ + legate::Core::perform_registration(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + auto task = runtime->create_task(context, 0); + + std::vector stores; + for (uint32_t idx = 0; idx < LEGION_MAX_FIELDS * 2; ++idx) { + auto store = runtime->create_store({10}, legate::int64()); + auto part = task->declare_partition(); + task->add_output(store, part); + stores.push_back(store); + } + runtime->submit(std::move(task)); +} + +} // namespace region_manager + +TEST(Integration, RegionManager) +{ + legate::initialize(0, NULL); + legate::set_main_function(region_manager::test_region_manager); + legate::start(0, NULL); +} From 646752a61c7b8d5079861eb8c8b2a89ddb8ea17c Mon Sep 17 00:00:00 2001 From: Irina Demeshko Date: Fri, 12 May 2023 08:53:02 -0700 Subject: [PATCH 0115/1425] Adding support for Provenance --- src/core/data/scalar.cc | 9 ++ src/core/data/scalar.h | 8 ++ src/core/runtime/context.cc | 2 + src/core/runtime/context.h | 3 + src/core/runtime/launcher.cc | 10 +- src/core/runtime/runtime.cc | 70 ++++++++++++- src/core/runtime/runtime.h | 35 +++++++ src/core/type/type_info.cc | 3 + src/core/type/type_info.h | 8 ++ src/core/type/type_traits.h | 6 ++ tests/cpp/integration/provenance.cc | 157 ++++++++++++++++++++++++++++ 11 files changed, 306 insertions(+), 5 deletions(-) create mode 100644 tests/cpp/integration/provenance.cc diff --git a/src/core/data/scalar.cc b/src/core/data/scalar.cc index 640f2c80c3..a5b792261b 100644 --- a/src/core/data/scalar.cc +++ b/src/core/data/scalar.cc @@ -25,6 +25,15 @@ Scalar::Scalar(std::unique_ptr type, const void* data) : type_(std::move(t { } +Scalar::Scalar(const std::string& string) : own_(true), type_(string_type()) +{ + auto data_size = sizeof(char) * string.size(); + auto buffer = malloc(sizeof(uint32_t) + data_size); + *static_cast(buffer) = string.size(); + memcpy(static_cast(buffer) + sizeof(uint32_t), string.data(), data_size); + data_ = buffer; +} + Scalar::~Scalar() { if (own_) diff --git a/src/core/data/scalar.h b/src/core/data/scalar.h index 1a22714e94..8b3207d49e 100644 --- a/src/core/data/scalar.h +++ b/src/core/data/scalar.h @@ -67,6 +67,14 @@ class Scalar { */ template Scalar(T value); + /** + * @brief Creates an owned scalar from a string. The value from the + * original string will be copied. + * + * @param string A string to create a `Scalar` with + */ + Scalar(const std::string& string); + /** * @brief Creates an owned scalar from a tuple of scalars. The values in the input vector * will be copied. diff --git a/src/core/runtime/context.cc b/src/core/runtime/context.cc index f1d59fa2d6..7c6addb6a6 100644 --- a/src/core/runtime/context.cc +++ b/src/core/runtime/context.cc @@ -296,4 +296,6 @@ std::vector TaskContext::get_return_values() const return std::move(return_values); } +const std::string& TaskContext::get_provenance() const { return task_->get_provenance_string(); } + } // namespace legate diff --git a/src/core/runtime/context.h b/src/core/runtime/context.h index c39a27afdf..4f56a55cf9 100644 --- a/src/core/runtime/context.h +++ b/src/core/runtime/context.h @@ -285,6 +285,9 @@ class TaskContext { ReturnValues pack_return_values_with_exception(int32_t index, const std::string& error_message) const; + public: + const std::string& get_provenance() const; + private: std::vector get_return_values() const; diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/launcher.cc index 71df2b74b4..72ac8adac1 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -185,13 +185,16 @@ std::unique_ptr TaskLauncher::build_single_task() buffer_->pack(0); pack_mapper_arg(); + auto* runtime = Runtime::get_runtime(); + auto& provenance = runtime->provenance_manager()->get_provenance(); auto single_task = std::make_unique(legion_task_id(), buffer_->to_legion_buffer(), Legion::Predicate::TRUE_PRED, legion_mapper_id(), tag_, - mapper_arg_->to_legion_buffer()); + mapper_arg_->to_legion_buffer(), + provenance.c_str()); for (auto& future : futures_) single_task->add_future(future); req_analyzer_->populate_launcher(single_task.get()); @@ -220,6 +223,8 @@ std::unique_ptr TaskLauncher::build_index_task( buffer_->pack(0); pack_mapper_arg(); + auto* runtime = Runtime::get_runtime(); + auto& provenance = runtime->provenance_manager()->get_provenance(); auto index_task = std::make_unique(legion_task_id(), launch_domain, @@ -229,7 +234,8 @@ std::unique_ptr TaskLauncher::build_index_task( false /*must*/, legion_mapper_id(), tag_, - mapper_arg_->to_legion_buffer()); + mapper_arg_->to_legion_buffer(), + provenance.c_str()); for (auto& future : futures_) index_task->add_future(future); for (auto& future_map : future_maps_) index_task->point_futures.push_back(future_map); diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index a7c3d28b61..a716302aea 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -60,6 +60,8 @@ static const char* const core_library_name = "legate.core"; /*static*/ LegateMainFnPtr Core::main_fn = nullptr; +static const std::string EMPTY_STRING = ""; + /*static*/ void Core::parse_config(void) { #ifndef LEGATE_USE_CUDA @@ -753,9 +755,10 @@ bool Runtime::is_in_callback() const { return in_callback_; } void Runtime::post_startup_initialization(Legion::Context legion_context) { - legion_context_ = legion_context; - core_context_ = find_library(core_library_name); - partition_manager_ = new PartitionManager(this, core_context_); + legion_context_ = legion_context; + core_context_ = find_library(core_library_name); + partition_manager_ = new PartitionManager(this, core_context_); + provenance_manager_ = new ProvenanceManager(); Core::retrieve_tunable(legion_context_, legion_runtime_, core_context_); initialize_toplevel_machine(); } @@ -901,6 +904,8 @@ FieldManager* Runtime::find_or_create_field_manager(const Domain& shape, uint32_ PartitionManager* Runtime::partition_manager() const { return partition_manager_; } +ProvenanceManager* Runtime::provenance_manager() const { return provenance_manager_; } + Legion::IndexSpace Runtime::find_or_create_index_space(const Domain& shape) { assert(nullptr != legion_context_); @@ -1124,6 +1129,65 @@ void set_main_function(LegateMainFnPtr main_fn) { Core::main_fn = main_fn; } int32_t start(int32_t argc, char** argv) { return Runtime::start(argc, argv); } +ProvenanceManager::ProvenanceManager() { provenance_.push_back(EMPTY_STRING); } + +const std::string& ProvenanceManager::get_provenance() +{ +#ifdef DEBUG_LEGATE + assert(provenance_.size() > 0); +#endif + return provenance_.back(); +} + +void ProvenanceManager::set_provenance(const std::string& p) +{ +#ifdef DEBUG_LEGATE + assert(provenance_.size() > 0); +#endif + provenance_.back() = p; +} + +void ProvenanceManager::reset_provenance() +{ +#ifdef DEBUG_LEGATE + assert(provenance_.size() > 0); +#endif + provenance_.back() = EMPTY_STRING; +} + +void ProvenanceManager::push_provenance(const std::string& p) { provenance_.push_back(p); } + +void ProvenanceManager::pop_provenance() +{ + if (provenance_.size() <= 1) { + throw std::underflow_error("can't pop from an empty provenance stack"); + } + provenance_.pop_back(); +} + +void ProvenanceManager::clear_all() +{ + provenance_.clear(); + provenance_.push_back(EMPTY_STRING); +} + +ProvenanceTracker::ProvenanceTracker(const std::string& p) +{ + auto* runtime = Runtime::get_runtime(); + runtime->provenance_manager()->push_provenance(p); +} + +ProvenanceTracker::~ProvenanceTracker() +{ + auto* runtime = Runtime::get_runtime(); + runtime->provenance_manager()->pop_provenance(); +} + +const std::string& ProvenanceTracker::get_current_provenance() const +{ + return Runtime::get_runtime()->provenance_manager()->get_provenance(); +} + } // namespace legate extern "C" { diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 50964b1e9e..ec776c23f6 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -103,6 +103,27 @@ class ResourceConfig; class Runtime; class Tiling; +class ProvenanceManager { + public: + ProvenanceManager(); + + public: + const std::string& get_provenance(); + + void set_provenance(const std::string& p); + + void reset_provenance(); + + void push_provenance(const std::string& p); + + void pop_provenance(); + + void clear_all(); + + private: + std::vector provenance_; +}; + class PartitionManager { public: PartitionManager(Runtime* runtime, const LibraryContext* context); @@ -188,6 +209,7 @@ class Runtime { RegionManager* find_or_create_region_manager(const Legion::Domain& shape); FieldManager* find_or_create_field_manager(const Legion::Domain& shape, uint32_t field_size); PartitionManager* partition_manager() const; + ProvenanceManager* provenance_manager() const; public: Legion::IndexSpace find_or_create_index_space(const Legion::Domain& shape); @@ -256,6 +278,7 @@ class Runtime { std::map field_managers_; std::map region_managers_; PartitionManager* partition_manager_{nullptr}; + ProvenanceManager* provenance_manager_{nullptr}; private: std::map index_spaces_; @@ -295,6 +318,18 @@ void set_main_function(LegateMainFnPtr p_main); int32_t start(int32_t argc, char** argv); +struct ProvenanceTracker { + ProvenanceTracker(const std::string& p); + ~ProvenanceTracker(); + const std::string& get_current_provenance() const; +}; + } // namespace legate +#define TRACK_PROVENANCE(STMT) \ + do { \ + legate::ProvenanceTracker track(std::string(__FILE__) + ":" + std::to_string(__LINE__)); \ + STMT; \ + } while (false) + #include "core/runtime/runtime.inl" diff --git a/src/core/type/type_info.cc b/src/core/type/type_info.cc index 8a7cf11da2..20cc340f37 100644 --- a/src/core/type/type_info.cc +++ b/src/core/type/type_info.cc @@ -58,6 +58,7 @@ const std::unordered_map TYPE_NAMES = { {Type::Code::FLOAT64, "float64"}, {Type::Code::COMPLEX64, "complex64"}, {Type::Code::COMPLEX128, "complex128"}, + {Type::Code::STRING, "string"}, }; const char* _VARIABLE_SIZE_ERROR_MESSAGE = "Variable-size element type cannot be used"; @@ -305,4 +306,6 @@ std::unique_ptr complex64() { return primitive_type(Type::Code::COMPLEX64) std::unique_ptr complex128() { return primitive_type(Type::Code::COMPLEX128); } +std::unique_ptr string() { return string_type(); } + } // namespace legate diff --git a/src/core/type/type_info.h b/src/core/type/type_info.h index 09d393a89f..cb9ca9c9c2 100644 --- a/src/core/type/type_info.h +++ b/src/core/type/type_info.h @@ -501,4 +501,12 @@ std::unique_ptr complex64(); */ std::unique_ptr complex128(); +/** + * @ingroup types + * @brief Creates a string type + * + * @return Type object + */ +std::unique_ptr string(); + } // namespace legate diff --git a/src/core/type/type_traits.h b/src/core/type/type_traits.h index aa9ef936a2..f6ebba8b3e 100644 --- a/src/core/type/type_traits.h +++ b/src/core/type/type_traits.h @@ -79,6 +79,8 @@ template <> PREFIX constexpr Type::Code legate_type_code_of> = Type::Code::COMPLEX64; template <> PREFIX constexpr Type::Code legate_type_code_of> = Type::Code::COMPLEX128; +template <> +PREFIX constexpr Type::Code legate_type_code_of = Type::Code::STRING; #undef PREFIX @@ -142,6 +144,10 @@ template <> struct LegateTypeOf { using type = complex; }; +template <> +struct LegateTypeOf { + using type = std::string; +}; /** * @ingroup util diff --git a/tests/cpp/integration/provenance.cc b/tests/cpp/integration/provenance.cc new file mode 100644 index 0000000000..d9460cec6b --- /dev/null +++ b/tests/cpp/integration/provenance.cc @@ -0,0 +1,157 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "core/mapping/mapping.h" +#include "legate.h" + +namespace provenance { + +static const char* library_name = "provenance"; +static legate::Logger logger(library_name); + +enum TaskIDs { + PROVENANCE = 0, +}; + +struct ProvenanceTask : public legate::LegateTask { + static const int32_t TASK_ID = PROVENANCE; + static void cpu_variant(legate::TaskContext& context); +}; + +void register_tasks() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + ProvenanceTask::register_variants(context); +} + +/*static*/ void ProvenanceTask::cpu_variant(legate::TaskContext& context) +{ + std::string scalar = context.scalars()[0].value(); + auto provenance = context.get_provenance(); + EXPECT_TRUE(provenance.find(scalar) != std::string::npos); +} + +void test_manual_provenance(legate::LibraryContext* context) +{ + auto runtime = legate::Runtime::get_runtime(); + std::string provenance = "test_manual_provenance"; + runtime->provenance_manager()->set_provenance(provenance); + // auto task + auto task = runtime->create_task(context, PROVENANCE); + task->add_scalar_arg(legate::Scalar(provenance)); + runtime->submit(std::move(task)); +} + +void test_push_provenance(legate::LibraryContext* context) +{ + auto runtime = legate::Runtime::get_runtime(); + std::string provenance = "test_push_provenance"; + runtime->provenance_manager()->push_provenance(provenance); + EXPECT_EQ(runtime->provenance_manager()->get_provenance(), provenance); + // auto task + auto task = runtime->create_task(context, PROVENANCE); + task->add_scalar_arg(legate::Scalar(provenance)); + runtime->submit(std::move(task)); +} + +void test_pop_provenance(legate::LibraryContext* context) +{ + auto runtime = legate::Runtime::get_runtime(); + runtime->provenance_manager()->clear_all(); + runtime->provenance_manager()->push_provenance("some provenance for provenance task"); + runtime->provenance_manager()->pop_provenance(); + // auto task + auto task = runtime->create_task(context, PROVENANCE); + std::string provenance = ""; + task->add_scalar_arg(legate::Scalar(provenance)); + runtime->submit(std::move(task)); +} + +void test_underflow(legate::LibraryContext* context) +{ + auto runtime = legate::Runtime::get_runtime(); + runtime->provenance_manager()->clear_all(); + runtime->provenance_manager()->set_provenance("some provenance for provenance task"); + EXPECT_THROW(runtime->provenance_manager()->pop_provenance(), std::runtime_error); +} + +void test_clear_provenance(legate::LibraryContext* context) +{ + auto runtime = legate::Runtime::get_runtime(); + runtime->provenance_manager()->push_provenance("provenance for provenance task"); + runtime->provenance_manager()->push_provenance("another provenance"); + runtime->provenance_manager()->clear_all(); + // auto task + auto task = runtime->create_task(context, PROVENANCE); + std::string provenance = ""; + task->add_scalar_arg(legate::Scalar(provenance)); + runtime->submit(std::move(task)); +} + +void test_provenance_tracker(legate::LibraryContext* context) +{ + auto runtime = legate::Runtime::get_runtime(); + // auto task + auto task = runtime->create_task(context, PROVENANCE); + std::string provenance = "provenance.cc:114"; + task->add_scalar_arg(legate::Scalar(provenance)); + TRACK_PROVENANCE(runtime->submit(std::move(task))); +} + +void test_nested_provenance_tracker(legate::LibraryContext* context) +{ + legate::ProvenanceTracker track(std::string(__FILE__) + std::to_string(__LINE__)); + test_provenance_tracker(context); +} + +void test_manual_tracker(legate::LibraryContext* context) +{ + legate::ProvenanceTracker track("manual provenance through tracker"); + auto runtime = legate::Runtime::get_runtime(); + // auto task + auto task = runtime->create_task(context, PROVENANCE); + task->add_scalar_arg(legate::Scalar(track.get_current_provenance())); + runtime->submit(std::move(task)); +} + +void legate_main(int32_t argc, char** argv) +{ + legate::Core::perform_registration(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + test_manual_provenance(context); + test_push_provenance(context); + test_pop_provenance(context); + test_underflow(context); + test_clear_provenance(context); + test_provenance_tracker(context); + test_nested_provenance_tracker(context); + test_manual_tracker(context); +} + +} // namespace provenance + +TEST(Integration, Provenance) +{ + legate::initialize(0, NULL); + legate::set_main_function(provenance::legate_main); + legate::start(0, NULL); +} From 41f39bb9fa342bff61ffd1f519f53633b1ce88a0 Mon Sep 17 00:00:00 2001 From: Vladislav Zhurba Date: Fri, 12 May 2023 14:38:23 -0700 Subject: [PATCH 0116/1425] Bootstrap using implicit top task * Rebase * Return non-standalone dependencies * Bootstrap using implicit top task See merge request legate/legate.core.internal!14 --- src/core/runtime/runtime.cc | 98 ++++++++++------------- src/core/runtime/runtime.h | 7 +- tests/cpp/CMakeLists.txt | 5 +- tests/cpp/integration/manual_simple.cc | 9 +-- tests/cpp/integration/multi_scalar_out.cc | 9 +-- tests/cpp/integration/provenance.cc | 9 +-- tests/cpp/integration/region_manager.cc | 9 +-- tests/cpp/main.cc | 26 ++++++ tests/cpp/unit/store.cc | 27 +------ 9 files changed, 83 insertions(+), 116 deletions(-) create mode 100644 tests/cpp/main.cc diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index a716302aea..0c38b4a1b9 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -56,10 +56,6 @@ static const char* const core_library_name = "legate.core"; /*static*/ bool Core::has_socket_mem = false; -/*static*/ bool Core::standalone = false; - -/*static*/ LegateMainFnPtr Core::main_fn = nullptr; - static const std::string EMPTY_STRING = ""; /*static*/ void Core::parse_config(void) @@ -102,24 +98,6 @@ static const std::string EMPTY_STRING = ""; parse_variable("LEGATE_LOG_MAPPING", log_mapping_decisions); } -static void toplevel_task(const Legion::Task* task, - const std::vector& regions, - Legion::Context ctx, - Legion::Runtime* legion_runtime) -{ - auto runtime = Runtime::get_runtime(); - runtime->post_startup_initialization(ctx); - - if (nullptr == Core::main_fn) { - log_legate.error( - "No main function was provided. Please register one with 'legate::set_main_function'."); - LEGATE_ABORT; - } - - auto args = Legion::Runtime::get_input_args(); - Core::main_fn(args.argc, args.argv); -} - static void extract_scalar_task( const void* args, size_t arglen, const void* userdata, size_t userlen, Legion::Processor p) { @@ -193,11 +171,6 @@ void register_legate_core_tasks(Legion::Machine machine, Legion::Runtime* runtime, const LibraryContext* context) { - auto toplevel_task_id = context->get_task_id(LEGATE_CORE_TOPLEVEL_TASK_ID); - const char* toplevel_task_name = "Legate Core Toplevel Task"; - runtime->attach_name( - toplevel_task_id, toplevel_task_name, false /*mutable*/, true /*local only*/); - auto extract_scalar_task_id = context->get_task_id(LEGATE_CORE_EXTRACT_SCALAR_TASK_ID); const char* extract_scalar_task_name = "core::extract_scalar"; runtime->attach_name( @@ -211,16 +184,6 @@ void register_legate_core_tasks(Legion::Machine machine, return registrar; }; - // Register the task variant for both CPUs and GPUs - { - Legion::TaskVariantRegistrar registrar(toplevel_task_id, toplevel_task_name); - registrar.add_constraint(Legion::ProcessorConstraint(Processor::LOC_PROC)); - registrar.set_leaf(false); - registrar.set_inner(false); - registrar.set_replicable(true); - registrar.global_registration = false; - runtime->register_task_variant(registrar, LEGATE_CPU_VARIANT); - } // Register the task variants auto register_extract_scalar = [&](auto proc_kind, auto variant_id) { auto registrar = make_registrar(extract_scalar_task_id, extract_scalar_task_name, proc_kind); @@ -289,9 +252,9 @@ void register_builtin_reduction_ops() RECORD_INT(XOR_LT) } -/*static*/ void core_library_registration_callback(Legion::Machine machine, - Legion::Runtime* legion_runtime, - const std::set& local_procs) +/*static*/ void core_library_registration(Legion::Machine machine, + Legion::Runtime* legion_runtime, + const std::set& local_procs) { Runtime::create_runtime(legion_runtime); @@ -316,24 +279,22 @@ void register_builtin_reduction_ops() register_legate_core_projection_functors(legion_runtime, core_lib); register_legate_core_sharding_functors(legion_runtime, core_lib); +} - if (!Core::standalone) - Runtime::get_runtime()->post_startup_initialization(Legion::Runtime::get_context()); +/*static*/ void core_library_registration_callback(Legion::Machine machine, + Legion::Runtime* legion_runtime, + const std::set& local_procs) +{ + core_library_registration(machine, legion_runtime, local_procs); + + Runtime::get_runtime()->post_startup_initialization(Legion::Runtime::get_context()); } /*static*/ void core_library_bootstrapping_callback(Legion::Machine machine, Legion::Runtime* legion_runtime, const std::set& local_procs) { - Core::standalone = true; - - core_library_registration_callback(machine, legion_runtime, local_procs); - - auto runtime = Runtime::get_runtime(); - - auto core_lib = runtime->find_library(core_library_name); - legion_runtime->set_top_level_task_id(core_lib->get_task_id(LEGATE_CORE_TOPLEVEL_TASK_ID)); - legion_runtime->set_top_level_task_mapper_id(core_lib->get_mapper_id()); + core_library_registration(machine, legion_runtime, local_procs); Core::parse_config(); } @@ -1113,7 +1074,36 @@ Legion::ProjectionID Runtime::get_delinearizing_projection() /*static*/ int32_t Runtime::start(int32_t argc, char** argv) { - return Legion::Runtime::start(argc, argv); + auto result = Legion::Runtime::start(argc, argv, true); + if (result != 0) { + log_legate.error("Legion Runtime failed to start."); + return result; + } + + // Get the runtime now that we've started it + auto runtime = Legion::Runtime::get_runtime(); + + // Then we can make this thread into an implicit top-level task + const char* toplevel_task_name = "Legate Core Toplevel Task"; + auto ctx = runtime->begin_implicit_task(LEGATE_CORE_TOPLEVEL_TASK_ID, + 0 /*mapper id*/, + Processor::LOC_PROC, + toplevel_task_name, + true /*control replicable*/); + Runtime::get_runtime()->post_startup_initialization(ctx); + + return result; +} + +int32_t Runtime::wait_for_shutdown() +{ + // Mark that we are done excecuting the top-level task + // After this call the context is no longer valid + Legion::Runtime::get_runtime()->finish_implicit_task(legion_context_); + + // The previous call is asynchronous so we still need to + // wait for the shutdown of the runtime to complete + return Legion::Runtime::wait_for_shutdown(); } /*static*/ Runtime* Runtime::get_runtime() { return Runtime::runtime_; } @@ -1125,10 +1115,10 @@ Legion::ProjectionID Runtime::get_delinearizing_projection() void initialize(int32_t argc, char** argv) { Runtime::initialize(argc, argv); } -void set_main_function(LegateMainFnPtr main_fn) { Core::main_fn = main_fn; } - int32_t start(int32_t argc, char** argv) { return Runtime::start(argc, argv); } +int32_t wait_for_shutdown() { return Runtime::get_runtime()->wait_for_shutdown(); } + ProvenanceManager::ProvenanceManager() { provenance_.push_back(EMPTY_STRING); } const std::string& ProvenanceManager::get_provenance() diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index ec776c23f6..f8ce43e1da 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -87,8 +87,6 @@ struct Core { static bool synchronize_stream_view; static bool log_mapping_decisions; static bool has_socket_mem; - static bool standalone; - static LegateMainFnPtr main_fn; }; class AutoTask; @@ -264,6 +262,7 @@ class Runtime { public: static Runtime* get_runtime(); static void create_runtime(Legion::Runtime* legion_runtime); + int32_t wait_for_shutdown(); private: static Runtime* runtime_; @@ -314,10 +313,10 @@ class Runtime { void initialize(int32_t argc, char** argv); -void set_main_function(LegateMainFnPtr p_main); - int32_t start(int32_t argc, char** argv); +int32_t wait_for_shutdown(); + struct ProvenanceTracker { ProvenanceTracker(const std::string& p); ~ProvenanceTracker(); diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt index bf39b9f8db..12d788bb3f 100644 --- a/tests/cpp/CMakeLists.txt +++ b/tests/cpp/CMakeLists.txt @@ -35,12 +35,13 @@ find_package(legate_core REQUIRED) enable_testing() +file(GLOB main_SRC ${PROJECT_SOURCE_DIR}/main.cc) file(GLOB integration_SRC ${PROJECT_SOURCE_DIR}/integration/*.cc) file(GLOB unit_SRC ${PROJECT_SOURCE_DIR}/unit/*.cc) -add_executable(cpp_tests ${integration_SRC} ${unit_SRC}) +add_executable(cpp_tests ${main_SRC} ${integration_SRC} ${unit_SRC}) -target_link_libraries(cpp_tests PRIVATE legate::core PRIVATE GTest::gtest_main) +target_link_libraries(cpp_tests PRIVATE legate::core PRIVATE GTest::gtest) include(GoogleTest) gtest_discover_tests(cpp_tests) diff --git a/tests/cpp/integration/manual_simple.cc b/tests/cpp/integration/manual_simple.cc index 9850a0a786..41a499ce9d 100644 --- a/tests/cpp/integration/manual_simple.cc +++ b/tests/cpp/integration/manual_simple.cc @@ -81,7 +81,7 @@ void print_store(legate::LibraryContext* context, legate::LogicalStore store) logger.print() << ss.str(); } -void legate_main(int32_t argc, char** argv) +TEST(Integration, ManualSimple) { legate::Core::perform_registration(); @@ -96,10 +96,3 @@ void legate_main(int32_t argc, char** argv) } } // namespace manualsimple - -TEST(Integration, ManualSimple) -{ - legate::initialize(0, NULL); - legate::set_main_function(manualsimple::legate_main); - legate::start(0, NULL); -} diff --git a/tests/cpp/integration/multi_scalar_out.cc b/tests/cpp/integration/multi_scalar_out.cc index 5c229068f4..e6466b7983 100644 --- a/tests/cpp/integration/multi_scalar_out.cc +++ b/tests/cpp/integration/multi_scalar_out.cc @@ -129,7 +129,7 @@ void print_stores(legate::LibraryContext* context, logger.print() << ss.str(); } -void legate_main(int32_t argc, char** argv) +TEST(Integration, ManualScalarOut) { legate::Core::perform_registration(); @@ -148,10 +148,3 @@ void legate_main(int32_t argc, char** argv) } } // namespace multiscalarout - -TEST(Integration, ManualScalarOut) -{ - legate::initialize(0, NULL); - legate::set_main_function(multiscalarout::legate_main); - legate::start(0, NULL); -} diff --git a/tests/cpp/integration/provenance.cc b/tests/cpp/integration/provenance.cc index d9460cec6b..4641c767c1 100644 --- a/tests/cpp/integration/provenance.cc +++ b/tests/cpp/integration/provenance.cc @@ -130,7 +130,7 @@ void test_manual_tracker(legate::LibraryContext* context) runtime->submit(std::move(task)); } -void legate_main(int32_t argc, char** argv) +TEST(Integration, Provenance) { legate::Core::perform_registration(); @@ -148,10 +148,3 @@ void legate_main(int32_t argc, char** argv) } } // namespace provenance - -TEST(Integration, Provenance) -{ - legate::initialize(0, NULL); - legate::set_main_function(provenance::legate_main); - legate::start(0, NULL); -} diff --git a/tests/cpp/integration/region_manager.cc b/tests/cpp/integration/region_manager.cc index d8bbeedef9..2f5e236168 100644 --- a/tests/cpp/integration/region_manager.cc +++ b/tests/cpp/integration/region_manager.cc @@ -38,7 +38,7 @@ void prepare() } // namespace -void test_region_manager(int32_t argc, char** argv) +TEST(Integration, RegionManager) { legate::Core::perform_registration(); @@ -57,10 +57,3 @@ void test_region_manager(int32_t argc, char** argv) } } // namespace region_manager - -TEST(Integration, RegionManager) -{ - legate::initialize(0, NULL); - legate::set_main_function(region_manager::test_region_manager); - legate::start(0, NULL); -} diff --git a/tests/cpp/main.cc b/tests/cpp/main.cc new file mode 100644 index 0000000000..b0c55c83e3 --- /dev/null +++ b/tests/cpp/main.cc @@ -0,0 +1,26 @@ +#include +#include "legate.h" + +class Environment : public ::testing::Environment { + public: + Environment(int argc, char** argv) : argc_(argc), argv_(argv) {} + + void SetUp() override + { + legate::initialize(argc_, argv_); + EXPECT_EQ(legate::start(argc_, argv_), 0); + } + void TearDown() override { EXPECT_EQ(legate::wait_for_shutdown(), 0); } + + private: + int argc_; + char** argv_; +}; + +int main(int argc, char** argv) +{ + ::testing::InitGoogleTest(&argc, argv); + ::testing::AddGlobalTestEnvironment(new Environment(argc, argv)); + + return RUN_ALL_TESTS(); +} diff --git a/tests/cpp/unit/store.cc b/tests/cpp/unit/store.cc index 234c970a14..d7cac58d8a 100644 --- a/tests/cpp/unit/store.cc +++ b/tests/cpp/unit/store.cc @@ -8,7 +8,7 @@ namespace unit { -void store_creation(int32_t argc, char** argv) +TEST(Store, Creation) { // Bound { @@ -33,7 +33,7 @@ void store_creation(int32_t argc, char** argv) } } -void store_valid_transform(int32_t argc, char** argv) +TEST(Store, Transform) { // Bound auto runtime = legate::Runtime::get_runtime(); @@ -63,7 +63,7 @@ void store_valid_transform(int32_t argc, char** argv) EXPECT_TRUE(delinearized.transformed()); } -void store_invalid_transform(int32_t argc, char** argv) +TEST(Store, InvalidTransform) { // Bound { @@ -101,24 +101,3 @@ void store_invalid_transform(int32_t argc, char** argv) } } // namespace unit - -TEST(Store, Creation) -{ - legate::initialize(0, NULL); - legate::set_main_function(unit::store_creation); - legate::start(0, NULL); -} - -TEST(Store, Transform) -{ - legate::initialize(0, NULL); - legate::set_main_function(unit::store_valid_transform); - legate::start(0, NULL); -} - -TEST(Store, InvalidTransform) -{ - legate::initialize(0, NULL); - legate::set_main_function(unit::store_invalid_transform); - legate::start(0, NULL); -} From 6f710854684138963307f86320fbb4d8f5eaf660 Mon Sep 17 00:00:00 2001 From: Vladislav Zhurba Date: Fri, 12 May 2023 15:20:31 -0700 Subject: [PATCH 0117/1425] Store features for unit-test * Prioritize throw messaging instead of logging * Add Slice, Transpose and Delinearize to Store * Replace abort for other store methods * Passing Store.Creation See merge request legate/legate.core.internal!16 --- src/core/data/logical_store_detail.cc | 136 +++++++++++++++++++------- src/core/runtime/runtime.cc | 15 ++- src/core/runtime/runtime.h | 1 + tests/cpp/unit/store.cc | 2 +- 4 files changed, 114 insertions(+), 40 deletions(-) diff --git a/src/core/data/logical_store_detail.cc b/src/core/data/logical_store_detail.cc index d02002360d..e5b32ca610 100644 --- a/src/core/data/logical_store_detail.cc +++ b/src/core/data/logical_store_detail.cc @@ -184,15 +184,24 @@ LogicalStore::~LogicalStore() bool LogicalStore::unbound() const { return storage_->unbound(); } -const Shape& LogicalStore::extents() const { return extents_; } +const Shape& LogicalStore::extents() const +{ + if (extents_.empty()) { + Runtime::get_runtime()->flush_scheduling_window(); + if (extents_.empty()) { + throw std::invalid_argument("Illegal to access an uninitialized unbound store"); + } + } + return extents_; +} -size_t LogicalStore::volume() const { return extents_.volume(); } +size_t LogicalStore::volume() const { return extents().volume(); } size_t LogicalStore::storage_size() const { return storage_->volume() * type().size(); } int32_t LogicalStore::dim() const { - return unbound() ? storage_->dim() : static_cast(extents_.size()); + return unbound() ? storage_->dim() : static_cast(extents().size()); } bool LogicalStore::has_scalar_storage() const { return storage_->kind() == Storage::Kind::FUTURE; } @@ -225,11 +234,11 @@ void LogicalStore::set_future(Legion::Future future) std::shared_ptr LogicalStore::promote(int32_t extra_dim, size_t dim_size) const { if (extra_dim < 0 || extra_dim > dim()) { - log_legate.error("Invalid promotion on dimension %d for a %d-D store", extra_dim, dim()); - LEGATE_ABORT; + throw std::invalid_argument("Invalid promotion on dimension " + std::to_string(extra_dim) + + " for a " + std::to_string(dim()) + "-D store"); } - auto new_extents = extents_.insert(extra_dim, dim_size); + auto new_extents = extents().insert(extra_dim, dim_size); auto transform = transform_->push(std::make_unique(extra_dim, dim_size)); return std::make_shared(std::move(new_extents), storage_, std::move(transform)); } @@ -237,26 +246,26 @@ std::shared_ptr LogicalStore::promote(int32_t extra_dim, size_t di std::shared_ptr LogicalStore::project(int32_t d, int64_t index) const { if (d < 0 || d >= dim()) { - log_legate.error("Invalid projection on dimension %d for a %d-D store", d, dim()); - LEGATE_ABORT; - } else if (index < 0 || index >= extents_[d]) { - log_legate.error("Projection index %ld is out of bounds [0, %zd)", index, extents_[d]); + throw std::invalid_argument("Invalid projection on dimension " + std::to_string(d) + " for a " + + std::to_string(dim()) + "-D store"); + } else if (index < 0 || index >= extents()[d]) { + throw std::invalid_argument("Projection index " + std::to_string(index) + + " is out of bounds [0, " + std::to_string(extents()[d]) + ")"); } - auto new_extents = extents_.remove(d); + auto new_extents = extents().remove(d); auto transform = transform_->push(std::make_unique(d, index)); return std::make_shared(std::move(new_extents), storage_, std::move(transform)); } std::shared_ptr LogicalStore::partition_by_tiling(Shape tile_shape) { - if (tile_shape.size() != extents_.size()) { - log_legate.error("Incompatible tile shape: expected a %zd-tuple, got a %zd-tuple", - extents_.size(), - tile_shape.size()); - LEGATE_ABORT; + if (tile_shape.size() != extents().size()) { + throw std::invalid_argument("Incompatible tile shape: expected a " + + std::to_string(extents().size()) + "-tuple, got a " + + std::to_string(tile_shape.size()) + "-tuple"); } - Shape color_shape(extents_); + Shape color_shape(extents()); // TODO: This better use std::transform for (size_t idx = 0; idx < tile_shape.size(); ++idx) color_shape[idx] = (color_shape[idx] + tile_shape[idx] - 1) / tile_shape[idx]; @@ -264,31 +273,93 @@ std::shared_ptr LogicalStore::partition_by_tiling(Shape t return create_partition(std::move(partition)); } -std::shared_ptr LogicalStore::slice(int32_t dim, std::slice sl) const +std::shared_ptr LogicalStore::slice(int32_t idx, std::slice sl) const { - log_legate.error("Slice not implemented"); - return nullptr; + if (idx < 0 || idx >= dim()) { + throw std::invalid_argument("Invalid slicing of dimension " + std::to_string(idx) + " for a " + + std::to_string(dim()) + "-D store"); + } + + auto start = sl.start(); + auto stop = sl.size(); + auto step = sl.stride(); + auto size = extents()[idx]; + if (start < 0) { start = start + size; } + if (stop < 0) { start = start + size; } + + if (step != 1) { + throw std::invalid_argument("Unsupported slicing step: " + std::to_string(step)); + } else if (start >= size || stop > size) { + throw std::invalid_argument("Out-of-bounds slicing on dimension " + std::to_string(idx) + + " for a store"); + } + + auto old_extents = extents(); + auto new_extents = Shape(); + for (int i = 0; i < idx; i++) { new_extents.append_inplace(old_extents[i]); } + new_extents.append_inplace(stop - start); + for (int i = idx + 1; i < old_extents.size(); i++) { new_extents.append_inplace(old_extents[i]); } + + auto transform = + (start == 0) ? transform_ : transform_->push(std::make_unique(idx, -start)); + return std::make_shared(std::move(new_extents), storage_, std::move(transform)); } std::shared_ptr LogicalStore::transpose(std::vector&& axes) const { - log_legate.error("Transpose not implemented"); - return nullptr; + if (axes.size() != dim()) { + throw std::invalid_argument("Dimension Mismatch: expected " + std::to_string(dim()) + + " axes, but got " + std::to_string(axes.size())); + } else if (axes.size() != (std::set(axes.begin(), axes.end())).size()) { + throw std::invalid_argument("Duplicate axes found"); + } + + for (int i = 0; i < axes.size(); i++) { + if (axes[i] < 0 || axes[i] >= dim()) { + throw std::invalid_argument("Invalid axis " + std::to_string(axes[i]) + " for a " + + std::to_string(dim()) + "-D store"); + } + } + + auto old_extents = extents(); + auto new_extents = Shape(); + for (int i = 0; i < axes.size(); i++) { new_extents.append_inplace(old_extents[axes[i]]); } + + auto transform = transform_->push(std::make_unique(std::move(axes))); + return std::make_shared(std::move(new_extents), storage_, std::move(transform)); } -std::shared_ptr LogicalStore::delinearize(int32_t dim, +std::shared_ptr LogicalStore::delinearize(int32_t idx, std::vector&& sizes) const { - log_legate.error("Delinearize not implemented"); - return nullptr; + if (idx < 0 || idx >= dim()) { + throw std::invalid_argument("Invalid delinearization on dimension " + std::to_string(idx) + + " for a " + std::to_string(dim()) + "-D store"); + } + + auto old_shape = extents(); + int64_t volume = 1; + for (int i = 0; i < sizes.size(); i++) { volume *= sizes[i]; } + + if (old_shape[idx] != volume) { + throw std::invalid_argument("Dimension of size " + std::to_string(old_shape[idx]) + + " cannot be delinearized into shape with volume " + + std::to_string(volume)); + } + + auto old_extents = extents(); + auto new_extents = Shape(); + for (int i = 0; i < idx; i++) { new_extents.append_inplace(old_extents[i]); } + for (int i = 0; i < sizes.size(); i++) { new_extents.append_inplace(sizes[i]); } + for (int i = idx + 1; i < old_extents.size(); i++) { new_extents.append_inplace(old_extents[i]); } + + auto transform = transform_->push(std::make_unique(idx, std::move(sizes))); + return std::make_shared(std::move(new_extents), storage_, std::move(transform)); } std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) { - if (unbound()) { - log_legate.error("Unbound store cannot be inlined mapped"); - LEGATE_ABORT; - } + if (unbound()) { throw std::invalid_argument("Unbound store cannot be inlined mapped"); } if (nullptr != mapped_) return mapped_; if (storage_->kind() == Storage::Kind::FUTURE) { // TODO: future wrappers from inline mappings are read-only for now @@ -378,10 +449,7 @@ void LogicalStore::reset_key_partition() { storage_->reset_key_partition(); } std::shared_ptr LogicalStore::create_partition( std::shared_ptr partition) { - if (unbound()) { - log_legate.error("Unbound store cannot be manually partitioned"); - LEGATE_ABORT; - } + if (unbound()) { throw std::invalid_argument("Unbound store cannot be manually partitioned"); } // TODO: the partition here should be inverted by the transform auto storage_partition = storage_->create_partition(partition); return std::make_shared(std::move(storage_partition), shared_from_this()); @@ -399,7 +467,7 @@ void LogicalStore::pack(BufferBuilder& buffer) const std::string LogicalStore::to_string() const { std::stringstream ss; - ss << "Store(" << store_id_ << ") {shape: " << extents_; + ss << "Store(" << store_id_ << ") {shape: " << extents(); if (!transform_->identity()) ss << ", transform: " << *transform_ << "}"; else diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 0c38b4a1b9..9fddf14f22 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -739,14 +739,19 @@ std::unique_ptr Runtime::create_task(LibraryContext* library, return std::unique_ptr(task); } +void Runtime::flush_scheduling_window() +{ + if (operations_.size() == 0) return; + + std::vector> to_schedule; + to_schedule.swap(operations_); + schedule(std::move(to_schedule)); +} + void Runtime::submit(std::unique_ptr op) { operations_.push_back(std::move(op)); - if (operations_.size() >= window_size_) { - std::vector> to_schedule; - to_schedule.swap(operations_); - schedule(std::move(to_schedule)); - } + if (operations_.size() >= window_size_) { flush_scheduling_window(); } } void Runtime::schedule(std::vector> operations) diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index f8ce43e1da..5a2a25b1a5 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -184,6 +184,7 @@ class Runtime { std::unique_ptr create_task(LibraryContext* library, int64_t task_id, const Shape& launch_shape); + void flush_scheduling_window(); void submit(std::unique_ptr op); public: diff --git a/tests/cpp/unit/store.cc b/tests/cpp/unit/store.cc index d7cac58d8a..7572a4c0aa 100644 --- a/tests/cpp/unit/store.cc +++ b/tests/cpp/unit/store.cc @@ -58,7 +58,7 @@ TEST(Store, Transform) EXPECT_EQ(transposed.extents(), (std::vector{3, 4})); EXPECT_TRUE(transposed.transformed()); - auto delinearized = store.delinearize(0, {2, 2}); + auto delinearized = store.delinearize(0, (std::vector{2, 2})); EXPECT_EQ(delinearized.extents(), (std::vector{2, 2, 3})); EXPECT_TRUE(delinearized.transformed()); } From b3cb68f03fe1431a396c6d61464f742b3c21c573 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Sun, 14 May 2023 17:11:38 -0700 Subject: [PATCH 0118/1425] Extents should be set before to_string is invoked * Extents should be set before to_string is invoked See merge request legate/legate.core.internal!23 --- src/core/data/logical_store_detail.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/data/logical_store_detail.cc b/src/core/data/logical_store_detail.cc index e5b32ca610..361163fbd6 100644 --- a/src/core/data/logical_store_detail.cc +++ b/src/core/data/logical_store_detail.cc @@ -154,12 +154,12 @@ LogicalStore::LogicalStore(std::shared_ptr&& storage) storage_(std::forward(storage)), transform_(std::make_shared()) { + if (!unbound()) extents_ = storage_->extents(); #ifdef DEBUG_LEGATE assert(transform_ != nullptr); log_legate.debug() << "Create " << to_string(); #endif - if (!unbound()) extents_ = storage_->extents(); } LogicalStore::LogicalStore(Shape&& extents, From efb0c0c61317a7edbf16a934027ba9833813e391 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Sun, 14 May 2023 17:22:06 -0700 Subject: [PATCH 0119/1425] Support for dynamic task id * Support for dynamic task id See merge request legate/legate.core.internal!24 --- src/core/runtime/context.h | 3 +++ src/core/runtime/resource.h | 6 ++++++ src/core/task/task.h | 18 +++++++++++++++++- src/core/task/task.inl | 11 ++++++++++- 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/core/runtime/context.h b/src/core/runtime/context.h index 4f56a55cf9..6aa25aca89 100644 --- a/src/core/runtime/context.h +++ b/src/core/runtime/context.h @@ -113,6 +113,9 @@ class LibraryContext { bool valid_projection_id(Legion::ProjectionID proj_id) const; bool valid_sharding_id(Legion::ShardingID shard_id) const; + public: + int64_t get_new_task_id() { return task_scope_.generate_id(); } + public: void record_task_name(int64_t local_task_id, const std::string& task_name); /** diff --git a/src/core/runtime/resource.h b/src/core/runtime/resource.h index 5025b33780..c9fb560b20 100644 --- a/src/core/runtime/resource.h +++ b/src/core/runtime/resource.h @@ -50,6 +50,11 @@ class ResourceIdScope { assert(in_scope(resource_id)); return resource_id - base_; } + int64_t generate_id() + { + if (next_ == size_) throw std::overflow_error("The scope ran out of IDs"); + return next_++; + } public: bool valid() const { return base_ != -1; } @@ -62,6 +67,7 @@ class ResourceIdScope { private: int64_t base_{-1}; int64_t size_{-1}; + int64_t next_{0}; }; } // namespace legate diff --git a/src/core/task/task.h b/src/core/task/task.h index 9d6a635ba8..7660ffbffd 100644 --- a/src/core/task/task.h +++ b/src/core/task/task.h @@ -73,7 +73,7 @@ struct LegateTask { * @brief Registers all variants of this task immediately. * * Unlike the other method, this one takes a library context so the registration can be done - * immediately. + * immediately. The value of T::TASK_ID is used as the task id. * * @param context Library to which the task should be registered * @param all_options Options for task variants. Variants with no entires in `all_options` will @@ -82,6 +82,22 @@ struct LegateTask { static void register_variants( LibraryContext* context, const std::map& all_options = {}); + /** + * @brief Registers all variants of this task immediately. + * + * Unlike the other method, this one takes a library context so the registration can be done + * immediately. + * + * @param context Library to which the task should be registered + * @param task_id Task id + * @param all_options Options for task variants. Variants with no entires in `all_options` will + * use the default set of options + */ + static void register_variants( + LibraryContext* context, + int64_t task_id, + const std::map& all_options = {}); + private: template typename, bool> friend struct detail::VariantHelper; diff --git a/src/core/task/task.inl b/src/core/task/task.inl index 29927bceec..34cc9f89ea 100644 --- a/src/core/task/task.inl +++ b/src/core/task/task.inl @@ -48,9 +48,18 @@ template template /*static*/ void LegateTask::register_variants( LibraryContext* context, const std::map& all_options) +{ + register_variants(context, T::TASK_ID, all_options); +} + +template +/*static*/ void LegateTask::register_variants( + LibraryContext* context, + int64_t task_id, + const std::map& all_options) { auto task_info = create_task_info(all_options); - context->register_task(T::TASK_ID, std::move(task_info)); + context->register_task(task_id, std::move(task_info)); } template From e8a2119a4bf30b5f8677ec734f951aca1251e728 Mon Sep 17 00:00:00 2001 From: Irina Demeshko Date: Mon, 15 May 2023 16:29:02 -0700 Subject: [PATCH 0120/1425] Preliminary resource scoping in C++ --- src/core/mapping/machine.cc | 152 +++++++++++-- src/core/mapping/machine.h | 33 ++- src/core/runtime/runtime.cc | 88 ++++++- src/core/runtime/runtime.h | 30 ++- tests/cpp/integration/machine_scope.cc | 150 ++++++++++++ tests/cpp/unit/machine.cc | 303 +++++++++++++++++++++++++ 6 files changed, 733 insertions(+), 23 deletions(-) create mode 100644 tests/cpp/integration/machine_scope.cc create mode 100644 tests/cpp/unit/machine.cc diff --git a/src/core/mapping/machine.cc b/src/core/mapping/machine.cc index 4c47b3c85a..d62a431f3e 100644 --- a/src/core/mapping/machine.cc +++ b/src/core/mapping/machine.cc @@ -44,6 +44,18 @@ Processor::Kind to_kind(TaskTarget target) return Processor::Kind::LOC_PROC; } +LegateVariantCode to_task_variant(TaskTarget target) +{ + switch (target) { + case TaskTarget::GPU: return LEGATE_GPU_VARIANT; + case TaskTarget::OMP: return LEGATE_OMP_VARIANT; + case TaskTarget::CPU: return LEGATE_CPU_VARIANT; + default: LEGATE_ABORT; + } + assert(false); + return LEGATE_CPU_VARIANT; +} + std::ostream& operator<<(std::ostream& stream, const TaskTarget& target) { switch (target) { @@ -63,6 +75,10 @@ std::ostream& operator<<(std::ostream& stream, const TaskTarget& target) return stream; } +///////////////////////////////////// +// legate::mapping::ProcessorRange +///////////////////////////////////// + ProcessorRange::ProcessorRange(uint32_t _low, uint32_t _high, uint32_t _per_node_count) : low(_low), high(_high), per_node_count(_per_node_count) { @@ -80,6 +96,13 @@ ProcessorRange ProcessorRange::operator&(const ProcessorRange& other) const return ProcessorRange(std::max(low, other.low), std::min(high, other.high), per_node_count); } +bool ProcessorRange::operator==(const ProcessorRange& other) const +{ + return other.low == low && other.high == high && other.per_node_count == per_node_count; +} + +bool ProcessorRange::operator!=(const ProcessorRange& other) const { return !operator==(other); } + uint32_t ProcessorRange::count() const { return high - low; } bool ProcessorRange::empty() const { return high <= low; } @@ -91,6 +114,19 @@ std::string ProcessorRange::to_string() const return ss.str(); } +std::pair ProcessorRange::get_node_range() const +{ + if (empty()) throw std::runtime_error("Illegal to get a node range of an empty processor range"); + return std::make_pair(low / per_node_count, high / per_node_count); +} + +ProcessorRange ProcessorRange::slice(const uint32_t& from, const uint32_t& to) const +{ + uint32_t new_low = std::min(low + from, high); + uint32_t new_high = std::min(low + to, high); + return ProcessorRange(new_low, new_high, per_node_count); +} + void ProcessorRange::pack(BufferBuilder& buffer) const { buffer.pack(low); @@ -98,6 +134,10 @@ void ProcessorRange::pack(BufferBuilder& buffer) const buffer.pack(per_node_count); } +/////////////////////////////////////////// +// legate::mapping::MachineDesc +////////////////////////////////////////// + MachineDesc::MachineDesc(const std::map& ranges) : processor_ranges(ranges) { @@ -108,22 +148,16 @@ MachineDesc::MachineDesc(const std::map& ranges) } } -void MachineDesc::pack(BufferBuilder& buffer) const +const ProcessorRange& MachineDesc::processor_range() const { - buffer.pack(static_cast(preferred_target)); - buffer.pack(processor_ranges.size()); - for (auto& [target, processor_range] : processor_ranges) { - buffer.pack(static_cast(target)); - processor_range.pack(buffer); - } + return processor_range(preferred_target); } +static const ProcessorRange empty_processor_range = {0, 0, 1}; -ProcessorRange MachineDesc::processor_range() const +const ProcessorRange& MachineDesc::processor_range(const TaskTarget& target) const { - auto finder = processor_ranges.find(preferred_target); -#ifdef DEBUG_LEGATE - assert(finder != processor_ranges.end()); -#endif + auto finder = processor_ranges.find(target); + if (finder == processor_ranges.end()) { return empty_processor_range; } return finder->second; } @@ -134,7 +168,8 @@ std::vector MachineDesc::valid_targets() const return std::move(result); } -std::vector MachineDesc::valid_targets_except(std::set&& to_exclude) const +std::vector MachineDesc::valid_targets_except( + const std::set& to_exclude) const { std::vector result; for (auto& [target, _] : processor_ranges) @@ -142,15 +177,101 @@ std::vector MachineDesc::valid_targets_except(std::set&& return std::move(result); } +size_t MachineDesc::count() const { return count(preferred_target); } + +size_t MachineDesc::count(const TaskTarget& target) const +{ + auto finder = processor_ranges.find(target); + if (finder == processor_ranges.end()) { + throw std::runtime_error( + "There is no requested target in the MachineDesc or MachineDesc is empty"); + } + return finder->second.count(); +} + std::string MachineDesc::to_string() const { std::stringstream ss; - ss << "Machine(preferred_kind: " << preferred_target; + ss << "Machine(preferred_target: " << preferred_target; for (auto& [kind, range] : processor_ranges) ss << ", " << kind << ": " << range.to_string(); ss << ")"; return ss.str(); } +void MachineDesc::pack(BufferBuilder& buffer) const +{ + buffer.pack(static_cast(preferred_target)); + buffer.pack(processor_ranges.size()); + for (auto& [target, processor_range] : processor_ranges) { + buffer.pack(static_cast(target)); + processor_range.pack(buffer); + } +} + +MachineDesc MachineDesc::only(const TaskTarget& target) const { return only(std::set({target})); } + +MachineDesc MachineDesc::only(const std::set& targets) const +{ + std::map new_processor_ranges; + for (auto t : targets) new_processor_ranges.insert({t, processor_range(t)}); + + return MachineDesc(new_processor_ranges); +} + +MachineDesc MachineDesc::slice(const uint32_t& from, + const uint32_t& to, + const TaskTarget& target) const +{ + return MachineDesc({{target, processor_range(target).slice(from, to)}}); +} + +MachineDesc MachineDesc::slice(const uint32_t& from, const uint32_t& to) const +{ + if (processor_ranges.size() > 1) + throw std::runtime_error( + "Ambiguous slicing: slicing is not allowed on a machine with more than one processor kind"); + + return slice(from, to, preferred_target); +} + +MachineDesc MachineDesc::operator[](const TaskTarget& target) const { return only(target); } + +bool MachineDesc::operator==(const MachineDesc& other) const +{ + if (preferred_target != other.preferred_target) return false; + if (processor_ranges.size() != other.processor_ranges.size()) return false; + for (auto const& r : processor_ranges) { + auto finder = other.processor_ranges.find(r.first); + if (finder == other.processor_ranges.end() || r.second != finder->second) return false; + } + return true; +} + +bool MachineDesc::operator!=(const MachineDesc& other) const { return !(*this == other); } + +MachineDesc MachineDesc::operator&(const MachineDesc& other) const +{ + std::map new_processor_ranges; + for (const auto& [target, range] : processor_ranges) { + auto finder = other.processor_ranges.find(target); + if (finder != other.processor_ranges.end()) { + new_processor_ranges[target] = finder->second & range; + } + } + return MachineDesc(new_processor_ranges); +} + +bool MachineDesc::empty() const +{ + for (const auto& r : processor_ranges) + if (!r.second.empty()) return false; + return true; +} + +//////////////////////////////////////// +// legate::mapping::LocalProcessorRange +///////////////////////////////////////// + LocalProcessorRange::LocalProcessorRange() : offset_(0), total_proc_count_(0), procs_() {} LocalProcessorRange::LocalProcessorRange(const std::vector& procs) @@ -175,6 +296,9 @@ const Processor& LocalProcessorRange::operator[](uint32_t idx) const return procs_[local_idx]; } +////////////////////////////// +// legate::mapping::Machine +////////////////////////////// Machine::Machine(Legion::Machine legion_machine) : local_node(Processor::get_executing_processor().address_space()), total_nodes(legion_machine.get_address_space_count()) diff --git a/src/core/mapping/machine.h b/src/core/mapping/machine.h index 34580b3657..1245271431 100644 --- a/src/core/mapping/machine.h +++ b/src/core/mapping/machine.h @@ -30,14 +30,19 @@ TaskTarget to_target(Processor::Kind kind); Processor::Kind to_kind(TaskTarget target); +LegateVariantCode to_task_variant(TaskTarget target); + struct ProcessorRange { ProcessorRange() {} ProcessorRange(uint32_t low, uint32_t high, uint32_t per_node_count); ProcessorRange operator&(const ProcessorRange&) const; + bool operator==(const ProcessorRange&) const; + bool operator!=(const ProcessorRange&) const; uint32_t count() const; bool empty() const; std::string to_string() const; - + std::pair get_node_range() const; + ProcessorRange slice(const uint32_t& lo, const uint32_t& hi) const; void pack(BufferBuilder& buffer) const; uint32_t low{0}; @@ -49,15 +54,33 @@ struct MachineDesc { MachineDesc() {} MachineDesc(const std::map& processor_ranges); - TaskTarget preferred_target{TaskTarget::CPU}; - std::map processor_ranges{}; + const ProcessorRange& processor_range() const; + const ProcessorRange& processor_range(const TaskTarget& target) const; - ProcessorRange processor_range() const; std::vector valid_targets() const; - std::vector valid_targets_except(std::set&& to_exclude) const; + std::vector valid_targets_except(const std::set& to_exclude) const; + + size_t count() const; + size_t count(const TaskTarget& target) const; + std::string to_string() const; void pack(BufferBuilder& buffer) const; + + MachineDesc only(const TaskTarget& target) const; + MachineDesc only(const std::set& target) const; + MachineDesc slice(const uint32_t& lo, const uint32_t& hi, const TaskTarget& target) const; + MachineDesc slice(const uint32_t& lo, const uint32_t& hi) const; + + MachineDesc operator[](const TaskTarget& target) const; + bool operator==(const MachineDesc& other) const; + bool operator!=(const MachineDesc& other) const; + MachineDesc operator&(const MachineDesc& other) const; + + bool empty() const; + + TaskTarget preferred_target{TaskTarget::CPU}; + std::map processor_ranges{}; }; std::ostream& operator<<(std::ostream& stream, const MachineDesc& info); diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 9fddf14f22..6af274ea27 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -617,6 +617,48 @@ void PartitionManager::record_index_partition(const Legion::IndexSpace& index_sp tiling_cache_[std::make_pair(index_space, tiling)] = index_partition; } +//////////////////////////////////////////// +// legate::MachineManager +//////////////////////////////////////////// +const mapping::MachineDesc& MachineManager::get_machine() const +{ +#ifdef DEBUG_LEGATE + assert(machines_.size() > 0); +#endif + return machines_.back(); +} + +void MachineManager::push_machine(const mapping::MachineDesc& m) { machines_.push_back(m); } + +void MachineManager::pop_machine() +{ + if (machines_.size() <= 1) throw std::underflow_error("can't pop from the empty machine stack"); + machines_.pop_back(); +} + +//////////////////////////////////////////// +// legate::MachineTracker +//////////////////////////////////////////// + +MachineTracker::MachineTracker(const mapping::MachineDesc& m) +{ + auto* runtime = Runtime::get_runtime(); + auto machine = m & Runtime::get_runtime()->get_machine(); + if (machine.count() == 0) throw std::runtime_error("Runtime can not use an empty machine"); + runtime->machine_manager()->push_machine(machine); +} + +MachineTracker::~MachineTracker() +{ + auto* runtime = Runtime::get_runtime(); + runtime->machine_manager()->pop_machine(); +} + +const mapping::MachineDesc& MachineTracker::get_current_machine() const +{ + return Runtime::get_runtime()->get_machine(); +} + //////////////////////////////////////////////////// // legate::Runtime //////////////////////////////////////////////////// @@ -719,14 +761,39 @@ void Runtime::post_startup_initialization(Legion::Context legion_context) legion_context_ = legion_context; core_context_ = find_library(core_library_name); partition_manager_ = new PartitionManager(this, core_context_); + machine_manager_ = new MachineManager(); provenance_manager_ = new ProvenanceManager(); Core::retrieve_tunable(legion_context_, legion_runtime_, core_context_); initialize_toplevel_machine(); } +mapping::MachineDesc Runtime::slice_machine_for_task(LibraryContext* library, int64_t task_id) +{ + auto task_info = library->find_task(task_id); +#ifdef DEBUG_LEGATE + assert(task_info != nullptr); +#endif + std::set task_targets; + auto& machine = machine_manager_->get_machine(); + for (const auto& t : machine.valid_targets()) { + auto variant = mapping::to_task_variant(t); + if (task_info->has_variant(variant)) task_targets.insert(t); + } + + auto new_machine = machine.only(task_targets); + if (new_machine.empty()) { + std::stringstream ss; + ss << "Task " << task_id << " " << task_info->name() << " does not have any valid variant for " + << "the current machine configuration."; + std::runtime_error(ss.str()); + } + return new_machine; +} + // This function should be moved to the library context std::unique_ptr Runtime::create_task(LibraryContext* library, int64_t task_id) { + MachineTracker track(slice_machine_for_task(library, task_id)); auto task = new AutoTask(library, task_id, next_unique_id_++); return std::unique_ptr(task); } @@ -735,6 +802,7 @@ std::unique_ptr Runtime::create_task(LibraryContext* library, int64_t task_id, const Shape& launch_shape) { + MachineTracker track(slice_machine_for_task(library, task_id)); auto task = new ManualTask(library, task_id, launch_shape, next_unique_id_++); return std::unique_ptr(task); } @@ -1014,15 +1082,19 @@ void Runtime::initialize_toplevel_machine() auto machine = new mapping::MachineDesc({{mapping::TaskTarget::GPU, create_range(num_gpus)}, {mapping::TaskTarget::OMP, create_range(num_omps)}, {mapping::TaskTarget::CPU, create_range(num_cpus)}}); - machine_.reset(machine); +#ifdef DEBUG_LEGATE + assert(machine_manager_ != nullptr); +#endif + + machine_manager_->push_machine(*machine); } const mapping::MachineDesc& Runtime::get_machine() const { #ifdef DEBUG_LEGATE - assert(machine_ != nullptr); + assert(machine_manager_ != nullptr); #endif - return *machine_; + return machine_manager_->get_machine(); } Legion::ProjectionID Runtime::get_projection(int32_t src_ndim, const proj::SymbolicPoint& point) @@ -1071,6 +1143,8 @@ Legion::ProjectionID Runtime::get_delinearizing_projection() return core_context_->get_projection_id(LEGATE_CORE_DELINEARIZE_PROJ_ID); } +MachineManager* Runtime::machine_manager() const { return machine_manager_; } + /*static*/ void Runtime::initialize(int32_t argc, char** argv) { Legion::Runtime::initialize(&argc, &argv, true /*filter legion and realm args*/); @@ -1124,6 +1198,10 @@ int32_t start(int32_t argc, char** argv) { return Runtime::start(argc, argv); } int32_t wait_for_shutdown() { return Runtime::get_runtime()->wait_for_shutdown(); } +////////////////////////////////////////////// +// ProvenanceManager +////////////////////////////////////////////// + ProvenanceManager::ProvenanceManager() { provenance_.push_back(EMPTY_STRING); } const std::string& ProvenanceManager::get_provenance() @@ -1166,6 +1244,10 @@ void ProvenanceManager::clear_all() provenance_.push_back(EMPTY_STRING); } +////////////////////////////////////// +// ProvenanceTracker +////////////////////////////////////// + ProvenanceTracker::ProvenanceTracker(const std::string& p) { auto* runtime = Runtime::get_runtime(); diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 5a2a25b1a5..c804e5448a 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -26,6 +26,7 @@ #include "core/data/shape.h" #include "core/data/store.h" #include "core/legate_c.h" +#include "core/mapping/machine.h" #include "core/runtime/resource.h" #include "core/task/exception.h" #include "core/type/type_info.h" @@ -147,6 +148,29 @@ class PartitionManager { std::map tiling_cache_; }; +class MachineManager { + public: + MachineManager(){}; + + public: + const mapping::MachineDesc& get_machine() const; + + void push_machine(const mapping::MachineDesc& m); + + void pop_machine(); + + private: + std::vector machines_; +}; + +struct MachineTracker { + MachineTracker(const mapping::MachineDesc& m); + + ~MachineTracker(); + + const mapping::MachineDesc& get_current_machine() const; +}; + class Runtime { public: Runtime(Legion::Runtime* legion_runtime); @@ -180,6 +204,7 @@ class Runtime { T get_tunable(Legion::MapperID mapper_id, int64_t tunable_id); public: + mapping::MachineDesc slice_machine_for_task(LibraryContext* library, int64_t task_id); std::unique_ptr create_task(LibraryContext* library, int64_t task_id); std::unique_ptr create_task(LibraryContext* library, int64_t task_id, @@ -309,7 +334,10 @@ class Runtime { std::map, int32_t> reduction_ops_{}; private: - std::unique_ptr machine_{nullptr}; + MachineManager* machine_manager_{nullptr}; + + public: + MachineManager* machine_manager() const; }; void initialize(int32_t argc, char** argv); diff --git a/tests/cpp/integration/machine_scope.cc b/tests/cpp/integration/machine_scope.cc new file mode 100644 index 0000000000..b07599aeef --- /dev/null +++ b/tests/cpp/integration/machine_scope.cc @@ -0,0 +1,150 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "core/mapping/mapping.h" +#include "legate.h" + +namespace machine_scope { + +static const char* library_name = "machine_scope"; +static legate::Logger logger(library_name); + +enum TaskIDs { + MULTI_VARIANT = 0, + CPU_VARIANT = 1, +}; + +void validate(legate::TaskContext& context) +{ + if (context.is_single_task()) return; + + int32_t num_tasks = context.get_launch_domain().get_volume(); + auto to_compare = context.scalars().at(0).value(); + assert(to_compare == num_tasks); +} + +struct MultiVariantTask : public legate::LegateTask { + static const int32_t TASK_ID = MULTI_VARIANT; + + static void cpu_variant(legate::TaskContext& context) { validate(context); } +#ifdef LEGATE_USE_OPENMP + static void omp_variant(legate::TaskContext& context) { validate(context); } +#endif +#ifdef LEGATE_USE_CUDA + static void gpu_variant(legate::TaskContext& context) { validate(context); } +#endif +}; + +struct CpuVariantOnlyTask : public legate::LegateTask { + static const int32_t TASK_ID = CPU_VARIANT; + static void cpu_variant(legate::TaskContext& context) { validate(context); } +}; + +void register_tasks() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + MultiVariantTask::register_variants(context); + CpuVariantOnlyTask::register_variants(context); +} + +void test_scoping(legate::LibraryContext* context) +{ + auto runtime = legate::Runtime::get_runtime(); + auto store = runtime->create_store({5, 5}, legate::int64()); + auto machine = runtime->get_machine(); + auto task = runtime->create_task(context, MULTI_VARIANT); + auto part = task->declare_partition(); + task->add_output(store, part); + task->add_scalar_arg(machine.count()); + runtime->submit(std::move(task)); + + if (machine.count(legate::mapping::TaskTarget::CPU) > 0) { + legate::MachineTracker tracker(machine.only(legate::mapping::TaskTarget::CPU)); + auto task_scoped = runtime->create_task(context, MULTI_VARIANT); + auto part_scoped = task_scoped->declare_partition(); + task_scoped->add_output(store, part_scoped); + task_scoped->add_scalar_arg(machine.count(legate::mapping::TaskTarget::CPU)); + runtime->submit(std::move(task_scoped)); + } + + if (machine.count(legate::mapping::TaskTarget::OMP) > 0) { + legate::MachineTracker tracker(machine.only(legate::mapping::TaskTarget::OMP)); + auto task_scoped = runtime->create_task(context, MULTI_VARIANT); + auto part_scoped = task_scoped->declare_partition(); + task_scoped->add_output(store, part_scoped); + task_scoped->add_scalar_arg(machine.count(legate::mapping::TaskTarget::OMP)); + runtime->submit(std::move(task_scoped)); + } + + if (machine.count(legate::mapping::TaskTarget::GPU) > 0) { + legate::MachineTracker tracker(machine.only(legate::mapping::TaskTarget::GPU)); + auto task_scoped = runtime->create_task(context, MULTI_VARIANT); + auto part_scoped = task_scoped->declare_partition(); + task_scoped->add_output(store, part_scoped); + task_scoped->add_scalar_arg(machine.count(legate::mapping::TaskTarget::GPU)); + runtime->submit(std::move(task_scoped)); + } +} + +void test_cpu_only(legate::LibraryContext* context) +{ + auto runtime = legate::Runtime::get_runtime(); + auto store = runtime->create_store({5, 5}, legate::int64()); + auto machine = runtime->get_machine(); + auto task = runtime->create_task(context, CPU_VARIANT); + auto part = task->declare_partition(); + task->add_output(store, part); + task->add_scalar_arg(machine.count()); + runtime->submit(std::move(task)); + + if (machine.count(legate::mapping::TaskTarget::CPU) > 0) { + legate::MachineTracker tracker(machine.only(legate::mapping::TaskTarget::CPU)); + auto task_scoped = runtime->create_task(context, CPU_VARIANT); + auto part_scoped = task_scoped->declare_partition(); + task_scoped->add_output(store, part_scoped); + task_scoped->add_scalar_arg(machine.count(legate::mapping::TaskTarget::CPU)); + runtime->submit(std::move(task_scoped)); + } + + // checking an empty machine + { + EXPECT_THROW( + legate::MachineTracker(machine.only(legate::mapping::TaskTarget::CPU).slice(15, 19)), + std::runtime_error); + } + + // check `slice_machine_for_task` logic + if (machine.count(legate::mapping::TaskTarget::GPU) > 0) { + legate::MachineTracker tracker(machine.only(legate::mapping::TaskTarget::GPU)); + EXPECT_THROW(runtime->create_task(context, CPU_VARIANT), std::runtime_error); + } +} + +TEST(Integration, MachineScope) +{ + legate::Core::perform_registration(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + test_scoping(context); + test_cpu_only(context); +} + +} // namespace machine_scope diff --git a/tests/cpp/unit/machine.cc b/tests/cpp/unit/machine.cc new file mode 100644 index 0000000000..69ea5e3a78 --- /dev/null +++ b/tests/cpp/unit/machine.cc @@ -0,0 +1,303 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include +#include +#include +#include + +#include "core/utilities/buffer_builder.h" +#include "legate.h" + +namespace unit { + +TEST(Machine, ProcessorRange) +{ + // create nonempty + { + legate::mapping::ProcessorRange range(1, 3, 1); + EXPECT_FALSE(range.empty()); + EXPECT_EQ(range.per_node_count, 1); + EXPECT_EQ(range.low, 1); + EXPECT_EQ(range.high, 3); + EXPECT_EQ(range.count(), 2); + EXPECT_EQ(range.get_node_range(), std::make_pair(uint32_t(1), uint32_t(3))); + } + + // create empty + { + legate::mapping::ProcessorRange range(1, 0, 1); + EXPECT_TRUE(range.empty()); + EXPECT_EQ(range.per_node_count, 1); + EXPECT_EQ(range.low, 0); + EXPECT_EQ(range.high, 0); + EXPECT_EQ(range.count(), 0); + EXPECT_THROW(range.get_node_range(), std::runtime_error); + } + + // check defaults + { + legate::mapping::ProcessorRange range; + EXPECT_TRUE(range.empty()); + EXPECT_EQ(range.per_node_count, 1); + EXPECT_EQ(range.low, 0); + EXPECT_EQ(range.high, 0); + EXPECT_EQ(range.count(), 0); + } + + // get_node_range + { + legate::mapping::ProcessorRange range(0, 7, 2); + EXPECT_EQ(range.get_node_range(), std::make_pair(uint32_t(0), uint32_t(3))); + } + + // intersection nonempty + { + legate::mapping::ProcessorRange range1(0, 3, 1); + legate::mapping::ProcessorRange range2(2, 4, 1); + auto range3 = range1 & range2; + EXPECT_EQ(range3, legate::mapping::ProcessorRange(2, 3, 1)); + } + + // intersection empty + { + legate::mapping::ProcessorRange range1(0, 2, 1); + legate::mapping::ProcessorRange range2(3, 5, 1); + auto range3 = range1 & range2; + EXPECT_EQ(range3, legate::mapping::ProcessorRange(0, 0, 1)); + EXPECT_EQ(range3.count(), 0); + } + + // empty slice empty range + { + legate::mapping::ProcessorRange range(3, 1, 1); + EXPECT_EQ(range.slice(0, 0).count(), 0); + EXPECT_EQ(range.slice(4, 6).count(), 0); + } + + // empty slice nonempty range + { + legate::mapping::ProcessorRange range(1, 3, 1); + EXPECT_EQ(range.slice(0, 0).count(), 0); + EXPECT_EQ(range.slice(4, 6).count(), 0); + } + + // nonempty slice nonempty range + { + legate::mapping::ProcessorRange range(1, 3, 1); + EXPECT_EQ(range.slice(1, 3).count(), 1); + EXPECT_EQ(range.slice(0, 2).count(), 2); + } +} + +TEST(Machine, MachineDesc) +{ + // test empty MachineDesc + { + legate::mapping::MachineDesc machine; + EXPECT_EQ(machine.preferred_target, legate::mapping::TaskTarget::CPU); + EXPECT_THROW(machine.count(), std::runtime_error); + EXPECT_THROW(machine.count(legate::mapping::TaskTarget::GPU), std::runtime_error); + EXPECT_EQ(machine.processor_range(), legate::mapping::ProcessorRange(0, 0, 1)); + EXPECT_EQ(machine.processor_range(legate::mapping::TaskTarget::GPU), + legate::mapping::ProcessorRange(0, 0, 1)); + EXPECT_EQ(machine.slice(0, 1), + legate::mapping::MachineDesc( + {{legate::mapping::TaskTarget::CPU, legate::mapping::ProcessorRange()}})); + } + + legate::mapping::ProcessorRange cpu_range(1, 3, 4); + legate::mapping::ProcessorRange omp_range(0, 3, 2); + legate::mapping::ProcessorRange gpu_range(3, 6, 3); + legate::mapping::ProcessorRange empty_range; + + // test equal + { + legate::mapping::MachineDesc machine1({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::OMP, omp_range}}); + legate::mapping::MachineDesc machine2({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::OMP, omp_range}}); + EXPECT_EQ(machine1, machine2); + + legate::mapping::MachineDesc machine3; + EXPECT_NE(machine1, machine3); + } + + // test preferred_target + { + legate::mapping::MachineDesc machine1({{legate::mapping::TaskTarget::CPU, cpu_range}}); + EXPECT_EQ(machine1.preferred_target, legate::mapping::TaskTarget::CPU); + + legate::mapping::MachineDesc machine2({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::OMP, omp_range}}); + EXPECT_EQ(machine2.preferred_target, legate::mapping::TaskTarget::OMP); + + legate::mapping::MachineDesc machine3({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::OMP, omp_range}, + {legate::mapping::TaskTarget::GPU, gpu_range}}); + EXPECT_EQ(machine3.preferred_target, legate::mapping::TaskTarget::GPU); + } + + // test processor_range + { + legate::mapping::MachineDesc machine1({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::OMP, omp_range}, + {legate::mapping::TaskTarget::GPU, gpu_range}}); + EXPECT_EQ(machine1.processor_range(), gpu_range); + EXPECT_EQ(machine1.processor_range(legate::mapping::TaskTarget::CPU), cpu_range); + EXPECT_EQ(machine1.processor_range(legate::mapping::TaskTarget::OMP), omp_range); + EXPECT_EQ(machine1.processor_range(legate::mapping::TaskTarget::GPU), gpu_range); + + legate::mapping::MachineDesc machine2({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::OMP, omp_range}}); + EXPECT_EQ(machine2.processor_range(), omp_range); + EXPECT_EQ(machine2.processor_range(legate::mapping::TaskTarget::CPU), cpu_range); + EXPECT_EQ(machine2.processor_range(legate::mapping::TaskTarget::GPU), + legate::mapping::ProcessorRange()); + } + + // test valid_targets + { + legate::mapping::MachineDesc machine1({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::OMP, omp_range}, + {legate::mapping::TaskTarget::GPU, gpu_range}}); + auto valid_targets1 = machine1.valid_targets(); + EXPECT_EQ(valid_targets1.size(), 3); + + legate::mapping::MachineDesc machine2({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::OMP, omp_range}}); + auto valid_targets2 = machine2.valid_targets(); + EXPECT_EQ(valid_targets2.size(), 2); + } + + // test valid_targets_except + { + legate::mapping::MachineDesc machine({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::OMP, omp_range}, + {legate::mapping::TaskTarget::GPU, gpu_range}}); + + std::set exclude_targets; + auto valid_targets1 = machine.valid_targets_except(exclude_targets); + EXPECT_EQ(valid_targets1.size(), 3); + + exclude_targets.insert(legate::mapping::TaskTarget::CPU); + auto valid_targets2 = machine.valid_targets_except(exclude_targets); + EXPECT_EQ(valid_targets2.size(), 2); + + exclude_targets.insert(legate::mapping::TaskTarget::OMP); + auto valid_targets3 = machine.valid_targets_except(exclude_targets); + EXPECT_EQ(valid_targets3.size(), 1); + + exclude_targets.insert(legate::mapping::TaskTarget::GPU); + auto valid_targets4 = machine.valid_targets_except(exclude_targets); + EXPECT_EQ(valid_targets4.size(), 0); + } + + // test count + { + legate::mapping::MachineDesc machine({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::OMP, omp_range}, + {legate::mapping::TaskTarget::GPU, gpu_range}}); + EXPECT_EQ(machine.count(), 3); + EXPECT_EQ(machine.count(legate::mapping::TaskTarget::CPU), 2); + EXPECT_EQ(machine.count(legate::mapping::TaskTarget::OMP), 3); + } + + // test_pack + { + legate::BufferBuilder buf; + legate::mapping::MachineDesc machine({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::GPU, gpu_range}}); + machine.pack(buf); + auto legion_buffer = buf.to_legion_buffer(); + legate::BaseDeserializer dez(legion_buffer.get_ptr(), + legion_buffer.get_size()); + auto machine_unpack = dez.unpack(); + EXPECT_EQ(machine_unpack, machine); + } + + // test only + { + legate::mapping::MachineDesc machine({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::GPU, gpu_range}}); + + auto machine1 = machine.only(legate::mapping::TaskTarget::CPU); + auto valid_targets1 = machine1.valid_targets(); + EXPECT_EQ(valid_targets1.size(), 1); + EXPECT_EQ(machine1.count(), 2); + EXPECT_EQ(machine1.preferred_target, legate::mapping::TaskTarget::CPU); + EXPECT_EQ(machine1.processor_range().per_node_count, 4); + + legate::mapping::MachineDesc machine2({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::OMP, omp_range}, + {legate::mapping::TaskTarget::GPU, gpu_range}}); + auto machine3 = + machine2.only({legate::mapping::TaskTarget::GPU, legate::mapping::TaskTarget::CPU}); + EXPECT_EQ(machine3, machine); + } + + // test slice + { + legate::mapping::MachineDesc machine1({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::GPU, gpu_range}}); + EXPECT_THROW(machine1.slice(0, 1), std::runtime_error); + + auto new_machine1 = machine1.slice(0, 2, legate::mapping::TaskTarget::GPU); + EXPECT_EQ(new_machine1.preferred_target, legate::mapping::TaskTarget::GPU); + EXPECT_EQ(new_machine1.processor_range().count(), 2); + + legate::mapping::MachineDesc machine2({{legate::mapping::TaskTarget::GPU, gpu_range}}); + auto new_machine2 = machine2.slice(0, 2); + EXPECT_EQ(new_machine2.preferred_target, legate::mapping::TaskTarget::GPU); + EXPECT_EQ(new_machine2.processor_range().count(), 2); + } + + // test operator[] + { + legate::mapping::MachineDesc machine({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::GPU, gpu_range}}); + + auto machine1 = machine[legate::mapping::TaskTarget::GPU]; + auto valid_targets1 = machine1.valid_targets(); + EXPECT_EQ(valid_targets1.size(), 1); + EXPECT_EQ(machine1.count(), 3); + EXPECT_EQ(machine1.preferred_target, legate::mapping::TaskTarget::GPU); + EXPECT_EQ(machine1.processor_range().per_node_count, 3); + } + + // test intersection + { + legate::mapping::MachineDesc machine1( + {{legate::mapping::TaskTarget::CPU, cpu_range.slice(1, 2)}, + {legate::mapping::TaskTarget::GPU, gpu_range}}); + + legate::mapping::MachineDesc machine2( + {{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::OMP, omp_range}, + {legate::mapping::TaskTarget::GPU, gpu_range.slice(0, 1)}}); + + legate::mapping::MachineDesc machine3( + {{legate::mapping::TaskTarget::CPU, cpu_range.slice(1, 2)}, + {legate::mapping::TaskTarget::GPU, gpu_range.slice(0, 1)}}); + + EXPECT_EQ(machine3, (machine1 & machine2)); + EXPECT_EQ(machine2.only(legate::mapping::TaskTarget::OMP) & machine1, + legate::mapping::MachineDesc()); + } +} + +} // namespace unit From b864bd49fb4138b088876dac24945549665f7744 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Tue, 16 May 2023 00:42:44 -0700 Subject: [PATCH 0121/1425] Fix for resource scoping * Minor tweaks * Make sure the machine is used when creating a key partition * Make sure the logger doesn't query the shape of unbound stores See merge request legate/legate.core.internal!25 --- src/core/data/logical_store_detail.cc | 25 ++++-- src/core/data/logical_store_detail.h | 11 ++- src/core/mapping/machine.cc | 2 +- src/core/mapping/machine.h | 8 +- src/core/partitioning/partitioner.cc | 2 +- src/core/runtime/launcher.cc | 14 +-- src/core/runtime/launcher.h | 13 ++- src/core/runtime/operation.cc | 26 ++++-- src/core/runtime/operation.h | 20 ++++- src/core/runtime/runtime.cc | 117 +++++++++++++++---------- src/core/runtime/runtime.h | 14 +-- tests/cpp/integration/machine_scope.cc | 6 +- 12 files changed, 169 insertions(+), 89 deletions(-) diff --git a/src/core/data/logical_store_detail.cc b/src/core/data/logical_store_detail.cc index 361163fbd6..6ea3e11a01 100644 --- a/src/core/data/logical_store_detail.cc +++ b/src/core/data/logical_store_detail.cc @@ -16,6 +16,7 @@ #include "core/data/logical_store_detail.h" +#include "core/mapping/machine.h" #include "core/runtime/req_analyzer.h" #include "core/type/type_traits.h" #include "core/utilities/buffer_builder.h" @@ -97,12 +98,14 @@ RegionField Storage::map(LibraryContext* context) return Runtime::get_runtime()->map_region_field(context, region_field_.get()); } -Partition* Storage::find_or_create_key_partition() +Partition* Storage::find_or_create_key_partition(const mapping::MachineDesc& machine) { - if (key_partition_ != nullptr) return key_partition_.get(); + uint32_t new_num_pieces = machine.count(); + if (num_pieces_ == new_num_pieces && key_partition_ != nullptr) return key_partition_.get(); auto part_mgr = Runtime::get_runtime()->partition_manager(); - auto launch_shape = part_mgr->compute_launch_shape(extents_); + auto launch_shape = part_mgr->compute_launch_shape(machine, extents_); + num_pieces_ = new_num_pieces; if (launch_shape.empty()) key_partition_ = create_no_partition(); else { @@ -423,17 +426,21 @@ std::unique_ptr LogicalStore::create_projection(const Partition* par return std::make_unique(legion_partition, proj_id); } -std::shared_ptr LogicalStore::find_or_create_key_partition() +std::shared_ptr LogicalStore::find_or_create_key_partition( + const mapping::MachineDesc& machine) { - if (key_partition_ != nullptr) return key_partition_; + uint32_t new_num_pieces = machine.count(); + if (num_pieces_ == new_num_pieces && key_partition_ != nullptr) return key_partition_; if (has_scalar_storage()) { + num_pieces_ = new_num_pieces; key_partition_ = create_no_partition(); return key_partition_; } - Partition* storage_part = storage_->find_or_create_key_partition(); + Partition* storage_part = storage_->find_or_create_key_partition(machine); auto store_part = transform_->convert(storage_part); + num_pieces_ = new_num_pieces; key_partition_ = std::move(store_part); return key_partition_; } @@ -467,7 +474,11 @@ void LogicalStore::pack(BufferBuilder& buffer) const std::string LogicalStore::to_string() const { std::stringstream ss; - ss << "Store(" << store_id_ << ") {shape: " << extents(); + ss << "Store(" << store_id_ << ") {shape: "; + if (unbound()) + ss << "(unbound)"; + else + ss << extents(); if (!transform_->identity()) ss << ", transform: " << *transform_ << "}"; else diff --git a/src/core/data/logical_store_detail.h b/src/core/data/logical_store_detail.h index ec1de634fa..d88328935d 100644 --- a/src/core/data/logical_store_detail.h +++ b/src/core/data/logical_store_detail.h @@ -23,6 +23,11 @@ #include "core/runtime/runtime.h" namespace legate { + +namespace mapping { +class MachineDesc; +} // namespace mapping + namespace detail { class StoragePartition; @@ -61,7 +66,7 @@ class Storage : public std::enable_shared_from_this { RegionField map(LibraryContext* context); public: - Partition* find_or_create_key_partition(); + Partition* find_or_create_key_partition(const mapping::MachineDesc& machine); void set_key_partition(std::unique_ptr&& key_partition); void reset_key_partition(); Legion::LogicalPartition find_or_create_legion_partition(const Partition* partition); @@ -80,6 +85,7 @@ class Storage : public std::enable_shared_from_this { Legion::Future future_{}; private: + uint32_t num_pieces_{0}; std::unique_ptr key_partition_{nullptr}; }; @@ -148,7 +154,7 @@ class LogicalStore : public std::enable_shared_from_this { public: std::unique_ptr create_projection(const Partition* partition, int32_t launch_ndim); - std::shared_ptr find_or_create_key_partition(); + std::shared_ptr find_or_create_key_partition(const mapping::MachineDesc& machine); void set_key_partition(const Partition* partition); void reset_key_partition(); @@ -171,6 +177,7 @@ class LogicalStore : public std::enable_shared_from_this { std::shared_ptr transform_; private: + uint32_t num_pieces_; std::shared_ptr key_partition_; std::shared_ptr mapped_{nullptr}; }; diff --git a/src/core/mapping/machine.cc b/src/core/mapping/machine.cc index d62a431f3e..97002824a6 100644 --- a/src/core/mapping/machine.cc +++ b/src/core/mapping/machine.cc @@ -44,7 +44,7 @@ Processor::Kind to_kind(TaskTarget target) return Processor::Kind::LOC_PROC; } -LegateVariantCode to_task_variant(TaskTarget target) +LegateVariantCode to_variant_code(TaskTarget target) { switch (target) { case TaskTarget::GPU: return LEGATE_GPU_VARIANT; diff --git a/src/core/mapping/machine.h b/src/core/mapping/machine.h index 1245271431..455fe1d06c 100644 --- a/src/core/mapping/machine.h +++ b/src/core/mapping/machine.h @@ -30,7 +30,7 @@ TaskTarget to_target(Processor::Kind kind); Processor::Kind to_kind(TaskTarget target); -LegateVariantCode to_task_variant(TaskTarget target); +LegateVariantCode to_variant_code(TaskTarget target); struct ProcessorRange { ProcessorRange() {} @@ -54,6 +54,12 @@ struct MachineDesc { MachineDesc() {} MachineDesc(const std::map& processor_ranges); + MachineDesc(const MachineDesc&) = default; + MachineDesc& operator=(const MachineDesc&) = default; + + MachineDesc(MachineDesc&&) = default; + MachineDesc& operator=(MachineDesc&&) = default; + const ProcessorRange& processor_range() const; const ProcessorRange& processor_range(const TaskTarget& target) const; diff --git a/src/core/partitioning/partitioner.cc b/src/core/partitioning/partitioner.cc index bdf8d6c3f0..3fbc2e59e9 100644 --- a/src/core/partitioning/partitioner.cc +++ b/src/core/partitioning/partitioner.cc @@ -230,7 +230,7 @@ std::unique_ptr Partitioner::solve() auto* op = part_symb->operation(); auto store = op->find_store(part_symb); - auto partition = store->find_or_create_key_partition(); + auto partition = store->find_or_create_key_partition(op->machine()); std::vector equiv_class; constraints.find_equivalence_class(part_symb, equiv_class); diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/launcher.cc index 72ac8adac1..a9f68f8a96 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -19,6 +19,7 @@ #include "core/data/logical_store.h" #include "core/data/logical_store_detail.h" #include "core/data/scalar.h" +#include "core/mapping/machine.h" #include "core/runtime/context.h" #include "core/runtime/launcher_arg.h" #include "core/runtime/req_analyzer.h" @@ -28,13 +29,17 @@ namespace legate { -TaskLauncher::TaskLauncher(LibraryContext* library, int64_t task_id, int64_t tag /*= 0*/) +TaskLauncher::TaskLauncher(LibraryContext* library, + const mapping::MachineDesc& machine, + int64_t task_id, + int64_t tag /*= 0*/) : library_(library), task_id_(task_id), tag_(tag) { req_analyzer_ = new RequirementAnalyzer(); out_analyzer_ = new OutputRequirementAnalyzer(); buffer_ = new BufferBuilder(); mapper_arg_ = new BufferBuilder(); + machine.pack(*mapper_arg_); } TaskLauncher::~TaskLauncher() @@ -159,9 +164,8 @@ void TaskLauncher::pack_args(const std::vector& args) for (auto& arg : args) arg->pack(*buffer_); } -void TaskLauncher::pack_mapper_arg() +void TaskLauncher::pack_sharding_functor_id() { - Runtime::get_runtime()->get_machine().pack(*mapper_arg_); // TODO: Generate the right sharding functor id mapper_arg_->pack(0); } @@ -184,7 +188,7 @@ std::unique_ptr TaskLauncher::build_single_task() // # communicators buffer_->pack(0); - pack_mapper_arg(); + pack_sharding_functor_id(); auto* runtime = Runtime::get_runtime(); auto& provenance = runtime->provenance_manager()->get_provenance(); @@ -222,7 +226,7 @@ std::unique_ptr TaskLauncher::build_index_task( // # communicators buffer_->pack(0); - pack_mapper_arg(); + pack_sharding_functor_id(); auto* runtime = Runtime::get_runtime(); auto& provenance = runtime->provenance_manager()->get_provenance(); diff --git a/src/core/runtime/launcher.h b/src/core/runtime/launcher.h index 9f601e120d..ea7593efa3 100644 --- a/src/core/runtime/launcher.h +++ b/src/core/runtime/launcher.h @@ -32,6 +32,12 @@ class OutputRequirementAnalyzer; class RequirementAnalyzer; class Scalar; +namespace mapping { + +class MachineDesc; + +} // namespace mapping + namespace detail { class LogicalStore; @@ -40,7 +46,10 @@ class LogicalStore; class TaskLauncher { public: - TaskLauncher(LibraryContext* library, int64_t task_id, int64_t tag = 0); + TaskLauncher(LibraryContext* library, + const mapping::MachineDesc& machine, + int64_t task_id, + int64_t tag = 0); ~TaskLauncher(); public: @@ -84,7 +93,7 @@ class TaskLauncher { private: void pack_args(const std::vector& args); - void pack_mapper_arg(); + void pack_sharding_functor_id(); std::unique_ptr build_index_task(const Legion::Domain& launch_domain); std::unique_ptr build_single_task(); void bind_region_fields_to_unbound_stores(); diff --git a/src/core/runtime/operation.cc b/src/core/runtime/operation.cc index 8366636cd4..55c3de8568 100644 --- a/src/core/runtime/operation.cc +++ b/src/core/runtime/operation.cc @@ -36,8 +36,8 @@ namespace legate { // legate::Operation //////////////////////////////////////////////////// -Operation::Operation(LibraryContext* library, uint64_t unique_id) - : library_(library), unique_id_(unique_id) +Operation::Operation(LibraryContext* library, uint64_t unique_id, mapping::MachineDesc&& machine) + : library_(library), unique_id_(unique_id), machine_(std::forward(machine)) { } @@ -60,8 +60,11 @@ detail::LogicalStore* Operation::find_store(const Variable* part_symb) const // legate::Task //////////////////////////////////////////////////// -Task::Task(LibraryContext* library, int64_t task_id, uint64_t unique_id) - : Operation(library, unique_id), task_id_(task_id) +Task::Task(LibraryContext* library, + int64_t task_id, + uint64_t unique_id, + mapping::MachineDesc&& machine) + : Operation(library, unique_id, std::forward(machine)), task_id_(task_id) { } @@ -70,7 +73,7 @@ void Task::add_scalar_arg(const Scalar& scalar) { scalars_.push_back(scalar); } void Task::launch(Strategy* p_strategy) { auto& strategy = *p_strategy; - TaskLauncher launcher(library_, task_id_); + TaskLauncher launcher(library_, machine_, task_id_); auto launch_domain = strategy.launch_domain(this); auto launch_ndim = launch_domain != nullptr ? launch_domain->dim : 0; @@ -191,8 +194,11 @@ std::string Task::to_string() const // legate::AutoTask //////////////////////////////////////////////////// -AutoTask::AutoTask(LibraryContext* library, int64_t task_id, uint64_t unique_id) - : Task(library, task_id, unique_id) +AutoTask::AutoTask(LibraryContext* library, + int64_t task_id, + uint64_t unique_id, + mapping::MachineDesc&& machine) + : Task(library, task_id, unique_id, std::forward(machine)) { } @@ -248,8 +254,10 @@ ManualTask::~ManualTask() {} ManualTask::ManualTask(LibraryContext* library, int64_t task_id, const Shape& launch_shape, - uint64_t unique_id) - : Task(library, task_id, unique_id), strategy_(std::make_unique()) + uint64_t unique_id, + mapping::MachineDesc&& machine) + : Task(library, task_id, unique_id, std::forward(machine)), + strategy_(std::make_unique()) { strategy_->set_launch_shape(this, launch_shape); } diff --git a/src/core/runtime/operation.h b/src/core/runtime/operation.h index c62c4a4a1e..83a07835bb 100644 --- a/src/core/runtime/operation.h +++ b/src/core/runtime/operation.h @@ -20,6 +20,7 @@ #include #include "core/data/logical_store.h" +#include "core/mapping/machine.h" #include "core/partitioning/constraint.h" #include "legion.h" @@ -41,7 +42,7 @@ class LogicalStore; class Operation { protected: using StoreArg = std::pair; - Operation(LibraryContext* library, uint64_t unique_id); + Operation(LibraryContext* library, uint64_t unique_id, mapping::MachineDesc&& machine); public: virtual ~Operation() {} @@ -55,6 +56,9 @@ class Operation { const Variable* declare_partition(); detail::LogicalStore* find_store(const Variable* variable) const; + public: + const mapping::MachineDesc& machine() const { return machine_; } + protected: LibraryContext* library_; uint64_t unique_id_; @@ -70,11 +74,15 @@ class Operation { uint32_t next_part_id_{0}; std::vector> partition_symbols_{}; std::map store_mappings_{}; + mapping::MachineDesc machine_; }; class Task : public Operation { protected: - Task(LibraryContext* library, int64_t task_id, uint64_t unique_id); + Task(LibraryContext* library, + int64_t task_id, + uint64_t unique_id, + mapping::MachineDesc&& machine); public: virtual ~Task() {} @@ -102,7 +110,10 @@ class Task : public Operation { class AutoTask : public Task { public: friend class Runtime; - AutoTask(LibraryContext* library, int64_t task_id, uint64_t unique_id); + AutoTask(LibraryContext* library, + int64_t task_id, + uint64_t unique_id, + mapping::MachineDesc&& machine); public: ~AutoTask() {} @@ -133,7 +144,8 @@ class ManualTask : public Task { ManualTask(LibraryContext* library, int64_t task_id, const Shape& launch_shape, - uint64_t unique_id); + uint64_t unique_id, + mapping::MachineDesc&& machine); public: ~ManualTask(); diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 6af274ea27..ad22de6fb2 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -441,32 +441,42 @@ std::shared_ptr FieldManager::import_field(const Legion::Log PartitionManager::PartitionManager(Runtime* runtime, const LibraryContext* context) { auto mapper_id = context->get_mapper_id(); - num_pieces_ = runtime->get_tunable(mapper_id, LEGATE_CORE_TUNABLE_NUM_PIECES); min_shard_volume_ = runtime->get_tunable(mapper_id, LEGATE_CORE_TUNABLE_MIN_SHARD_VOLUME); - assert(num_pieces_ > 0); +#ifdef DEBUG_LEGATE assert(min_shard_volume_ > 0); +#endif +} - int32_t remaining_pieces = num_pieces_; - auto push_factors = [&](auto prime) { - while (remaining_pieces % prime == 0) { - piece_factors_.push_back(prime); - remaining_pieces /= prime; - } - }; +const std::vector& PartitionManager::get_factors(const mapping::MachineDesc& machine) +{ + uint32_t curr_num_pieces = machine.count(); - push_factors(11); - push_factors(7); - push_factors(5); - push_factors(3); - push_factors(2); + auto finder = all_factors_.find(curr_num_pieces); + if (all_factors_.end() == finder) { + uint32_t remaining_pieces = curr_num_pieces; + std::vector factors; + auto push_factors = [&factors, &remaining_pieces](uint32_t prime) { + while (remaining_pieces % prime == 0) { + factors.push_back(prime); + remaining_pieces /= prime; + } + }; + for (uint32_t factor : {11, 7, 5, 3, 2}) push_factors(factor); + all_factors_.insert({curr_num_pieces, std::move(factors)}); + finder = all_factors_.find(curr_num_pieces); + } + + return finder->second; } -Shape PartitionManager::compute_launch_shape(const Shape& shape) +Shape PartitionManager::compute_launch_shape(const mapping::MachineDesc& machine, + const Shape& shape) { + uint32_t curr_num_pieces = machine.count(); // Easy case if we only have one piece: no parallel launch space - if (num_pieces_ == 1) return {}; + if (1 == curr_num_pieces) return {}; // If we only have one point then we never do parallel launches if (shape.all([](auto extent) { return 1 == extent; })) return {}; @@ -492,7 +502,7 @@ Shape PartitionManager::compute_launch_shape(const Shape& shape) // Otherwise we need to compute it ourselves // TODO: a better heuristic here. // For now if we can make at least two pieces then we will make N pieces. - max_pieces = num_pieces_; + max_pieces = curr_num_pieces; // First compute the N-th root of the number of pieces uint32_t ndim = temp_shape.size(); @@ -548,7 +558,7 @@ Shape PartitionManager::compute_launch_shape(const Shape& shape) temp_result.resize(ndim); std::fill(temp_result.begin(), temp_result.end(), 1); size_t factor_prod = 1; - for (auto factor : piece_factors_) { + for (auto factor : get_factors(machine)) { // Avoid exceeding the maximum number of pieces if (factor * factor_prod > max_pieces) break; @@ -628,7 +638,15 @@ const mapping::MachineDesc& MachineManager::get_machine() const return machines_.back(); } -void MachineManager::push_machine(const mapping::MachineDesc& m) { machines_.push_back(m); } +void MachineManager::push_machine(const mapping::MachineDesc& machine) +{ + machines_.push_back(machine); +} + +void MachineManager::push_machine(mapping::MachineDesc&& machine) +{ + machines_.emplace_back(machine); +} void MachineManager::pop_machine() { @@ -640,12 +658,13 @@ void MachineManager::pop_machine() // legate::MachineTracker //////////////////////////////////////////// -MachineTracker::MachineTracker(const mapping::MachineDesc& m) +MachineTracker::MachineTracker(const mapping::MachineDesc& machine) { auto* runtime = Runtime::get_runtime(); - auto machine = m & Runtime::get_runtime()->get_machine(); - if (machine.count() == 0) throw std::runtime_error("Runtime can not use an empty machine"); - runtime->machine_manager()->push_machine(machine); + auto result = machine & Runtime::get_runtime()->get_machine(); + if (result.count() == 0) + throw std::runtime_error("Empty machines cannot be used for resource scoping"); + runtime->machine_manager()->push_machine(std::move(result)); } MachineTracker::~MachineTracker() @@ -770,31 +789,35 @@ void Runtime::post_startup_initialization(Legion::Context legion_context) mapping::MachineDesc Runtime::slice_machine_for_task(LibraryContext* library, int64_t task_id) { auto task_info = library->find_task(task_id); -#ifdef DEBUG_LEGATE - assert(task_info != nullptr); -#endif + if (nullptr == task_info) { + std::stringstream ss; + ss << "Library " << library->get_library_name() << " does not have task " << task_id; + throw std::invalid_argument(std::move(ss).str()); + } + std::set task_targets; auto& machine = machine_manager_->get_machine(); for (const auto& t : machine.valid_targets()) { - auto variant = mapping::to_task_variant(t); + auto variant = mapping::to_variant_code(t); if (task_info->has_variant(variant)) task_targets.insert(t); } - auto new_machine = machine.only(task_targets); - if (new_machine.empty()) { + auto sliced = machine.only(task_targets); + if (sliced.empty()) { std::stringstream ss; - ss << "Task " << task_id << " " << task_info->name() << " does not have any valid variant for " + ss << "Task " << task_id << " (" << task_info->name() << ") of library " + << library->get_library_name() << " does not have any valid variant for " << "the current machine configuration."; - std::runtime_error(ss.str()); + throw std::invalid_argument(ss.str()); } - return new_machine; + return std::move(sliced); } // This function should be moved to the library context std::unique_ptr Runtime::create_task(LibraryContext* library, int64_t task_id) { - MachineTracker track(slice_machine_for_task(library, task_id)); - auto task = new AutoTask(library, task_id, next_unique_id_++); + auto machine = slice_machine_for_task(library, task_id); + auto task = new AutoTask(library, task_id, next_unique_id_++, std::move(machine)); return std::unique_ptr(task); } @@ -802,8 +825,8 @@ std::unique_ptr Runtime::create_task(LibraryContext* library, int64_t task_id, const Shape& launch_shape) { - MachineTracker track(slice_machine_for_task(library, task_id)); - auto task = new ManualTask(library, task_id, launch_shape, next_unique_id_++); + auto machine = slice_machine_for_task(library, task_id); + auto task = new ManualTask(library, task_id, launch_shape, next_unique_id_++, std::move(machine)); return std::unique_ptr(task); } @@ -1027,10 +1050,9 @@ Legion::FutureMap Runtime::dispatch(Legion::IndexTaskLauncher* launcher, Legion::Future Runtime::extract_scalar(const Legion::Future& result, uint32_t idx) const { - // FIXME: One of two things should happen: - // 1) the variant should be picked based on available processor kinds - // 2) the variant should be picked by the core mapper - TaskLauncher launcher(core_context_, LEGATE_CORE_EXTRACT_SCALAR_TASK_ID, LEGATE_CPU_VARIANT); + auto& machine = get_machine(); + auto variant = mapping::to_variant_code(machine.preferred_target); + TaskLauncher launcher(core_context_, machine, LEGATE_CORE_EXTRACT_SCALAR_TASK_ID, variant); launcher.add_future(result); launcher.add_scalar(Scalar(idx)); return launcher.execute_single(); @@ -1040,10 +1062,9 @@ Legion::FutureMap Runtime::extract_scalar(const Legion::FutureMap& result, uint32_t idx, const Legion::Domain& launch_domain) const { - // FIXME: One of two things should happen: - // 1) the variant should be picked based on available processor kinds - // 2) the variant should be picked by the core mapper - TaskLauncher launcher(core_context_, LEGATE_CORE_EXTRACT_SCALAR_TASK_ID, LEGATE_CPU_VARIANT); + auto& machine = get_machine(); + auto variant = mapping::to_variant_code(machine.preferred_target); + TaskLauncher launcher(core_context_, machine, LEGATE_CORE_EXTRACT_SCALAR_TASK_ID, variant); launcher.add_future_map(result); launcher.add_scalar(Scalar(idx)); return launcher.execute(launch_domain); @@ -1079,14 +1100,14 @@ void Runtime::initialize_toplevel_machine() return mapping::ProcessorRange(0, num_procs, per_node_count); }; - auto machine = new mapping::MachineDesc({{mapping::TaskTarget::GPU, create_range(num_gpus)}, - {mapping::TaskTarget::OMP, create_range(num_omps)}, - {mapping::TaskTarget::CPU, create_range(num_cpus)}}); + mapping::MachineDesc machine({{mapping::TaskTarget::GPU, create_range(num_gpus)}, + {mapping::TaskTarget::OMP, create_range(num_omps)}, + {mapping::TaskTarget::CPU, create_range(num_cpus)}}); #ifdef DEBUG_LEGATE assert(machine_manager_ != nullptr); #endif - machine_manager_->push_machine(*machine); + machine_manager_->push_machine(std::move(machine)); } const mapping::MachineDesc& Runtime::get_machine() const diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index c804e5448a..579b2d9072 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -41,7 +41,6 @@ class LibraryContext; namespace mapping { -class MachineDesc; class Mapper; } // namespace mapping @@ -128,7 +127,10 @@ class PartitionManager { PartitionManager(Runtime* runtime, const LibraryContext* context); public: - Shape compute_launch_shape(const Shape& shape); + const std::vector& get_factors(const mapping::MachineDesc& machine); + + public: + Shape compute_launch_shape(const mapping::MachineDesc& machine, const Shape& shape); Shape compute_tile_shape(const Shape& extents, const Shape& launch_shape); public: @@ -139,9 +141,8 @@ class PartitionManager { const Legion::IndexPartition& index_partition); private: - int32_t num_pieces_; int64_t min_shard_volume_; - std::vector piece_factors_; + std::unordered_map> all_factors_; private: using TilingCacheKey = std::pair; @@ -155,7 +156,8 @@ class MachineManager { public: const mapping::MachineDesc& get_machine() const; - void push_machine(const mapping::MachineDesc& m); + void push_machine(const mapping::MachineDesc& machine); + void push_machine(mapping::MachineDesc&& machine); void pop_machine(); @@ -164,7 +166,7 @@ class MachineManager { }; struct MachineTracker { - MachineTracker(const mapping::MachineDesc& m); + MachineTracker(const mapping::MachineDesc& machine); ~MachineTracker(); diff --git a/tests/cpp/integration/machine_scope.cc b/tests/cpp/integration/machine_scope.cc index b07599aeef..7bdad1e8c8 100644 --- a/tests/cpp/integration/machine_scope.cc +++ b/tests/cpp/integration/machine_scope.cc @@ -35,7 +35,7 @@ void validate(legate::TaskContext& context) int32_t num_tasks = context.get_launch_domain().get_volume(); auto to_compare = context.scalars().at(0).value(); - assert(to_compare == num_tasks); + EXPECT_EQ(to_compare, num_tasks); } struct MultiVariantTask : public legate::LegateTask { @@ -110,7 +110,7 @@ void test_cpu_only(legate::LibraryContext* context) auto task = runtime->create_task(context, CPU_VARIANT); auto part = task->declare_partition(); task->add_output(store, part); - task->add_scalar_arg(machine.count()); + task->add_scalar_arg(machine.count(legate::mapping::TaskTarget::CPU)); runtime->submit(std::move(task)); if (machine.count(legate::mapping::TaskTarget::CPU) > 0) { @@ -132,7 +132,7 @@ void test_cpu_only(legate::LibraryContext* context) // check `slice_machine_for_task` logic if (machine.count(legate::mapping::TaskTarget::GPU) > 0) { legate::MachineTracker tracker(machine.only(legate::mapping::TaskTarget::GPU)); - EXPECT_THROW(runtime->create_task(context, CPU_VARIANT), std::runtime_error); + EXPECT_THROW(runtime->create_task(context, CPU_VARIANT), std::invalid_argument); } } From 0e98ce9a53c0bf8d7a302ececfa4b0416faf9246 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 17 May 2023 23:50:47 -0700 Subject: [PATCH 0122/1425] Broadcast constraints for auto-partitioner * Test broadcast constraints with promotion * Add an integration test for broadcast constraints * Support for broadcast constraints See merge request legate/legate.core.internal!27 --- legate_core_cpp.cmake | 1 + src/core/data/logical_store_detail.cc | 25 ++-- src/core/data/logical_store_detail.h | 9 +- src/core/data/transform.cc | 36 ++++++ src/core/data/transform.h | 8 ++ src/core/partitioning/constraint.cc | 28 +++++ src/core/partitioning/constraint.h | 58 +++++++-- src/core/partitioning/constraint_graph.cc | 76 +++++++++--- src/core/partitioning/constraint_graph.h | 11 +- src/core/partitioning/partitioner.cc | 11 +- src/core/partitioning/restriction.cc | 36 ++++++ src/core/partitioning/restriction.h | 40 ++++++ src/core/runtime/runtime.cc | 3 +- src/core/runtime/runtime.h | 5 +- .../cpp/integration/broadcast_constraints.cc | 115 ++++++++++++++++++ 15 files changed, 412 insertions(+), 50 deletions(-) create mode 100644 src/core/partitioning/restriction.cc create mode 100644 src/core/partitioning/restriction.h create mode 100644 tests/cpp/integration/broadcast_constraints.cc diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index e161b2a4ee..92eadeb6e5 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -211,6 +211,7 @@ list(APPEND legate_core_SOURCES src/core/partitioning/constraint_graph.cc src/core/partitioning/partition.cc src/core/partitioning/partitioner.cc + src/core/partitioning/restriction.cc src/core/runtime/context.cc src/core/runtime/launcher_arg.cc src/core/runtime/launcher.cc diff --git a/src/core/data/logical_store_detail.cc b/src/core/data/logical_store_detail.cc index 6ea3e11a01..a40101eecb 100644 --- a/src/core/data/logical_store_detail.cc +++ b/src/core/data/logical_store_detail.cc @@ -98,14 +98,17 @@ RegionField Storage::map(LibraryContext* context) return Runtime::get_runtime()->map_region_field(context, region_field_.get()); } -Partition* Storage::find_or_create_key_partition(const mapping::MachineDesc& machine) +Partition* Storage::find_or_create_key_partition(const mapping::MachineDesc& machine, + const Restrictions& restrictions) { uint32_t new_num_pieces = machine.count(); - if (num_pieces_ == new_num_pieces && key_partition_ != nullptr) return key_partition_.get(); + if (num_pieces_ == new_num_pieces && key_partition_ != nullptr && restrictions_ == restrictions) + return key_partition_.get(); auto part_mgr = Runtime::get_runtime()->partition_manager(); - auto launch_shape = part_mgr->compute_launch_shape(machine, extents_); + auto launch_shape = part_mgr->compute_launch_shape(machine, restrictions, extents_); num_pieces_ = new_num_pieces; + restrictions_ = restrictions; if (launch_shape.empty()) key_partition_ = create_no_partition(); else { @@ -427,21 +430,25 @@ std::unique_ptr LogicalStore::create_projection(const Partition* par } std::shared_ptr LogicalStore::find_or_create_key_partition( - const mapping::MachineDesc& machine) + const mapping::MachineDesc& machine, const Restrictions& restrictions) { uint32_t new_num_pieces = machine.count(); - if (num_pieces_ == new_num_pieces && key_partition_ != nullptr) return key_partition_; + if (num_pieces_ == new_num_pieces && key_partition_ != nullptr && restrictions_ == restrictions) + return key_partition_; if (has_scalar_storage()) { num_pieces_ = new_num_pieces; + restrictions_ = restrictions; key_partition_ = create_no_partition(); return key_partition_; } - Partition* storage_part = storage_->find_or_create_key_partition(machine); - auto store_part = transform_->convert(storage_part); - num_pieces_ = new_num_pieces; - key_partition_ = std::move(store_part); + Partition* storage_part = + storage_->find_or_create_key_partition(machine, transform_->invert(restrictions)); + auto store_part = transform_->convert(storage_part); + num_pieces_ = new_num_pieces; + restrictions_ = restrictions; + key_partition_ = std::move(store_part); return key_partition_; } diff --git a/src/core/data/logical_store_detail.h b/src/core/data/logical_store_detail.h index d88328935d..4aa0be7eaa 100644 --- a/src/core/data/logical_store_detail.h +++ b/src/core/data/logical_store_detail.h @@ -20,6 +20,7 @@ #include "core/data/logical_region_field.h" #include "core/partitioning/partition.h" +#include "core/partitioning/restriction.h" #include "core/runtime/runtime.h" namespace legate { @@ -66,7 +67,8 @@ class Storage : public std::enable_shared_from_this { RegionField map(LibraryContext* context); public: - Partition* find_or_create_key_partition(const mapping::MachineDesc& machine); + Partition* find_or_create_key_partition(const mapping::MachineDesc& machine, + const Restrictions& restrictions); void set_key_partition(std::unique_ptr&& key_partition); void reset_key_partition(); Legion::LogicalPartition find_or_create_legion_partition(const Partition* partition); @@ -86,6 +88,7 @@ class Storage : public std::enable_shared_from_this { private: uint32_t num_pieces_{0}; + Restrictions restrictions_{}; std::unique_ptr key_partition_{nullptr}; }; @@ -154,7 +157,8 @@ class LogicalStore : public std::enable_shared_from_this { public: std::unique_ptr create_projection(const Partition* partition, int32_t launch_ndim); - std::shared_ptr find_or_create_key_partition(const mapping::MachineDesc& machine); + std::shared_ptr find_or_create_key_partition(const mapping::MachineDesc& machine, + const Restrictions& restrictions); void set_key_partition(const Partition* partition); void reset_key_partition(); @@ -178,6 +182,7 @@ class LogicalStore : public std::enable_shared_from_this { private: uint32_t num_pieces_; + Restrictions restrictions_{}; std::shared_ptr key_partition_; std::shared_ptr mapped_{nullptr}; }; diff --git a/src/core/data/transform.cc b/src/core/data/transform.cc index 13548d8432..c8498b5b35 100644 --- a/src/core/data/transform.cc +++ b/src/core/data/transform.cc @@ -67,10 +67,20 @@ std::unique_ptr TransformStack::invert(const Partition* partition) co proj::SymbolicPoint TransformStack::invert(const proj::SymbolicPoint& point) const { + if (identity()) return point; + auto result = transform_->invert(point); return parent_->identity() ? result : parent_->invert(result); } +Restrictions TransformStack::invert(const Restrictions& restrictions) const +{ + if (identity()) return restrictions; + + auto result = transform_->invert(restrictions); + return parent_->identity() ? std::move(result) : parent_->invert(result); +} + void TransformStack::pack(BufferBuilder& buffer) const { if (identity()) @@ -179,6 +189,8 @@ std::unique_ptr Shift::invert(const Partition* partition) const { ret // the shift transform makes no change on the store's dimensions proj::SymbolicPoint Shift::invert(const proj::SymbolicPoint& point) const { return point; } +Restrictions Shift::invert(const Restrictions& restrictions) const { return restrictions; } + void Shift::pack(BufferBuilder& buffer) const { buffer.pack(LEGATE_CORE_TRANSFORM_SHIFT); @@ -280,6 +292,11 @@ proj::SymbolicPoint Promote::invert(const proj::SymbolicPoint& point) const return point.remove(extra_dim_); } +Restrictions Promote::invert(const Restrictions& restrictions) const +{ + return restrictions.remove(extra_dim_); +} + void Promote::pack(BufferBuilder& buffer) const { buffer.pack(LEGATE_CORE_TRANSFORM_PROMOTE); @@ -380,6 +397,11 @@ proj::SymbolicPoint Project::invert(const proj::SymbolicPoint& point) const return point.insert(dim_, proj::SymbolicExpr()); } +Restrictions Project::invert(const Restrictions& restrictions) const +{ + return restrictions.insert(dim_, Restriction::ALLOW); +} + void Project::pack(BufferBuilder& buffer) const { buffer.pack(LEGATE_CORE_TRANSFORM_PROJECT); @@ -452,6 +474,13 @@ proj::SymbolicPoint Transpose::invert(const proj::SymbolicPoint& point) const return proj::SymbolicPoint(std::move(exprs)); } +Restrictions Transpose::invert(const Restrictions& restrictions) const +{ + std::vector result; + for (int32_t dim : inverse_) result.push_back(restrictions[dim]); + return Restrictions(std::move(result)); +} + namespace { // anonymous template void print_vector(std::ostream& out, const std::vector& vec) @@ -563,6 +592,13 @@ proj::SymbolicPoint Delinearize::invert(const proj::SymbolicPoint& point) const return proj::SymbolicPoint(std::move(exprs)); } +Restrictions Delinearize::invert(const Restrictions& restrictions) const +{ + auto result = restrictions; + for (uint32_t dim = 1; dim < sizes_.size(); ++dim) result.remove_inplace(dim + 1); + return std::move(result); +} + void Delinearize::pack(BufferBuilder& buffer) const { buffer.pack(LEGATE_CORE_TRANSFORM_DELINEARIZE); diff --git a/src/core/data/transform.h b/src/core/data/transform.h index 068b423082..725bda6566 100644 --- a/src/core/data/transform.h +++ b/src/core/data/transform.h @@ -18,6 +18,7 @@ #include +#include "core/partitioning/restriction.h" #include "core/runtime/projection.h" #include "core/utilities/typedefs.h" @@ -32,6 +33,7 @@ struct Transform { virtual std::unique_ptr convert(const Partition* partition) const = 0; virtual std::unique_ptr invert(const Partition* partition) const = 0; virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const = 0; + virtual Restrictions invert(const Restrictions& restrictions) const = 0; virtual void pack(BufferBuilder& buffer) const = 0; virtual void print(std::ostream& out) const = 0; }; @@ -55,6 +57,7 @@ struct TransformStack : public Transform, std::enable_shared_from_this convert(const Partition* partition) const override; virtual std::unique_ptr invert(const Partition* partition) const override; virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; + virtual Restrictions invert(const Restrictions& restrictions) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; @@ -81,6 +84,7 @@ class Shift : public StoreTransform { virtual std::unique_ptr convert(const Partition* partition) const override; virtual std::unique_ptr invert(const Partition* partition) const override; virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; + virtual Restrictions invert(const Restrictions& restrictions) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; @@ -102,6 +106,7 @@ class Promote : public StoreTransform { virtual std::unique_ptr convert(const Partition* partition) const override; virtual std::unique_ptr invert(const Partition* partition) const override; virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; + virtual Restrictions invert(const Restrictions& restrictions) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; @@ -124,6 +129,7 @@ class Project : public StoreTransform { virtual std::unique_ptr convert(const Partition* partition) const override; virtual std::unique_ptr invert(const Partition* partition) const override; virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; + virtual Restrictions invert(const Restrictions& restrictions) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; @@ -145,6 +151,7 @@ class Transpose : public StoreTransform { virtual std::unique_ptr convert(const Partition* partition) const override; virtual std::unique_ptr invert(const Partition* partition) const override; virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; + virtual Restrictions invert(const Restrictions& restrictions) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; @@ -166,6 +173,7 @@ class Delinearize : public StoreTransform { virtual std::unique_ptr convert(const Partition* partition) const override; virtual std::unique_ptr invert(const Partition* partition) const override; virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; + virtual Restrictions invert(const Restrictions& restrictions) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; diff --git a/src/core/partitioning/constraint.cc b/src/core/partitioning/constraint.cc index fcb7a4760e..9b16e4d660 100644 --- a/src/core/partitioning/constraint.cc +++ b/src/core/partitioning/constraint.cc @@ -68,6 +68,23 @@ std::string Alignment::to_string() const return std::move(ss).str(); } +Broadcast::Broadcast(std::unique_ptr variable, tuple&& axes) + : variable_(std::move(variable)), axes_(std::forward>(axes)) +{ +} + +void Broadcast::find_partition_symbols(std::vector& partition_symbols) const +{ + partition_symbols.push_back(variable_.get()); +} + +std::string Broadcast::to_string() const +{ + std::stringstream ss; + ss << "Broadcast(" << variable_->to_string() << ", " << axes_.to_string() << ")"; + return std::move(ss).str(); +} + std::unique_ptr align(const Variable* lhs, const Variable* rhs) { // Since an Alignment object owns child nodes, inputs need to be copied @@ -75,4 +92,15 @@ std::unique_ptr align(const Variable* lhs, const Variable* rhs) std::make_unique(*rhs)); } +std::unique_ptr broadcast(const Variable* variable, const tuple& axes) +{ + return broadcast(variable, tuple(axes)); +} + +std::unique_ptr broadcast(const Variable* variable, tuple&& axes) +{ + return std::make_unique(std::make_unique(*variable), + std::forward>(axes)); +} + } // namespace legate diff --git a/src/core/partitioning/constraint.h b/src/core/partitioning/constraint.h index 290a37bbbb..cd2cfa573b 100644 --- a/src/core/partitioning/constraint.h +++ b/src/core/partitioning/constraint.h @@ -20,9 +20,12 @@ #include #include +#include "core/utilities/tuple.h" + namespace legate { struct Alignment; +struct Broadcast; struct Constraint; struct Literal; struct Operation; @@ -105,33 +108,40 @@ struct Variable : public Expr { }; struct Constraint { - public: + enum class Kind : int32_t { + ALIGNMENT = 0, + BROADCAST = 1, + }; + virtual ~Constraint() {} - public: + virtual Kind kind() const = 0; + virtual void find_partition_symbols(std::vector& partition_symbols) const = 0; - public: virtual std::string to_string() const = 0; - public: virtual const Alignment* as_alignment() const = 0; + virtual const Broadcast* as_broadcast() const = 0; }; // Constraint AST nodes own their child nodes -struct Alignment : public Constraint { +class Alignment : public Constraint { public: Alignment(std::unique_ptr&& lhs, std::unique_ptr&& rhs); public: - virtual void find_partition_symbols( - std::vector& partition_symbols) const override; + Kind kind() const override { return Kind::ALIGNMENT; } public: - virtual std::string to_string() const override; + void find_partition_symbols(std::vector& partition_symbols) const override; + + public: + std::string to_string() const override; public: - virtual const Alignment* as_alignment() const override { return this; } + const Alignment* as_alignment() const override { return this; } + const Broadcast* as_broadcast() const override { return nullptr; } public: const Expr* lhs() const { return lhs_.get(); } @@ -142,6 +152,36 @@ struct Alignment : public Constraint { std::unique_ptr rhs_; }; +class Broadcast : public Constraint { + public: + Broadcast(std::unique_ptr variable, tuple&& axes); + + public: + Kind kind() const override { return Kind::BROADCAST; } + + public: + void find_partition_symbols(std::vector& partition_symbols) const override; + + public: + std::string to_string() const override; + + public: + const Alignment* as_alignment() const override { return nullptr; } + const Broadcast* as_broadcast() const override { return this; } + + public: + const Variable* variable() const { return variable_.get(); } + const tuple& axes() const { return axes_; } + + private: + std::unique_ptr variable_; + tuple axes_; +}; + std::unique_ptr align(const Variable* lhs, const Variable* rhs); +std::unique_ptr broadcast(const Variable* variable, const tuple& axes); + +std::unique_ptr broadcast(const Variable* variable, tuple&& axes); + } // namespace legate diff --git a/src/core/partitioning/constraint_graph.cc b/src/core/partitioning/constraint_graph.cc index d86bd63412..2d5bbb8965 100644 --- a/src/core/partitioning/constraint_graph.cc +++ b/src/core/partitioning/constraint_graph.cc @@ -18,8 +18,10 @@ #include "legion.h" +#include "core/data/logical_store_detail.h" #include "core/partitioning/constraint.h" #include "core/partitioning/constraint_graph.h" +#include "core/runtime/operation.h" namespace legate { @@ -38,13 +40,21 @@ void ConstraintGraph::add_constraint(const Constraint* constraint) void ConstraintGraph::compute_equivalence_classes() { for (auto& part_symb : partition_symbols()) { - all_equiv_class_entries_.emplace_back(new EquivClass(part_symb)); + // TODO: partition symbols can be independent of any stores of the operation + // (e.g., when a symbol subsumes a union of two other symbols) + int32_t ndim = part_symb->operation()->find_store(part_symb)->dim(); + all_equiv_class_entries_.emplace_back(new EquivClass(part_symb, ndim)); equiv_classes_.insert({*part_symb, all_equiv_class_entries_.back().get()}); } - for (auto& constraint : constraints_) { - auto* alignment = constraint->as_alignment(); - if (nullptr == alignment) continue; + auto update_table = [&](auto* new_cls, auto* orig_cls) { + while (orig_cls != nullptr) { + equiv_classes_[*orig_cls->partition_symbol] = new_cls; + orig_cls = orig_cls->next; + } + }; + + auto handle_alignment = [&](const Alignment* alignment) { #ifdef DEBUG_LEGATE assert(alignment->lhs()->as_variable() != nullptr && alignment->rhs()->as_variable() != nullptr); @@ -60,12 +70,6 @@ void ConstraintGraph::compute_equivalence_classes() #ifdef DEBUG_LEGATE assert(equiv_class != nullptr); #endif - auto update_table = [&](auto* new_cls, auto* orig_cls) { - while (orig_cls != nullptr) { - equiv_classes_[*orig_cls->partition_symbol] = new_cls; - orig_cls = orig_cls->next; - } - }; for (size_t idx = 1; idx < part_symbs_to_unify.size(); ++idx) { auto class_to_unify = equiv_classes_[*part_symbs_to_unify[idx]]; auto result = equiv_class->unify(class_to_unify); @@ -74,22 +78,56 @@ void ConstraintGraph::compute_equivalence_classes() if (result != class_to_unify) update_table(result, class_to_unify); equiv_class = result; } + }; + + auto handle_broadcast = [&](const Broadcast* broadcast) { + auto* variable = broadcast->variable(); + auto& axes = broadcast->axes(); + EquivClass* equiv_class = equiv_classes_.at(*variable); + for (uint32_t idx = 0; idx < axes.size(); ++idx) { + uint32_t axis = axes[idx]; + // TODO: We want to check the axis eagerly and raise an exception + // if it is out of bounds + if (axis >= equiv_class->restrictions.size()) continue; + equiv_class->restrictions[axes[idx]] = Restriction::FORBID; + } + }; + + for (auto& constraint : constraints_) switch (constraint->kind()) { + case Constraint::Kind::ALIGNMENT: { + handle_alignment(constraint->as_alignment()); + break; + } + case Constraint::Kind::BROADCAST: { + handle_broadcast(constraint->as_broadcast()); + break; + } + } +} + +std::vector ConstraintGraph::find_equivalence_class( + const Variable* partition_symbol) const +{ + auto equiv_class = equiv_classes_.at(*partition_symbol); + std::vector result; + result.reserve(equiv_class->size); + while (equiv_class != nullptr) { + result.push_back(equiv_class->partition_symbol); + equiv_class = equiv_class->next; } + return std::move(result); } -void ConstraintGraph::find_equivalence_class(const Variable* partition_symbol, - std::vector& out_equiv_class) const +Restrictions ConstraintGraph::find_restrictions(const Variable* partition_symbol) const { - auto finder = equiv_classes_.find(*partition_symbol); -#ifdef DEBUG_LEGATE - assert(finder != equiv_classes_.end()); -#endif - auto equiv_class = finder->second; - out_equiv_class.reserve(equiv_class->size); + auto equiv_class = equiv_classes_.at(*partition_symbol); + Restrictions result = equiv_class->restrictions; + equiv_class = equiv_class->next; while (equiv_class != nullptr) { - out_equiv_class.push_back(equiv_class->partition_symbol); + join_inplace(result, equiv_class->restrictions); equiv_class = equiv_class->next; } + return std::move(result); } void ConstraintGraph::dump() diff --git a/src/core/partitioning/constraint_graph.h b/src/core/partitioning/constraint_graph.h index 884b4e9d1e..5bc737812a 100644 --- a/src/core/partitioning/constraint_graph.h +++ b/src/core/partitioning/constraint_graph.h @@ -21,12 +21,16 @@ #include #include #include "core/partitioning/constraint.h" +#include "core/partitioning/restriction.h" #include "core/utilities/ordered_set.h" namespace legate { struct EquivClass { - EquivClass(const Variable* symb) : partition_symbol(symb), next(nullptr), size(1) {} + EquivClass(const Variable* symb, int32_t ndim) + : partition_symbol(symb), next(nullptr), size(1), restrictions(ndim, Restriction::ALLOW) + { + } EquivClass* unify(EquivClass* other) { @@ -43,6 +47,7 @@ struct EquivClass { const Variable* partition_symbol; EquivClass* next; size_t size; + Restrictions restrictions; }; struct ConstraintGraph { @@ -59,8 +64,8 @@ struct ConstraintGraph { public: void compute_equivalence_classes(); - void find_equivalence_class(const Variable* partition_symbol, - std::vector& out_equiv_class) const; + std::vector find_equivalence_class(const Variable* partition_symbol) const; + Restrictions find_restrictions(const Variable* partition_symbol) const; private: ordered_set partition_symbols_; diff --git a/src/core/partitioning/partitioner.cc b/src/core/partitioning/partitioner.cc index 3fbc2e59e9..f86024704f 100644 --- a/src/core/partitioning/partitioner.cc +++ b/src/core/partitioning/partitioner.cc @@ -228,12 +228,12 @@ std::unique_ptr Partitioner::solve() for (auto& part_symb : partition_symbols) { if (strategy->has_assignment(part_symb)) continue; + auto equiv_class = constraints.find_equivalence_class(part_symb); + auto restrictions = constraints.find_restrictions(part_symb); + auto* op = part_symb->operation(); auto store = op->find_store(part_symb); - auto partition = store->find_or_create_key_partition(op->machine()); - - std::vector equiv_class; - constraints.find_equivalence_class(part_symb, equiv_class); + auto partition = store->find_or_create_key_partition(op->machine(), restrictions); for (auto symb : equiv_class) strategy->insert(symb, partition); } @@ -261,8 +261,7 @@ void Partitioner::solve_for_unbound_stores(std::vector& partiti continue; } - std::vector equiv_class; - constraints.find_equivalence_class(part_symb, equiv_class); + auto equiv_class = constraints.find_equivalence_class(part_symb); std::shared_ptr partition(create_no_partition()); auto field_space = runtime->create_field_space(); diff --git a/src/core/partitioning/restriction.cc b/src/core/partitioning/restriction.cc new file mode 100644 index 0000000000..ac31687161 --- /dev/null +++ b/src/core/partitioning/restriction.cc @@ -0,0 +1,36 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/partitioning/restriction.h" + +namespace legate { + +Restriction join(Restriction lhs, Restriction rhs) { return std::max(lhs, rhs); } + +tuple join(const tuple& lhs, const tuple& rhs) +{ + auto result = lhs; + join_inplace(result, rhs); + return std::move(result); +} + +void join_inplace(Restrictions& lhs, const Restrictions& rhs) +{ + if (lhs.size() != rhs.size()) throw std::invalid_argument("Restrictions must have the same size"); + for (uint32_t idx = 0; idx < lhs.size(); ++idx) lhs[idx] = join(lhs[idx], rhs[idx]); +} + +} // namespace legate diff --git a/src/core/partitioning/restriction.h b/src/core/partitioning/restriction.h new file mode 100644 index 0000000000..143c398428 --- /dev/null +++ b/src/core/partitioning/restriction.h @@ -0,0 +1,40 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include "core/utilities/tuple.h" + +namespace legate { + +/** + * @brief Enum to describe partitioning preference on dimensions of a store + */ +enum class Restriction : int32_t { + ALLOW = 0, /*!< The dimension can be partitioned */ + AVOID = 1, /*!< The dimension can be partitioned, but other dimensions are preferred */ + FORBID = 2, /*!< The dimension must not be partitioned */ +}; + +using Restrictions = tuple; + +Restriction join(Restriction lhs, Restriction rhs); + +tuple join(const Restrictions& lhs, const Restrictions& rhs); + +void join_inplace(Restrictions& lhs, const Restrictions& rhs); + +} // namespace legate diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index ad22de6fb2..9406e43244 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -472,6 +472,7 @@ const std::vector& PartitionManager::get_factors(const mapping::Machin } Shape PartitionManager::compute_launch_shape(const mapping::MachineDesc& machine, + const Restrictions& restrictions, const Shape& shape) { uint32_t curr_num_pieces = machine.count(); @@ -487,7 +488,7 @@ Shape PartitionManager::compute_launch_shape(const mapping::MachineDesc& machine int64_t volume = 1; for (uint32_t dim = 0; dim < shape.size(); ++dim) { auto extent = shape[dim]; - if (1 == extent) continue; + if (1 == extent || restrictions[dim] == Restriction::FORBID) continue; temp_shape.push_back(extent); temp_dims.push_back(dim); volume *= extent; diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 579b2d9072..0677f55a60 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -27,6 +27,7 @@ #include "core/data/store.h" #include "core/legate_c.h" #include "core/mapping/machine.h" +#include "core/partitioning/restriction.h" #include "core/runtime/resource.h" #include "core/task/exception.h" #include "core/type/type_info.h" @@ -130,7 +131,9 @@ class PartitionManager { const std::vector& get_factors(const mapping::MachineDesc& machine); public: - Shape compute_launch_shape(const mapping::MachineDesc& machine, const Shape& shape); + Shape compute_launch_shape(const mapping::MachineDesc& machine, + const Restrictions& restrictions, + const Shape& shape); Shape compute_tile_shape(const Shape& extents, const Shape& launch_shape); public: diff --git a/tests/cpp/integration/broadcast_constraints.cc b/tests/cpp/integration/broadcast_constraints.cc new file mode 100644 index 0000000000..81130dec21 --- /dev/null +++ b/tests/cpp/integration/broadcast_constraints.cc @@ -0,0 +1,115 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "legate.h" + +namespace broadcast_constraints { + +namespace { + +static const char* library_name = "test_broadcast_constraints"; + +constexpr size_t EXT_SMALL = 10; +constexpr size_t EXT_LARGE = 100; + +struct TesterTask : public legate::LegateTask { + static const int32_t TASK_ID = 0; + static void cpu_variant(legate::TaskContext& context) + { + auto shape = context.outputs().at(0).shape<3>(); + auto extent = context.scalars().at(0).value(); + auto dims = context.scalars().at(1).values(); + + for (auto dim : dims) { + EXPECT_EQ(shape.lo[dim], 0); + EXPECT_EQ(shape.hi[dim], extent - 1); + } + } +}; + +void prepare() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + TesterTask::register_variants(context); +} + +void test_normal_store() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + auto launch_tester = [&](const std::vector& dims) { + std::vector extents(3, EXT_SMALL); + for (auto dim : dims) extents[dim] = EXT_LARGE; + auto store = runtime->create_store(extents, legate::int64()); + auto task = runtime->create_task(context, 0); + auto part = task->declare_partition(); + task->add_output(store, part); + task->add_scalar_arg(legate::Scalar(EXT_LARGE)); + task->add_scalar_arg(legate::Scalar(dims)); + task->add_constraint(legate::broadcast(part, dims)); + runtime->submit(std::move(task)); + }; + + launch_tester({0}); + launch_tester({1}); + launch_tester({2}); + launch_tester({0, 1}); + launch_tester({1, 2}); + launch_tester({0, 2}); + launch_tester({0, 1, 2}); +} + +void test_promoted_store() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + auto launch_tester = [&](const int32_t dim) { + std::vector extents(2, EXT_SMALL); + extents[dim] = EXT_LARGE; + auto store = runtime->create_store(extents, legate::int64()); + auto promoted = store.promote(2, EXT_LARGE); + auto task = runtime->create_task(context, 0); + auto part = task->declare_partition(); + task->add_output(promoted, part); + task->add_scalar_arg(legate::Scalar(EXT_LARGE)); + task->add_scalar_arg(legate::Scalar(std::vector{dim})); + task->add_constraint(legate::broadcast(part, {dim})); + runtime->submit(std::move(task)); + }; + + launch_tester(0); + launch_tester(1); +} + +} // namespace + +TEST(Integration, BroadcastConstraints) +{ + legate::Core::perform_registration(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + test_normal_store(); + test_promoted_store(); +} + +} // namespace broadcast_constraints From 2b8cc38e3b4152a1490519828555754b88d62ebc Mon Sep 17 00:00:00 2001 From: Vladislav Zhurba Date: Mon, 22 May 2023 10:02:21 -0700 Subject: [PATCH 0123/1425] Fix extent calculation of inclusive domain range * Fix extent calculation of inclusive domain range See merge request legate/legate.core.internal!30 --- src/core/data/logical_store_detail.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/data/logical_store_detail.cc b/src/core/data/logical_store_detail.cc index a40101eecb..e6ce8c9659 100644 --- a/src/core/data/logical_store_detail.cc +++ b/src/core/data/logical_store_detail.cc @@ -84,7 +84,7 @@ void Storage::set_region_field(std::shared_ptr&& region_fiel auto lo = domain.lo(); auto hi = domain.hi(); std::vector extents; - for (int32_t idx = 0; idx < lo.dim; ++idx) extents.push_back(hi[idx] - lo[idx]); + for (int32_t idx = 0; idx < lo.dim; ++idx) extents.push_back(hi[idx] - lo[idx] + 1); extents_ = extents; } From f667c59542858386928bff622a14fe9f146e883a Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Tue, 23 May 2023 13:10:50 -0700 Subject: [PATCH 0124/1425] Missing header file * Missing header file to install See merge request legate/legate.core.internal!32 --- legate_core_cpp.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index 92eadeb6e5..f2300647fd 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -434,6 +434,7 @@ install( install( FILES src/core/partitioning/constraint.h + src/core/partitioning/restriction.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/legate/core/partitioning) install( From f54b0de6f0811a7969c19714a9b1d1f1a245e5a2 Mon Sep 17 00:00:00 2001 From: Vladislav Zhurba Date: Wed, 24 May 2023 09:44:34 -0700 Subject: [PATCH 0125/1425] Port examples * Remove comment * Final code review * Typos * Apply code review * Remove typos * Port Legateio example to c++ * Checkpoint See merge request legate/legate.core.internal!26 --- examples/cpp/CMakeLists.txt | 49 ++ examples/cpp/build.sh | 5 + examples/cpp/hello/hello.cc | 95 ++++ examples/cpp/hello/hello_print.cc | 54 ++ examples/cpp/hello/task_hello.cc | 109 ++++ examples/cpp/hello/task_hello.h | 61 +++ examples/cpp/io/io.cc | 395 +++++++++++++++ examples/cpp/io/task_io.cc | 465 ++++++++++++++++++ examples/cpp/io/task_io.h | 97 ++++ examples/cpp/main.cc | 42 ++ examples/cpp/run.sh | 4 + src/core/data/logical_store.cc | 5 + src/core/data/logical_store.h | 3 + src/core/partitioning/partition.h | 25 +- src/legate.h | 1 + tests/cpp/CMakeLists.txt | 3 +- tests/cpp/integration/manual_simple.cc | 42 +- tests/cpp/integration/multi_scalar_out.cc | 63 +-- tests/cpp/integration/region_manager.cc | 23 +- .../integration/tasks/task_region_manager.cc | 32 ++ .../integration/tasks/task_region_manager.h | 41 ++ tests/cpp/integration/tasks/task_simple.cc | 72 +++ tests/cpp/integration/tasks/task_simple.h | 55 +++ tests/cpp/main.cc | 16 + tests/cpp/unit/store.cc | 16 + 25 files changed, 1657 insertions(+), 116 deletions(-) create mode 100644 examples/cpp/CMakeLists.txt create mode 100755 examples/cpp/build.sh create mode 100644 examples/cpp/hello/hello.cc create mode 100644 examples/cpp/hello/hello_print.cc create mode 100644 examples/cpp/hello/task_hello.cc create mode 100644 examples/cpp/hello/task_hello.h create mode 100644 examples/cpp/io/io.cc create mode 100644 examples/cpp/io/task_io.cc create mode 100644 examples/cpp/io/task_io.h create mode 100644 examples/cpp/main.cc create mode 100755 examples/cpp/run.sh create mode 100644 tests/cpp/integration/tasks/task_region_manager.cc create mode 100644 tests/cpp/integration/tasks/task_region_manager.h create mode 100644 tests/cpp/integration/tasks/task_simple.cc create mode 100644 tests/cpp/integration/tasks/task_simple.h diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt new file mode 100644 index 0000000000..439fd11051 --- /dev/null +++ b/examples/cpp/CMakeLists.txt @@ -0,0 +1,49 @@ +#============================================================================= +# Copyright 2023 NVIDIA Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +cmake_minimum_required(VERSION 3.22.1 FATAL_ERROR) + +project(cpp_tests VERSION 0.1 LANGUAGES C CXX) + +if (NOT CMAKE_CXX_STANDARD) + set(CMAKE_CXX_STANDARD 17) +endif() + +include(FetchContent) +FetchContent_Declare( + googletest + URL https://github.com/google/googletest/archive/refs/tags/v1.13.0.zip +) + +set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) +FetchContent_MakeAvailable(googletest) + +find_package(legate_core REQUIRED) + +enable_testing() + +file(GLOB main_SRC ${PROJECT_SOURCE_DIR}/main.cc) +file(GLOB hello_SRC ${PROJECT_SOURCE_DIR}/hello/*.cc) +file(GLOB io_SRC ${PROJECT_SOURCE_DIR}/io/*.cc) + +add_executable(cpp_tests ${main_SRC} ${hello_SRC} ${io_SRC}) + +target_link_libraries(cpp_tests PRIVATE legate::core PRIVATE GTest::gtest) + +include(GoogleTest) +gtest_discover_tests(cpp_tests) + +install(TARGETS cpp_tests DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/cmake-install") diff --git a/examples/cpp/build.sh b/examples/cpp/build.sh new file mode 100755 index 0000000000..dd0fb9706a --- /dev/null +++ b/examples/cpp/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +rm -rf build +cmake -B build -S . -D legate_core_ROOT=../../build -D CMAKE_BUILD_TYPE=Debug +cmake --build build -j 4 diff --git a/examples/cpp/hello/hello.cc b/examples/cpp/hello/hello.cc new file mode 100644 index 0000000000..7b32c41a70 --- /dev/null +++ b/examples/cpp/hello/hello.cc @@ -0,0 +1,95 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "core/mapping/mapping.h" +#include "legate.h" +#include "task_hello.h" + +namespace hello { + +legate::LogicalStore iota(legate::LibraryContext* context, size_t size) +{ + auto runtime = legate::Runtime::get_runtime(); + auto task = runtime->create_task(context, task::hello::IOTA); + auto output = runtime->create_store({size}, legate::float32(), true); + auto part = task->declare_partition(); + task->add_output(output, part); + runtime->submit(std::move(task)); + return output; +} + +legate::LogicalStore square(legate::LibraryContext* context, legate::LogicalStore input) +{ + auto runtime = legate::Runtime::get_runtime(); + auto task = runtime->create_task(context, task::hello::SQUARE); + auto output = runtime->create_store(input.extents().data(), legate::float32(), true); + auto part1 = task->declare_partition(); + auto part2 = task->declare_partition(); + + task->add_input(input, part1); + task->add_output(output, part2); + runtime->submit(std::move(task)); + return output; +} + +legate::LogicalStore sum(legate::LibraryContext* context, + legate::LogicalStore input, + const void* bytearray) +{ + auto runtime = legate::Runtime::get_runtime(); + auto task = runtime->create_task(context, task::hello::SUM); + + auto output = runtime->create_store(legate::Scalar(legate::float32(), bytearray)); + auto redop = input.type().find_reduction_operator(legate::ReductionOpKind::ADD); + + auto part1 = task->declare_partition(); + auto part2 = task->declare_partition(); + + task->add_input(input, part1); + task->add_reduction(output, redop, part2); + runtime->submit(std::move(task)); + return output; +} + +float to_scalar(legate::LibraryContext* context, legate::LogicalStore scalar) +{ + auto runtime = legate::Runtime::get_runtime(); + auto p_scalar = scalar.get_physical_store(context); + auto acc = p_scalar->read_accessor(); + float output = static_cast(acc[{0}]); + return output; +} + +TEST(Example, Hello) +{ + legate::Core::perform_registration(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(task::hello::library_name); + + float bytearray = 0; + + auto store = iota(context, 5); + auto storeSquared = square(context, store); + auto storeSummed = sum(context, storeSquared, &bytearray); + float scalar = to_scalar(context, storeSummed); + + ASSERT_EQ(scalar, 55); +} + +} // namespace hello diff --git a/examples/cpp/hello/hello_print.cc b/examples/cpp/hello/hello_print.cc new file mode 100644 index 0000000000..1a7db58098 --- /dev/null +++ b/examples/cpp/hello/hello_print.cc @@ -0,0 +1,54 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "core/mapping/mapping.h" +#include "legate.h" +#include "task_hello.h" + +namespace helloprint { + +void test_print_hello(legate::LibraryContext* context, std::string str) +{ + auto runtime = legate::Runtime::get_runtime(); + auto task = runtime->create_task(context, task::hello::HELLO_WORLD); + + task->add_scalar_arg(legate::Scalar(str)); + runtime->submit(std::move(task)); +} + +void test_print_hellos(legate::LibraryContext* context, std::string str, size_t count) +{ + auto runtime = legate::Runtime::get_runtime(); + auto task = runtime->create_task(context, task::hello::HELLO_WORLD, legate::Shape({count})); + + task->add_scalar_arg(legate::Scalar(str)); + runtime->submit(std::move(task)); +} + +TEST(Example, HelloPrint) +{ + legate::Core::perform_registration(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(task::hello::library_name); + + test_print_hello(context, "Hello World!"); + test_print_hellos(context, "Hello World! x3", 3); +} + +} // namespace helloprint diff --git a/examples/cpp/hello/task_hello.cc b/examples/cpp/hello/task_hello.cc new file mode 100644 index 0000000000..117c4fc4e1 --- /dev/null +++ b/examples/cpp/hello/task_hello.cc @@ -0,0 +1,109 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "task_hello.h" + +namespace task { + +namespace hello { + +Legion::Logger logger(library_name); + +void register_tasks() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + HelloWorldTask::register_variants(context); + IotaTask::register_variants(context); + SquareTask::register_variants(context); + SumTask::register_variants(context); +} + +/*static*/ void HelloWorldTask::cpu_variant(legate::TaskContext& context) +{ + std::string message = context.scalars().at(0).value(); + std::cout << message << std::endl; +} + +/*static*/ void SumTask::cpu_variant(legate::TaskContext& context) +{ + legate::Store& input = context.inputs().at(0); + legate::Rect<1> input_shape = input.shape<1>(); // should be a 1-Dim array + auto in = input.read_accessor(); + + logger.info() << "Sum [" << input_shape.lo << "," << input_shape.hi << "]"; + + float total = 0; + // i is a global index for the complete array + for (size_t i = input_shape.lo; i <= input_shape.hi; ++i) { total += in[i]; } + + /** + The task launch as a whole will return a single value (Store of size 1) + to the caller. However, each point task gets a separate Store of the + same size as the result, to reduce into. This "local accumulator" will + be initialized by the runtime, and all we need to do is call .reduce() + to add our local contribution. After all point tasks return, the runtime + will make sure to combine all their buffers into the single final result. + */ + using Reduce = Legion::SumReduction; + legate::Store& output = context.reductions().at(0); + auto sum = output.reduce_accessor(); + // Best-practice is to validate types + assert(output.code() == legate::Type::Code::FLOAT32); + assert(output.dim() == 1); + assert(output.shape<1>() == legate::Rect<1>(0, 0)); + sum.reduce(0, total); +} + +/*static*/ void SquareTask::cpu_variant(legate::TaskContext& context) +{ + legate::Store& output = context.outputs().at(0); + // Best-practice to validate the store types + assert(output.code() == legate::Type::Code::FLOAT32); + assert(output.dim() == 1); + legate::Rect<1> output_shape = output.shape<1>(); + auto out = output.write_accessor(); + + legate::Store& input = context.inputs().at(0); + // Best-practice to validate the store types + assert(input.code() == legate::Type::Code::FLOAT32); + assert(input.dim() == 1); + legate::Rect<1> input_shape = input.shape<1>(); // should be a 1-Dim array + auto in = input.read_accessor(); + + assert(input_shape == output_shape); + + logger.info() << "Elementwise square [" << output_shape.lo << "," << output_shape.hi << "]"; + + // i is a global index for the complete array + for (size_t i = input_shape.lo; i <= input_shape.hi; ++i) { out[i] = in[i] * in[i]; } +} + +/*static*/ void IotaTask::cpu_variant(legate::TaskContext& context) +{ + legate::Store& output = context.outputs().at(0); + legate::Rect<1> output_shape = output.shape<1>(); + auto out = output.write_accessor(); + + logger.info() << "Iota task [" << output_shape.lo << "," << output_shape.hi << "]"; + + // i is a global index for the complete array + for (size_t i = output_shape.lo; i <= output_shape.hi; ++i) { out[i] = i + 1; } +} + +} // namespace hello + +} // namespace task diff --git a/examples/cpp/hello/task_hello.h b/examples/cpp/hello/task_hello.h new file mode 100644 index 0000000000..c88cebd26b --- /dev/null +++ b/examples/cpp/hello/task_hello.h @@ -0,0 +1,61 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include "core/mapping/mapping.h" +#include "legate.h" + +namespace task { + +namespace hello { + +enum TaskOpCode { + _OP_CODE_BASE = 0, + HELLO_WORLD = 1, + SUM = 2, + SQUARE = 3, + IOTA = 4, +}; + +static const char* library_name = "legate.hello"; +extern Legion::Logger logger; + +void register_tasks(); + +struct HelloWorldTask : public legate::LegateTask { + static const int32_t TASK_ID = HELLO_WORLD; + static void cpu_variant(legate::TaskContext& context); +}; + +struct SumTask : public legate::LegateTask { + static const int32_t TASK_ID = SUM; + static void cpu_variant(legate::TaskContext& context); +}; + +struct SquareTask : public legate::LegateTask { + static const int32_t TASK_ID = SQUARE; + static void cpu_variant(legate::TaskContext& context); +}; + +struct IotaTask : public legate::LegateTask { + static const int32_t TASK_ID = IOTA; + static void cpu_variant(legate::TaskContext& context); +}; + +} // namespace hello + +} // namespace task diff --git a/examples/cpp/io/io.cc b/examples/cpp/io/io.cc new file mode 100644 index 0000000000..1c118b338c --- /dev/null +++ b/examples/cpp/io/io.cc @@ -0,0 +1,395 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include + +#include "core/mapping/mapping.h" +#include "legate.h" +#include "task_io.h" + +namespace legateio { + +class Array { + public: + Array(legate::LibraryContext* context, legate::LogicalStore store) + : context_(context), store_(store) + { + } + + legate::LogicalStore store() { return store_; } + + protected: + legate::LibraryContext* context_; + legate::LogicalStore store_; +}; + +class IOArray : public Array { + public: + using Array::Array; + + void to_file(std::string filename) + { + auto runtime = legate::Runtime::get_runtime(); + auto task = runtime->create_task(context_, task::legateio::WRITE_FILE); + auto part = task->declare_partition(); + + task->add_scalar_arg(legate::Scalar(filename)); + task->add_input(store_, part); + + // Request a broadcasting for the input. Since this is the only store + // argument to the task, Legate will launch a single task from this + // task descriptor. + task->add_constraint(legate::broadcast(part, {0})); + + runtime->submit(std::move(task)); + } + + void to_uneven_tiles(std::string path) + { + int result = mkdir(path.c_str(), 0775); + if (result == -1) { + EXPECT_EQ(errno, EEXIST); + } else { + EXPECT_EQ(result, 0); + } + + auto runtime = legate::Runtime::get_runtime(); + auto task = runtime->create_task(context_, task::legateio::WRITE_UNEVEN_TILES); + auto part = task->declare_partition(); + + task->add_scalar_arg(legate::Scalar(path)); + task->add_input(store_, part); + runtime->submit(std::move(task)); + } + + void to_even_tiles(std::string path, uint32_t tile_shape) + { + int result = mkdir(path.c_str(), 0775); + if (result == -1) { + EXPECT_EQ(errno, EEXIST); + } else { + EXPECT_EQ(result, 0); + } + + auto runtime = legate::Runtime::get_runtime(); + auto store_partition = store_.partition_by_tiling({tile_shape, tile_shape}); + auto launch_shape = store_partition.partition()->color_shape(); + + auto task = runtime->create_task(context_, task::legateio::WRITE_EVEN_TILES, launch_shape); + task->add_input(store_partition); + task->add_scalar_arg(legate::Scalar(path)); + + auto extents = store_.extents(); + EXPECT_EQ(extents.size(), 2); + task->add_scalar_arg( + legate::Scalar(std::vector{(uint32_t)extents[0], (uint32_t)extents[1]})); + task->add_scalar_arg(legate::Scalar(std::vector{tile_shape, tile_shape})); + runtime->submit(std::move(task)); + } +}; + +legate::LogicalStore iota(legate::LibraryContext* context, uint32_t size) +{ + auto runtime = legate::Runtime::get_runtime(); + auto task = runtime->create_task(context, task::legateio::IOTA); + auto output = runtime->create_store({size}, legate::int8(), true); + auto part = task->declare_partition(); + task->add_output(output, part); + runtime->submit(std::move(task)); + return output; +} + +legate::LogicalStore iota_2d(legate::LibraryContext* context, uint32_t size) +{ + auto runtime = legate::Runtime::get_runtime(); + auto task = runtime->create_task(context, task::legateio::IOTA_2D); + auto output = runtime->create_store({size, size}, legate::int8(), true); + auto part = task->declare_partition(); + task->add_output(output, part); + runtime->submit(std::move(task)); + return output; +} + +void array_equal(legate::LibraryContext* context, IOArray c1, IOArray c2) +{ + int8_t bytearray = 1; + auto runtime = legate::Runtime::get_runtime(); + auto task_id = + c1.store().extents().size() == 1 ? task::legateio::EQUAL : task::legateio::EQUAL_2D; + auto task = runtime->create_task(context, task_id); + auto output = runtime->create_store(legate::Scalar(legate::int8(), &bytearray)); + auto redop = c1.store().type().find_reduction_operator(legate::ReductionOpKind::MUL); + + auto part1 = task->declare_partition(); + auto part2 = task->declare_partition(); + auto part3 = task->declare_partition(); + + task->add_input(c1.store(), part1); + task->add_input(c2.store(), part2); + task->add_reduction(output, redop, part3); + runtime->submit(std::move(task)); + + auto p_scalar = output.get_physical_store(context); + auto acc = p_scalar->read_accessor(); + int8_t output_value = static_cast(acc[{0}]); + EXPECT_EQ(output_value, 1); +} + +// TODO: return struct from an interface supporting legate arrays +IOArray read_file(legate::LibraryContext* context, + std::string filename, + std::unique_ptr dtype) +{ + auto runtime = legate::Runtime::get_runtime(); + auto output = runtime->create_store(std::move(dtype), 1); + auto task = runtime->create_task(context, task::legateio::READ_FILE); + auto part = task->declare_partition(); + + task->add_scalar_arg(legate::Scalar(filename)); + task->add_output(output, part); + + runtime->submit(std::move(task)); + return IOArray(context, output); +} + +// TODO: return struct from an interface supporting legate arrays +IOArray read_file_parallel(legate::LibraryContext* context, + std::string filename, + std::unique_ptr dtype, + uint parallelism = 0) +{ + auto runtime = legate::Runtime::get_runtime(); + if (parallelism == 0) { + parallelism = runtime->get_machine().count(legate::mapping::TaskTarget::CPU); + } + + auto output = runtime->create_store(std::move(dtype), 1); + auto task = + runtime->create_task(context, task::legateio::READ_FILE, legate::Shape({parallelism})); + + task->add_scalar_arg(legate::Scalar(filename)); + task->add_output(output); + + runtime->submit(std::move(task)); + return IOArray(context, output); +} + +void _read_header_uneven(std::string path, std::vector& color_shape) +{ + std::ifstream in(path.c_str(), std::ios::binary | std::ios::in); + + uint32_t code; + uint32_t dim; + + in.read(reinterpret_cast(&code), sizeof(uint32_t)); + in.read(reinterpret_cast(&dim), sizeof(uint32_t)); + + EXPECT_EQ(code, static_cast(legate::Type::Code::UINT8)); + + int64_t data; + for (int i = 0; i < dim; i++) { + in.read(reinterpret_cast(&data), sizeof(int64_t)); + color_shape.push_back(data); + } + in.close(); +} + +IOArray read_uneven_tiles(legate::LibraryContext* context, std::string path) +{ + // Read the dataset's header to find the type code and the color shape of + // the partition, which are laid out in the header in the following way: + // + // +-----------+--------+----------+----- + // | type code | # dims | extent 0 | ... + // | (4B) | (4B) | (8B) | + // +-----------+--------+----------+----- + // + std::vector color_shape; + _read_header_uneven(path + "/.header", color_shape); + + // Create a multi-dimensional unbound store + // + // Like 1D unbound stores, the shape of this store is also determined by + // outputs from the tasks. For example, if the store is 2D and there are + // four tasks (0, 0), (0, 1), (1, 0), and (1, 1) that respectively produce + // 2x3, 2x4, 3x3, and 3x4 outputs, the store's shape would be (5, 7) and + // internally partitioned in the following way: + // + // 0 1 2 3 4 5 6 + // +--------+------------+ + // 0 | (0, 0) | (0, 1) | + // 1 | | | + // +--------+------------+ + // 2 | | | + // 3 | (1, 0) | (1, 1) | + // 4 | | | + // +--------+------------+ + // + auto runtime = legate::Runtime::get_runtime(); + auto output = runtime->create_store(legate::int8(), color_shape.size()); + auto task = + runtime->create_task(context, task::legateio::READ_UNEVEN_TILES, legate::Shape(color_shape)); + + task->add_output(output); + task->add_scalar_arg(legate::Scalar(path)); + runtime->submit(std::move(task)); + + return IOArray(context, output); +} + +void _read_header_even(std::string path, + std::vector& shape, + std::vector& tile_shape) +{ + std::ifstream in(path.c_str(), std::ios::binary | std::ios::in); + + uint32_t code; + uint32_t dim; + + in.read(reinterpret_cast(&code), sizeof(uint32_t)); + in.read(reinterpret_cast(&dim), sizeof(uint32_t)); + + EXPECT_EQ(code, static_cast(legate::Type::Code::UINT8)); + + uint32_t data; + for (int i = 0; i < dim; i++) { + in.read(reinterpret_cast(&data), sizeof(uint32_t)); + shape.push_back(data); + } + for (int i = 0; i < dim; i++) { + in.read(reinterpret_cast(&data), sizeof(uint32_t)); + tile_shape.push_back(data); + } + in.close(); +} + +IOArray read_even_tiles(legate::LibraryContext* context, std::string path) +{ + // Read the dataset's header to find the type code, the array's shape, and + // the tile shape. The following shows the header's format: + // + // +-----------+--------+----------+-----+----------+----- + // | | | shape | | tile | + // | type code | # dims | extent 0 | ... | extent 0 | ... + // | (4B) | (4B) | (4B) | | (4B) | + // +-----------+--------+----------+-----+----------+----- + // + std::vector shape; + std::vector tile_shape; + _read_header_even(path + "/.header", shape, tile_shape); + + auto runtime = legate::Runtime::get_runtime(); + auto output = runtime->create_store(shape, legate::int8()); + auto output_partition = output.partition_by_tiling(tile_shape); + auto launch_shape = output_partition.partition()->color_shape(); + auto task = runtime->create_task(context, task::legateio::READ_EVEN_TILES, launch_shape); + + task->add_output(output_partition); + task->add_scalar_arg(legate::Scalar(path)); + runtime->submit(std::move(task)); + + // Unlike AutoTask, manually parallelized tasks don't update the "key" + // partition for their outputs. + output.set_key_partition(output_partition.partition().get()); + + return IOArray(context, output); +} + +TEST(Example, SingleFileIO) +{ + legate::Core::perform_registration(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(task::legateio::library_name); + + int n = 10; + std::string filename = "test.dat"; + + auto c1_store = iota(context, n); + IOArray c1(context, c1_store); + + // Dump the IOArray to a file + c1.to_file(filename); + + // Issue an execution fence to make sure the writer task finishes before + // any of the downstream tasks start. + runtime->issue_execution_fence(); + + // Read the file into a IOArray + IOArray c2 = read_file(context, filename, legate::int8()); + c2.to_file(filename); + array_equal(context, c1, c2); + + // Read the file into a IOArray with a fixed degree of parallelism + IOArray c3 = read_file_parallel(context, filename, legate::int8(), 2); + array_equal(context, c1, c3); + + // Read the file into a IOArray with the library-chosen degree of + // parallelism + IOArray c4 = read_file_parallel(context, filename, legate::int8()); + array_equal(context, c1, c4); +} + +TEST(Example, EvenTilesIO) +{ + legate::Core::perform_registration(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(task::legateio::library_name); + + std::string dataset_name = "even_datafiles"; + uint32_t store_shape = 8; + uint32_t tile_shape = 3; + + auto c1_store = iota_2d(context, store_shape); + IOArray c1(context, c1_store); + + // Dump the IOArray to a dataset of even tiles + c1.to_even_tiles(dataset_name, tile_shape); + + runtime->issue_execution_fence(true); + + // Read the dataset into an IOArray + IOArray c2 = read_even_tiles(context, dataset_name); + // TODO: Ideally we collapse the shapes to be the same + array_equal(context, c1, c2); +} + +TEST(Example, UnevenTilesIO) +{ + legate::Core::perform_registration(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(task::legateio::library_name); + + std::string dataset_name = "uneven_datafiles"; + uint32_t store_shape = 8; + + auto c1_store = iota_2d(context, store_shape); + IOArray c1(context, c1_store); + + // Dump the IOArray to a dataset of even tiles + c1.to_uneven_tiles(dataset_name); + + runtime->issue_execution_fence(true); + + // Read the dataset into an IOArray + IOArray c2 = read_uneven_tiles(context, dataset_name); + array_equal(context, c1, c2); +} + +} // namespace legateio diff --git a/examples/cpp/io/task_io.cc b/examples/cpp/io/task_io.cc new file mode 100644 index 0000000000..d6154db7db --- /dev/null +++ b/examples/cpp/io/task_io.cc @@ -0,0 +1,465 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include + +#include "task_io.h" + +namespace fs = std::filesystem; + +namespace task { + +namespace legateio { + +Legion::Logger logger(library_name); + +void register_tasks() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + ReadEvenTilesTask::register_variants(context); + ReadFileTask::register_variants(context); + ReadUnevenTilesTask::register_variants(context); + WriteEvenTilesTask::register_variants(context); + WriteFileTask::register_variants(context); + WriteUnevenTilesTask::register_variants(context); + IotaTask::register_variants(context); + Iota2DTask::register_variants(context); + EqualTask::register_variants(context); + EqualTask2D::register_variants(context); +} + +namespace utils { + +struct write_util_fn { + template + void operator()(const legate::Store& store, const fs::path& path) + { + using VAL = legate::legate_type_of; + + auto shape = store.shape(); + auto empty = shape.empty(); + auto extents = + empty ? legate::Point::ZEROES() : shape.hi - shape.lo + legate::Point::ONES(); + + int32_t dim = DIM; + int32_t code = store.code(); + + logger.debug() << "Write a sub-array " << shape << " to " << path; + + std::ofstream out(path, std::ios::binary | std::ios::out | std::ios::trunc); + // Each file for a chunk starts with the extents + for (int32_t idx = 0; idx < DIM; ++idx) + out.write(reinterpret_cast(&extents[idx]), sizeof(legate::coord_t)); + + if (empty) return; + auto acc = store.read_accessor(); + // The iteration order here should be consistent with that in the reader task, otherwise + // the read data can be transposed. + for (legate::PointInRectIterator it(shape, false /*fortran_order*/); it.valid(); ++it) { + auto ptr = acc.ptr(*it); + out.write(reinterpret_cast(ptr), sizeof(VAL)); + } + } +}; + +std::filesystem::path get_unique_path_for_task_index(const legate::TaskContext& context, + int32_t ndim, + const std::string& dirname) +{ + auto task_index = context.get_task_index(); + // If this was a single task, we use (0, ..., 0) for the task index + if (context.is_single_task()) { + task_index = legate::DomainPoint(); + task_index.dim = ndim; + } + + std::stringstream ss; + for (int32_t idx = 0; idx < task_index.dim; ++idx) { + if (idx != 0) ss << "."; + ss << task_index[idx]; + } + auto filename = ss.str(); + + return fs::path(dirname) / filename; +} + +void write_to_file(legate::TaskContext& task_context, + const std::string& dirname, + const legate::Store& store) +{ + auto path = get_unique_path_for_task_index(task_context, store.dim(), dirname); + // double_dispatch converts the first two arguments to non-type template arguments + legate::double_dispatch(store.dim(), store.code(), write_util_fn{}, store, path); +} + +} // namespace utils + +struct read_even_fn { + template + void operator()(legate::Store& output, const fs::path& path) + { + using VAL = legate::legate_type_of; + + legate::Rect shape = output.shape(); + + if (shape.empty()) return; + + std::ifstream in(path, std::ios::binary | std::ios::in); + + legate::Point extents; + for (int32_t idx = 0; idx < DIM; ++idx) + in.read(reinterpret_cast(&extents[idx]), sizeof(legate::coord_t)); + + // Since the shape is already fixed on the Python side, the sub-store's extents should be the + // same as what's stored in the file + assert(shape.hi - shape.lo + legate::Point::ONES() == extents); + + logger.debug() << "Read a sub-array of rect " << shape << " from " << path; + + auto acc = output.write_accessor(); + for (legate::PointInRectIterator it(shape, false /*fortran_order*/); it.valid(); ++it) { + auto ptr = acc.ptr(*it); + in.read(reinterpret_cast(ptr), sizeof(VAL)); + } + } +}; + +/*static*/ void ReadEvenTilesTask::cpu_variant(legate::TaskContext& context) +{ + auto dirname = context.scalars().at(0).value(); + auto& output = context.outputs().at(0); + + auto path = utils::get_unique_path_for_task_index(context, output.dim(), dirname); + // double_dispatch converts the first two arguments to non-type template arguments + legate::double_dispatch(output.dim(), output.code(), read_even_fn{}, output, path); +} + +struct read_fn { + template + void operator()(legate::Store& output, + const std::string& filename, + int64_t my_id, + int64_t num_readers) + { + using VAL = legate::legate_type_of; + + int64_t code; + size_t size; + + // All reader tasks need to read the header to figure out the total number of elements + std::ifstream in(filename, std::ios::binary | std::ios::in); + in.read(reinterpret_cast(&code), sizeof(int64_t)); + in.read(reinterpret_cast(&size), sizeof(size_t)); + + if (static_cast(code) != CODE) { + logger.error() << "Type mismatch: " << CODE << " != " << code; + LEGATE_ABORT; + } + + // Compute the absolute offsets to the section that this reader task + // is supposed to read from the file + int64_t my_lo = my_id * size / num_readers; + int64_t my_hi = std::min((my_id + 1) * size / num_readers, size); + + // Then compute the extent for the output and create the output buffer to populate + int64_t my_ext = my_hi - my_lo; + auto buf = output.create_output_buffer(legate::Point<1>(my_ext)); + + // Skip to the right offset where the data assigned to this reader task actually starts + if (my_lo != 0) in.seekg(my_lo * sizeof(VAL), std::ios_base::cur); + for (int64_t idx = 0; idx < my_ext; ++idx) { + auto ptr = buf.ptr(legate::Point<1>(idx)); + in.read(reinterpret_cast(ptr), sizeof(VAL)); + } + + // Finally, bind the output buffer to the store + // + // Some minor details about unbound stores: + // + // 1) Though this example binds the buffer after it's populated, it's not strictly necessary. + // In fact, the create_output_buffer call takes an optional boolean argument that binds + // the created buffer immediately. + // 2) The bind_data call takes the extents of data as an argument, just in case the buffer + // is actually bigger than extents of the actual data. + // + output.bind_data(buf, legate::Point<1>(my_ext)); + } +}; + +/*static*/ void ReadFileTask::cpu_variant(legate::TaskContext& context) +{ + auto filename = context.scalars().at(0).value(); + auto& output = context.outputs().at(0); + + // The task context contains metadata about the launch so each reader task can figure out + // which part of the file it needs to read into the output. + int64_t my_id = context.is_single_task() ? 0 : context.get_task_index()[0]; + int64_t num_readers = context.is_single_task() ? 1 : context.get_launch_domain().get_volume(); + logger.debug() << "Read " << filename << " (" << my_id + 1 << "/" << num_readers << ")"; + + // type_dispatch converts the first argument to a non-type template argument + legate::type_dispatch(output.code(), read_fn{}, output, filename, my_id, num_readers); +} + +struct read_uneven_fn { + template + void operator()(legate::Store& output, const fs::path& path) + { + using VAL = legate::legate_type_of; + + std::ifstream in(path, std::ios::binary | std::ios::in); + + // Read the header of each file to extract the extents + legate::Point extents; + for (int32_t idx = 0; idx < DIM; ++idx) + in.read(reinterpret_cast(&extents[idx]), sizeof(legate::coord_t)); + + logger.debug() << "Read a sub-array of extents " << extents << " from " << path; + + // Use the extents to create an output buffer + auto buf = output.create_output_buffer(extents); + legate::Rect shape(legate::Point::ZEROES(), extents - legate::Point::ONES()); + if (!shape.empty()) + // Read the file data. The iteration order here should be the same as in the writer task + for (legate::PointInRectIterator it(shape, false /*fortran_order*/); it.valid(); ++it) { + auto ptr = buf.ptr(*it); + in.read(reinterpret_cast(ptr), sizeof(VAL)); + } + + // Finally, bind the output buffer to the store + output.bind_data(buf, extents); + } +}; + +/*static*/ void ReadUnevenTilesTask::cpu_variant(legate::TaskContext& context) +{ + auto dirname = context.scalars().at(0).value(); + auto& output = context.outputs().at(0); + + auto path = utils::get_unique_path_for_task_index(context, output.dim(), dirname); + // double_dispatch converts the first two arguments to non-type template arguments + legate::double_dispatch(output.dim(), output.code(), read_uneven_fn{}, output, path); +} + +void write_header(std::ofstream& out, + legate::Type::Code type_code, + const legate::Span& shape, + const legate::Span& tile_shape) +{ + assert(shape.size() == tile_shape.size()); + int32_t dim = shape.size(); + // Dump the type code, the array's shape and the tile shape to the header + out.write(reinterpret_cast(&type_code), sizeof(int32_t)); + out.write(reinterpret_cast(&dim), sizeof(int32_t)); + for (auto& v : shape) out.write(reinterpret_cast(&v), sizeof(int32_t)); + for (auto& v : tile_shape) out.write(reinterpret_cast(&v), sizeof(int32_t)); +} + +/*static*/ void WriteEvenTilesTask::cpu_variant(legate::TaskContext& context) +{ + auto dirname = context.scalars().at(0).value(); + legate::Span shape = context.scalars().at(1).values(); + legate::Span tile_shape = context.scalars().at(2).values(); + auto& input = context.inputs().at(0); + + auto launch_domain = context.get_launch_domain(); + auto task_index = context.get_task_index(); + auto is_first_task = context.is_single_task() || task_index == launch_domain.lo(); + + if (is_first_task) { + auto header = fs::path(dirname) / ".header"; + logger.debug() << "Write to " << header; + std::ofstream out(header, std::ios::binary | std::ios::out | std::ios::trunc); + write_header(out, input.code(), shape, tile_shape); + } + + utils::write_to_file(context, dirname, input); +} + +struct write_fn { + template + void operator()(const legate::Store& input, const std::string& filename) + { + using VAL = legate::legate_type_of; + + auto shape = input.shape<1>(); + auto code = input.code(); + size_t size = shape.volume(); + + // Store the type code and the number of elements in the array at the beginning of the file + std::ofstream out(filename, std::ios::binary | std::ios::out | std::ios::trunc); + out.write(reinterpret_cast(&code), sizeof(int64_t)); + out.write(reinterpret_cast(&size), sizeof(size_t)); + + auto acc = input.read_accessor(); + for (legate::PointInRectIterator<1> it(shape); it.valid(); ++it) { + auto ptr = acc.ptr(*it); + out.write(reinterpret_cast(ptr), sizeof(VAL)); + } + } +}; + +/*statis*/ void WriteFileTask::cpu_variant(legate::TaskContext& context) +{ + auto filename = context.scalars().at(0).value(); + auto& input = context.inputs().at(0); + logger.debug() << "Write to " << filename; + + legate::type_dispatch(input.code(), write_fn{}, input, filename); +} + +struct header_write_fn { + template + void operator()(std::ofstream& out, + const legate::Domain& launch_domain, + legate::Type::Code type_code) + { + legate::Rect rect(launch_domain); + auto extents = rect.hi - rect.lo + legate::Point::ONES(); + + // The header contains the type code and the launch shape + out.write(reinterpret_cast(&type_code), sizeof(int32_t)); + out.write(reinterpret_cast(&launch_domain.dim), sizeof(int32_t)); + for (int32_t idx = 0; idx < DIM; ++idx) + out.write(reinterpret_cast(&extents[idx]), sizeof(legate::coord_t)); + } +}; + +/*statis*/ void WriteUnevenTilesTask::cpu_variant(legate::TaskContext& context) +{ + auto dirname = context.scalars().at(0).value(); + auto& input = context.inputs().at(0); + + auto launch_domain = context.get_launch_domain(); + auto task_index = context.get_task_index(); + auto is_first_task = context.is_single_task() || task_index == launch_domain.lo(); + + // Only the first task needs to dump the header + if (is_first_task) { + // When the task is a single task, we create a launch domain of volume 1 + if (context.is_single_task()) { + launch_domain = legate::Domain(); + launch_domain.dim = input.dim(); + } + + auto header = fs::path(dirname) / ".header"; + logger.debug() << "Write to " << header; + std::ofstream out(header, std::ios::binary | std::ios::out | std::ios::trunc); + legate::dim_dispatch(launch_domain.dim, header_write_fn{}, out, launch_domain, input.code()); + } + + utils::write_to_file(context, dirname, input); +} + +/*static*/ void IotaTask::cpu_variant(legate::TaskContext& context) +{ + legate::Store& output = context.outputs().at(0); + legate::Rect<1> output_shape = output.shape<1>(); + auto out = output.write_accessor(); + + logger.debug() << "Iota task [" << output_shape.lo << "," << output_shape.hi << "]"; + + // i is a global index for the complete array + for (size_t i = output_shape.lo; i <= output_shape.hi; ++i) { out[i] = i + 1; } +} + +/*static*/ void Iota2DTask::cpu_variant(legate::TaskContext& context) +{ + legate::Store& output = context.outputs().at(0); + legate::Rect<2> output_shape = output.shape<2>(); + auto out = output.write_accessor(); + + logger.debug() << "Iota 2d task [" << output_shape.lo << "," << output_shape.hi << "]"; + + // i is a global index for the complete array + for (auto i = output_shape.lo[0]; i <= output_shape.hi[0]; ++i) { + for (auto j = output_shape.lo[1]; i <= output_shape.hi[1]; ++i) { + out[i][j] = i * output_shape.hi[0] + j; + } + } +} + +/*static*/ void EqualTask::cpu_variant(legate::TaskContext& context) +{ + legate::Store& input0 = context.inputs().at(0); + legate::Store& input1 = context.inputs().at(1); + + legate::Rect<1> input0_shape = input0.shape<1>(); + legate::Rect<1> input1_shape = input1.shape<1>(); + + auto in0 = input0.read_accessor(); + auto in1 = input1.read_accessor(); + + logger.debug() << "Equal task in0 [" << input0_shape.lo << "," << input0_shape.hi << "], in1 [" + << input1_shape.lo << "," << input1_shape.hi << "]"; + + using Reduce = Legion::ProdReduction; + legate::Store& output = context.reductions().at(0); + auto out = output.reduce_accessor(); + + if (input0_shape.lo != input1_shape.lo || input0_shape.hi != input1_shape.hi) { + logger.debug() << "Shape different"; + out.reduce(0, false); + return; + } + + for (size_t i = input0_shape.lo; i <= input0_shape.hi; ++i) { + if (in0[i] != in1[i]) { + logger.debug() << "Value different"; + out.reduce(0, false); + return; + } + } + out.reduce(0, true); +} + +/*static*/ void EqualTask2D::cpu_variant(legate::TaskContext& context) +{ + legate::Store& input0 = context.inputs().at(0); + legate::Store& input1 = context.inputs().at(1); + + legate::Rect<2> input0_shape = input0.shape<2>(); + legate::Rect<2> input1_shape = input1.shape<2>(); + + auto in0 = input0.read_accessor(); + auto in1 = input1.read_accessor(); + + logger.debug() << "Equal task 2D in0 [" << input0_shape.lo[0] << "," << input0_shape.lo[1] + << "] to [" << input0_shape.hi[0] << "," << input0_shape.hi[1] << "]"; + logger.debug() << "Equal task 2D in1 [" << input1_shape.lo[0] << "," << input1_shape.lo[1] + << "] to [" << input1_shape.hi[0] << "," << input1_shape.hi[1] << "]"; + + using Reduce = Legion::ProdReduction; + legate::Store& output = context.reductions().at(0); + auto out = output.reduce_accessor(); + + for (size_t i = input1_shape.lo[0]; i <= input1_shape.hi[0]; ++i) { + for (size_t j = input1_shape.lo[1]; j <= input1_shape.hi[1]; ++j) { + if (in0[i][j] != in1[i][j]) { + logger.debug() << "Value different"; + out.reduce(0, false); + return; + } + } + } + out.reduce(0, true); +} + +} // namespace legateio + +} // namespace task diff --git a/examples/cpp/io/task_io.h b/examples/cpp/io/task_io.h new file mode 100644 index 0000000000..9554384a1e --- /dev/null +++ b/examples/cpp/io/task_io.h @@ -0,0 +1,97 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include "core/mapping/mapping.h" +#include "legate.h" + +namespace task { + +namespace legateio { + +enum LegateIOOpCode { + _OP_CODE_BASE = 0, + READ_EVEN_TILES = 1, + READ_FILE = 2, + READ_UNEVEN_TILES = 3, + WRITE_EVEN_TILES = 4, + WRITE_FILE = 5, + WRITE_UNEVEN_TILES = 6, + IOTA = 7, + IOTA_2D = 8, + EQUAL = 9, + EQUAL_2D = 10, +}; + +static const char* library_name = "legateio"; +extern Legion::Logger logger; + +void register_tasks(); + +struct ReadEvenTilesTask : public legate::LegateTask { + static const int32_t TASK_ID = READ_EVEN_TILES; + static void cpu_variant(legate::TaskContext& context); +}; + +struct ReadFileTask : public legate::LegateTask { + static const int32_t TASK_ID = READ_FILE; + static void cpu_variant(legate::TaskContext& context); +}; + +struct ReadUnevenTilesTask : public legate::LegateTask { + static const int32_t TASK_ID = READ_UNEVEN_TILES; + static void cpu_variant(legate::TaskContext& context); +}; + +struct WriteEvenTilesTask : public legate::LegateTask { + static const int32_t TASK_ID = WRITE_EVEN_TILES; + static void cpu_variant(legate::TaskContext& context); +}; + +struct WriteFileTask : public legate::LegateTask { + static const int32_t TASK_ID = WRITE_FILE; + static void cpu_variant(legate::TaskContext& context); +}; + +struct WriteUnevenTilesTask : public legate::LegateTask { + static const int32_t TASK_ID = WRITE_UNEVEN_TILES; + static void cpu_variant(legate::TaskContext& context); +}; + +struct IotaTask : public legate::LegateTask { + static const int32_t TASK_ID = IOTA; + static void cpu_variant(legate::TaskContext& context); +}; + +struct Iota2DTask : public legate::LegateTask { + static const int32_t TASK_ID = IOTA_2D; + static void cpu_variant(legate::TaskContext& context); +}; + +struct EqualTask : public legate::LegateTask { + static const int32_t TASK_ID = EQUAL; + static void cpu_variant(legate::TaskContext& context); +}; + +struct EqualTask2D : public legate::LegateTask { + static const int32_t TASK_ID = EQUAL_2D; + static void cpu_variant(legate::TaskContext& context); +}; + +} // namespace legateio + +} // namespace task diff --git a/examples/cpp/main.cc b/examples/cpp/main.cc new file mode 100644 index 0000000000..747c335690 --- /dev/null +++ b/examples/cpp/main.cc @@ -0,0 +1,42 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include "legate.h" + +class Environment : public ::testing::Environment { + public: + Environment(int argc, char** argv) : argc_(argc), argv_(argv) {} + + void SetUp() override + { + legate::initialize(argc_, argv_); + EXPECT_EQ(legate::start(argc_, argv_), 0); + } + void TearDown() override { EXPECT_EQ(legate::wait_for_shutdown(), 0); } + + private: + int argc_; + char** argv_; +}; + +int main(int argc, char** argv) +{ + ::testing::InitGoogleTest(&argc, argv); + ::testing::AddGlobalTestEnvironment(new Environment(argc, argv)); + + return RUN_ALL_TESTS(); +} diff --git a/examples/cpp/run.sh b/examples/cpp/run.sh new file mode 100755 index 0000000000..1717b0a8ed --- /dev/null +++ b/examples/cpp/run.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +cd build +ctest --output-on-failure diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index 0b7400aaa7..a16542f4da 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -95,6 +95,11 @@ std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) return impl_->get_physical_store(context); } +void LogicalStore::set_key_partition(const Partition* partition) +{ + impl_->set_key_partition(partition); +} + LogicalStorePartition::LogicalStorePartition(std::shared_ptr&& impl) : impl_(std::forward(impl)) { diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index 97dd0192ac..a232931fc6 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -78,6 +78,9 @@ class LogicalStore { public: std::shared_ptr get_physical_store(LibraryContext* context); + public: + void set_key_partition(const Partition* partition); + public: std::shared_ptr impl() const { return impl_; } diff --git a/src/core/partitioning/partition.h b/src/core/partitioning/partition.h index 6d6abb8fc8..859f0f89d6 100644 --- a/src/core/partitioning/partition.h +++ b/src/core/partitioning/partition.h @@ -65,6 +65,11 @@ struct Partition { public: virtual std::string to_string() const = 0; + + public: + virtual const Shape& tile_shape() const = 0; + virtual const Shape& color_shape() const = 0; + virtual const Shape& offsets() const = 0; }; class NoPartition : public Partition { @@ -94,6 +99,20 @@ class NoPartition : public Partition { public: virtual std::string to_string() const override; + + public: + const Shape& tile_shape() const override + { + throw std::invalid_argument("Partition kind doesn't support tile_shape"); + } + const Shape& color_shape() const override + { + throw std::invalid_argument("Partition kind doesn't support color_shape"); + } + const Shape& offsets() const override + { + throw std::invalid_argument("Partition kind doesn't support offsets"); + } }; class Tiling : public Partition { @@ -132,9 +151,9 @@ class Tiling : public Partition { virtual std::string to_string() const override; public: - const Shape& tile_shape() const { return tile_shape_; } - const Shape& color_shape() const { return color_shape_; } - const Shape& offsets() const { return offsets_; } + const Shape& tile_shape() const override { return tile_shape_; } + const Shape& color_shape() const override { return color_shape_; } + const Shape& offsets() const override { return offsets_; } private: Shape tile_shape_; diff --git a/src/legate.h b/src/legate.h index a224137df8..e1d92fee50 100644 --- a/src/legate.h +++ b/src/legate.h @@ -30,6 +30,7 @@ #include "core/data/store.h" #include "core/legate_c.h" #include "core/partitioning/constraint.h" +#include "core/partitioning/partition.h" #include "core/runtime/operation.h" #include "core/runtime/runtime.h" #include "core/task/registrar.h" diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt index 12d788bb3f..ef89bf6960 100644 --- a/tests/cpp/CMakeLists.txt +++ b/tests/cpp/CMakeLists.txt @@ -37,9 +37,10 @@ enable_testing() file(GLOB main_SRC ${PROJECT_SOURCE_DIR}/main.cc) file(GLOB integration_SRC ${PROJECT_SOURCE_DIR}/integration/*.cc) +file(GLOB tasks_SRC ${PROJECT_SOURCE_DIR}/integration/tasks/*.cc) file(GLOB unit_SRC ${PROJECT_SOURCE_DIR}/unit/*.cc) -add_executable(cpp_tests ${main_SRC} ${integration_SRC} ${unit_SRC}) +add_executable(cpp_tests ${main_SRC} ${tasks_SRC} ${integration_SRC} ${unit_SRC}) target_link_libraries(cpp_tests PRIVATE legate::core PRIVATE GTest::gtest) diff --git a/tests/cpp/integration/manual_simple.cc b/tests/cpp/integration/manual_simple.cc index 41a499ce9d..e765aa417b 100644 --- a/tests/cpp/integration/manual_simple.cc +++ b/tests/cpp/integration/manual_simple.cc @@ -18,44 +18,14 @@ #include "core/mapping/mapping.h" #include "legate.h" +#include "tasks/task_simple.h" namespace manualsimple { -static const char* library_name = "manual"; -static legate::Logger logger(library_name); - -enum TaskIDs { - HELLO = 0, -}; - -struct HelloTask : public legate::LegateTask { - static const int32_t TASK_ID = HELLO; - static void cpu_variant(legate::TaskContext& context); -}; - -void register_tasks() -{ - auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->create_library(library_name); - HelloTask::register_variants(context); -} - -/*static*/ void HelloTask::cpu_variant(legate::TaskContext& context) -{ - auto& output = context.outputs()[0]; - auto shape = output.shape<2>(); - - if (shape.empty()) return; - - auto acc = output.write_accessor(shape); - for (legate::PointInRectIterator<2> it(shape); it.valid(); ++it) - acc[*it] = (*it)[0] + (*it)[1] * 1000; -} - void test_auto_task(legate::LibraryContext* context, legate::LogicalStore store) { auto runtime = legate::Runtime::get_runtime(); - auto task = runtime->create_task(context, HELLO); + auto task = runtime->create_task(context, task::simple::HELLO); auto part = task->declare_partition(); task->add_output(store, part); runtime->submit(std::move(task)); @@ -64,7 +34,7 @@ void test_auto_task(legate::LibraryContext* context, legate::LogicalStore store) void test_manual_task(legate::LibraryContext* context, legate::LogicalStore store) { auto runtime = legate::Runtime::get_runtime(); - auto task = runtime->create_task(context, HELLO, {3, 3}); + auto task = runtime->create_task(context, task::simple::HELLO, {3, 3}); auto part = store.partition_by_tiling({2, 2}); task->add_output(part); runtime->submit(std::move(task)); @@ -78,15 +48,15 @@ void print_store(legate::LibraryContext* context, legate::LogicalStore store) auto shape = p_store->shape<2>(); std::stringstream ss; for (legate::PointInRectIterator<2> it(shape); it.valid(); ++it) ss << acc[*it] << " "; - logger.print() << ss.str(); + task::simple::logger.print() << ss.str(); } TEST(Integration, ManualSimple) { - legate::Core::perform_registration(); + legate::Core::perform_registration(); auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library(library_name); + auto context = runtime->find_library(task::simple::library_name); auto store = runtime->create_store({5, 5}, legate::int64()); test_auto_task(context, store); diff --git a/tests/cpp/integration/multi_scalar_out.cc b/tests/cpp/integration/multi_scalar_out.cc index e6466b7983..ef4f3f2ec4 100644 --- a/tests/cpp/integration/multi_scalar_out.cc +++ b/tests/cpp/integration/multi_scalar_out.cc @@ -18,65 +18,16 @@ #include "core/mapping/mapping.h" #include "legate.h" +#include "tasks/task_simple.h" namespace multiscalarout { -static const char* library_name = "multi_scalar"; -static legate::Logger logger(library_name); - -enum TaskIDs { - WRITER = 0, - REDUCER = 1, -}; - -struct WriterTask : public legate::LegateTask { - static const int32_t TASK_ID = WRITER; - static void cpu_variant(legate::TaskContext& context); -}; - -struct ReducerTask : public legate::LegateTask { - static const int32_t TASK_ID = REDUCER; - static void cpu_variant(legate::TaskContext& context); -}; - -void register_tasks() -{ - auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->create_library(library_name); - WriterTask::register_variants(context); - ReducerTask::register_variants(context); -} - -/*static*/ void WriterTask::cpu_variant(legate::TaskContext& context) -{ - auto& output1 = context.outputs()[0]; - auto& output2 = context.outputs()[1]; - - auto acc1 = output1.write_accessor(); - auto acc2 = output2.write_accessor(); - - acc1[{0, 0}] = 10; - acc2[{0, 0, 0}] = 20; -} - -/*static*/ void ReducerTask::cpu_variant(legate::TaskContext& context) -{ - auto& red1 = context.reductions()[0]; - auto& red2 = context.reductions()[1]; - - auto acc1 = red1.reduce_accessor, true, 2>(); - auto acc2 = red2.reduce_accessor, true, 3>(); - - acc1[{0, 0}].reduce(10); - acc2[{0, 0, 0}].reduce(2); -} - void test_writer_auto(legate::LibraryContext* context, legate::LogicalStore scalar1, legate::LogicalStore scalar2) { auto runtime = legate::Runtime::get_runtime(); - auto task = runtime->create_task(context, WRITER); + auto task = runtime->create_task(context, task::simple::WRITER); auto part1 = task->declare_partition(); auto part2 = task->declare_partition(); task->add_output(scalar1, part1); @@ -90,7 +41,7 @@ void test_reducer_auto(legate::LibraryContext* context, legate::LogicalStore store) { auto runtime = legate::Runtime::get_runtime(); - auto task = runtime->create_task(context, REDUCER); + auto task = runtime->create_task(context, task::simple::REDUCER); auto part1 = task->declare_partition(); auto part2 = task->declare_partition(); auto part3 = task->declare_partition(); @@ -107,7 +58,7 @@ void test_reducer_manual(legate::LibraryContext* context, legate::LogicalStore scalar2) { auto runtime = legate::Runtime::get_runtime(); - auto task = runtime->create_task(context, REDUCER, legate::Shape({2})); + auto task = runtime->create_task(context, task::simple::REDUCER, legate::Shape({2})); auto redop1 = scalar1.type().find_reduction_operator(legate::ReductionOpKind::ADD); auto redop2 = scalar2.type().find_reduction_operator(legate::ReductionOpKind::MUL); task->add_reduction(scalar1, redop1); @@ -126,15 +77,15 @@ void print_stores(legate::LibraryContext* context, auto acc2 = p_scalar2->read_accessor(); std::stringstream ss; ss << static_cast(acc1[{0, 0}]) << " " << acc2[{0, 0, 0}]; - logger.print() << ss.str(); + task::simple::logger.print() << ss.str(); } TEST(Integration, ManualScalarOut) { - legate::Core::perform_registration(); + legate::Core::perform_registration(); auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library(library_name); + auto context = runtime->find_library(task::simple::library_name); auto scalar1 = runtime->create_store({1, 1}, legate::int8(), true); auto scalar2 = runtime->create_store({1, 1, 1}, legate::int32(), true); diff --git a/tests/cpp/integration/region_manager.cc b/tests/cpp/integration/region_manager.cc index 2f5e236168..b503602fd3 100644 --- a/tests/cpp/integration/region_manager.cc +++ b/tests/cpp/integration/region_manager.cc @@ -17,33 +17,16 @@ #include #include "legate.h" +#include "tasks/task_region_manager.h" namespace region_manager { -namespace { - -static const char* library_name = "test_region_manager"; - -struct TesterTask : public legate::LegateTask { - static const int32_t TASK_ID = 0; - static void cpu_variant(legate::TaskContext& context) {} -}; - -void prepare() -{ - auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->create_library(library_name); - TesterTask::register_variants(context); -} - -} // namespace - TEST(Integration, RegionManager) { - legate::Core::perform_registration(); + legate::Core::perform_registration(); auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library(library_name); + auto context = runtime->find_library(task::region_manager::library_name); auto task = runtime->create_task(context, 0); std::vector stores; diff --git a/tests/cpp/integration/tasks/task_region_manager.cc b/tests/cpp/integration/tasks/task_region_manager.cc new file mode 100644 index 0000000000..4cc1db44de --- /dev/null +++ b/tests/cpp/integration/tasks/task_region_manager.cc @@ -0,0 +1,32 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "task_region_manager.h" + +namespace task { + +namespace region_manager { + +void register_tasks() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + TesterTask::register_variants(context); +} + +} // namespace region_manager + +} // namespace task diff --git a/tests/cpp/integration/tasks/task_region_manager.h b/tests/cpp/integration/tasks/task_region_manager.h new file mode 100644 index 0000000000..d186d06417 --- /dev/null +++ b/tests/cpp/integration/tasks/task_region_manager.h @@ -0,0 +1,41 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include "core/mapping/mapping.h" +#include "legate.h" + +namespace task { + +namespace region_manager { + +enum TaskOpCode { + PROVENANCE = 0, +}; + +static const char* library_name = "test_region_manager"; + +void register_tasks(); + +struct TesterTask : public legate::LegateTask { + static const int32_t TASK_ID = 0; + static void cpu_variant(legate::TaskContext& context) {} +}; + +} // namespace region_manager + +} // namespace task diff --git a/tests/cpp/integration/tasks/task_simple.cc b/tests/cpp/integration/tasks/task_simple.cc new file mode 100644 index 0000000000..41a955670f --- /dev/null +++ b/tests/cpp/integration/tasks/task_simple.cc @@ -0,0 +1,72 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "task_simple.h" + +namespace task { + +namespace simple { + +Legion::Logger logger(library_name); + +void register_tasks() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + HelloTask::register_variants(context); + WriterTask::register_variants(context); + ReducerTask::register_variants(context); +} + +/*static*/ void HelloTask::cpu_variant(legate::TaskContext& context) +{ + auto& output = context.outputs()[0]; + auto shape = output.shape<2>(); + + if (shape.empty()) return; + + auto acc = output.write_accessor(shape); + for (legate::PointInRectIterator<2> it(shape); it.valid(); ++it) + acc[*it] = (*it)[0] + (*it)[1] * 1000; +} + +/*static*/ void WriterTask::cpu_variant(legate::TaskContext& context) +{ + auto& output1 = context.outputs()[0]; + auto& output2 = context.outputs()[1]; + + auto acc1 = output1.write_accessor(); + auto acc2 = output2.write_accessor(); + + acc1[{0, 0}] = 10; + acc2[{0, 0, 0}] = 20; +} + +/*static*/ void ReducerTask::cpu_variant(legate::TaskContext& context) +{ + auto& red1 = context.reductions()[0]; + auto& red2 = context.reductions()[1]; + + auto acc1 = red1.reduce_accessor, true, 2>(); + auto acc2 = red2.reduce_accessor, true, 3>(); + + acc1[{0, 0}].reduce(10); + acc2[{0, 0, 0}].reduce(2); +} + +} // namespace simple + +} // namespace task diff --git a/tests/cpp/integration/tasks/task_simple.h b/tests/cpp/integration/tasks/task_simple.h new file mode 100644 index 0000000000..be17ec11a4 --- /dev/null +++ b/tests/cpp/integration/tasks/task_simple.h @@ -0,0 +1,55 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include "core/mapping/mapping.h" +#include "legate.h" + +namespace task { + +namespace simple { + +enum TaskOpCode { + _OP_CODE_BASE = 0, + HELLO = 1, + WRITER = 2, + REDUCER = 3, +}; + +static const char* library_name = "legate.simple"; +extern Legion::Logger logger; + +void register_tasks(); + +struct HelloTask : public legate::LegateTask { + static const int32_t TASK_ID = HELLO; + static void cpu_variant(legate::TaskContext& context); +}; + +struct WriterTask : public legate::LegateTask { + static const int32_t TASK_ID = WRITER; + static void cpu_variant(legate::TaskContext& context); +}; + +struct ReducerTask : public legate::LegateTask { + static const int32_t TASK_ID = REDUCER; + static void cpu_variant(legate::TaskContext& context); +}; + +} // namespace simple + +} // namespace task diff --git a/tests/cpp/main.cc b/tests/cpp/main.cc index b0c55c83e3..747c335690 100644 --- a/tests/cpp/main.cc +++ b/tests/cpp/main.cc @@ -1,3 +1,19 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + #include #include "legate.h" diff --git a/tests/cpp/unit/store.cc b/tests/cpp/unit/store.cc index 7572a4c0aa..1b60e45584 100644 --- a/tests/cpp/unit/store.cc +++ b/tests/cpp/unit/store.cc @@ -1,3 +1,19 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + #include #include #include From 32f95c1e6001e0125aef839bfd5af0bf635977f6 Mon Sep 17 00:00:00 2001 From: Vladislav Zhurba Date: Wed, 24 May 2023 10:01:47 -0700 Subject: [PATCH 0126/1425] Correct code check for io example * Correct code check for io example See merge request legate/legate.core.internal!33 --- examples/cpp/io/io.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/cpp/io/io.cc b/examples/cpp/io/io.cc index 1c118b338c..f82b5f323e 100644 --- a/examples/cpp/io/io.cc +++ b/examples/cpp/io/io.cc @@ -198,7 +198,7 @@ void _read_header_uneven(std::string path, std::vector& color_shape) in.read(reinterpret_cast(&code), sizeof(uint32_t)); in.read(reinterpret_cast(&dim), sizeof(uint32_t)); - EXPECT_EQ(code, static_cast(legate::Type::Code::UINT8)); + EXPECT_EQ(code, static_cast(legate::Type::Code::INT8)); int64_t data; for (int i = 0; i < dim; i++) { @@ -263,7 +263,7 @@ void _read_header_even(std::string path, in.read(reinterpret_cast(&code), sizeof(uint32_t)); in.read(reinterpret_cast(&dim), sizeof(uint32_t)); - EXPECT_EQ(code, static_cast(legate::Type::Code::UINT8)); + EXPECT_EQ(code, static_cast(legate::Type::Code::INT8)); uint32_t data; for (int i = 0; i < dim; i++) { From d040c037fe1892501ac8f99b3d5f20919e42815a Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 24 May 2023 14:38:38 -0700 Subject: [PATCH 0127/1425] Add docstrings to the undocumented user-facing APIs * Clean up tests * Update docstrings for machine descriptors and trackers * docstrings for partitioning constraints * Enable the rust profiler build * docstrings for the runtime, logical stores, and tasks * Split up runtime.cc into separate files for different managers See merge request legate/legate.core.internal!35 --- legate_core_cpp.cmake | 20 +- src/core/data/logical_store.h | 255 +++++++++++++ src/core/data/logical_store_detail.cc | 1 + src/core/data/store.h | 1 + src/core/mapping/machine.cc | 53 +-- src/core/mapping/machine.h | 234 +++++++++++- src/core/mapping/mapping.h | 3 + src/core/partitioning/constraint.cc | 6 +- src/core/partitioning/constraint.h | 181 +++++++-- src/core/partitioning/partition.cc | 1 + src/core/runtime/context.h | 15 +- src/core/runtime/field_manager.cc | 68 ++++ src/core/runtime/field_manager.h | 47 +++ src/core/runtime/launcher.cc | 13 +- src/core/runtime/launcher.h | 2 + src/core/runtime/machine_manager.cc | 48 +++ src/core/runtime/machine_manager.h | 42 +++ src/core/runtime/operation.cc | 7 +- src/core/runtime/operation.h | 119 ++++++ src/core/runtime/partition_manager.cc | 217 +++++++++++ src/core/runtime/partition_manager.h | 67 ++++ src/core/runtime/provenance_manager.cc | 68 ++++ src/core/runtime/provenance_manager.h | 45 +++ src/core/runtime/region_manager.cc | 50 +++ src/core/runtime/region_manager.h | 61 +++ src/core/runtime/runtime.cc | 494 ++----------------------- src/core/runtime/runtime.h | 235 +++++++----- src/core/runtime/tracker.cc | 70 ++++ src/core/runtime/tracker.h | 91 +++++ src/core/type/type_info.h | 5 +- src/core/utilities/buffer_builder.h | 36 ++ src/legate.h | 5 + tests/cpp/integration/provenance.cc | 13 +- tests/cpp/unit/machine.cc | 10 +- 34 files changed, 1909 insertions(+), 674 deletions(-) create mode 100644 src/core/runtime/field_manager.cc create mode 100644 src/core/runtime/field_manager.h create mode 100644 src/core/runtime/machine_manager.cc create mode 100644 src/core/runtime/machine_manager.h create mode 100644 src/core/runtime/partition_manager.cc create mode 100644 src/core/runtime/partition_manager.h create mode 100644 src/core/runtime/provenance_manager.cc create mode 100644 src/core/runtime/provenance_manager.h create mode 100644 src/core/runtime/region_manager.cc create mode 100644 src/core/runtime/region_manager.h create mode 100644 src/core/runtime/tracker.cc create mode 100644 src/core/runtime/tracker.h diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index f2300647fd..ff6ae28226 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -213,12 +213,18 @@ list(APPEND legate_core_SOURCES src/core/partitioning/partitioner.cc src/core/partitioning/restriction.cc src/core/runtime/context.cc + src/core/runtime/field_manager.cc src/core/runtime/launcher_arg.cc src/core/runtime/launcher.cc + src/core/runtime/machine_manager.cc src/core/runtime/operation.cc + src/core/runtime/partition_manager.cc src/core/runtime/projection.cc + src/core/runtime/provenance_manager.cc + src/core/runtime/region_manager.cc src/core/runtime/req_analyzer.cc src/core/runtime/runtime.cc + src/core/runtime/tracker.cc src/core/runtime/shard.cc src/core/task/registrar.cc src/core/task/return.cc @@ -333,6 +339,7 @@ if (legate_core_BUILD_DOCS) list(APPEND legate_core_DOC_SOURCES # type src/core/type/type_info.h + src/core/type/type_traits.h # task src/core/task/task.h src/core/task/registrar.h @@ -345,19 +352,25 @@ if (legate_core_BUILD_DOCS) src/core/data/buffer.h src/core/utilities/span.h src/core/data/allocator.h + src/core/data/logical_store.h # runtime src/core/runtime/runtime.h src/core/runtime/runtime.inl src/core/runtime/context.h + # operation + src/core/runtime/operation.h + # partitioning + src/core/partitioning/constraint.h # mapping + src/core/mapping/machine.h src/core/mapping/mapping.h src/core/mapping/operation.h # aliases src/core/utilities/typedefs.h # utilities + src/core/runtime/tracker.h src/core/utilities/debug.h src/core/utilities/dispatch.h - src/core/utilities/type_traits.h # main page src/legate.h ) @@ -368,6 +381,8 @@ if (legate_core_BUILD_DOCS) set(DOXYGEN_EXTENSION_MAPPING cu=C++ cuh=C++) set(DOXYGEN_HIDE_UNDOC_MEMBERS YES) set(DOXYGEN_HIDE_UNDOC_CLASSES YES) + set(DOXYGEN_USE_MATHJAX YES) + set(DOXYGEN_MATHJAX_VERSION MathJax_3) set(DOXYGEN_STRIP_FROM_INC_PATH ${CMAKE_SOURCE_DIR}/src) doxygen_add_docs("doxygen_legate" ALL ${legate_core_DOC_SOURCES} @@ -440,11 +455,14 @@ install( install( FILES src/core/runtime/context.h src/core/runtime/context.inl + src/core/runtime/machine_manager.h src/core/runtime/operation.h + src/core/runtime/provenance_manager.h src/core/runtime/resource.h src/core/runtime/projection.h src/core/runtime/runtime.h src/core/runtime/runtime.inl + src/core/runtime/tracker.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/legate/core/runtime) install( diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index a232931fc6..85bc6af1ee 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -26,6 +26,12 @@ #include "core/type/type_info.h" #include "core/utilities/typedefs.h" +/** + * @file + * @brief Class definition for legate::LogicalStore and + * legate::LogicalStorePartition + */ + namespace legate { class BufferBuilder; @@ -43,6 +49,38 @@ class LogicalStorePartition; } // namespace detail +/** + * @ingroup data + * + * @brief A multi-dimensional data container + * + * `LogicalStore` is a multi-dimensional data container for fixed-size elements. Stores are + * internally partitioned and distributed across the system. By default, Legate clients need + * not create nor maintain the partitions explicitly, and the Legate runtime is responsible + * for managing them. Legate clients can control how stores should be partitioned for a given + * task by attaching partitioning constraints to the task (see the constraint module for + * partitioning constraint APIs). + * + * Each logical store object is a logical handle to the data and is not immediately associated + * with a physical allocation. To access the data, a client must `map` the store to a physical + * store (`Store`). A client can map a store by passing it to a task, in which case the task + * body can see the allocation, or calling `get_physical_store`, which gives the client a handle + * to the physical allocation (see `Store` for details about physical stores). + * + * Normally, a logical store gets a fixed shape upon creation. However, there is a special type of + * logical stores called `unbound` stores whose shapes are unknown at creation time. (see `Runtime` + * for the logical store creation API.) The shape of an unbound store is determined by a task that + * first updates the store; upon the submission of the task, the logical store becomes a normal + * store. Passing an unbound store as a read-only argument or requesting a physical store of an + * unbound store are invalid. + * + * One consequence due to the nature of unbound stores is that querying the shape of a previously + * unbound store can block the client's control flow for an obvious reason; to know the shape of + * the logical store whose shape was unknown at creation time, the client must wait until the + * updater task to finish. However, passing a previously unbound store to a downstream operation can + * be non-blocking, as long as the operation requires no changes in the partitioning and mapping for + * the logical store. + */ class LogicalStore { private: friend class Runtime; @@ -58,24 +96,241 @@ class LogicalStore { LogicalStore& operator=(LogicalStore&& other) = default; public: + /** + * @brief Returns the number of dimensions of the store. + * + * @return The number of dimensions + */ int32_t dim() const; + /** + * @brief Returns the element type of the store. + * + * @return Type of elements in the store + */ const Type& type() const; + /** + * @brief Returns the shape of the store. + * + * Flushes the scheduling window if the store is unbound and has no shape assigned. + * + * @return The store's shape + */ const Shape& extents() const; size_t volume() const; + /** + * @brief Indicates whether the store is unbound + * + * @return true The store is unbound + * @return false The store is normal + */ bool unbound() const; + /** + * @brief Indicates whether the store is transformed + * + * @return true The store is transformed + * @return false The store is not transformed + */ bool transformed() const; public: + /** + * @brief Adds an extra dimension to the store. + * + * Value of `extra_dim` decides where a new dimension should be added, and each dimension + * @f$i@f$, where @f$i@f$ >= `extra_dim`, is mapped to dimension @f$i+1@f$ in a returned store. + * A returned store provides a view to the input store where the values are broadcasted along + * the new dimension. + * + * For example, for a 1D store `A` contains `[1, 2, 3]`, `A.promote(0, 2)` yields a store + * equivalent to: + * + * @code{.unparsed} + * [[1, 2, 3], + * [1, 2, 3]] + * @endcode + * + * whereas `A.promote(1, 2)` yields: + * + * @code{.unparsed} + * [[1, 1], + * [2, 2], + * [3, 3]] + * @endcode + * + * @param extra_dim Position for a new dimension + * @param dim_size Extent of the new dimension + * + * @return A new store with an extra dimension + * + * @throw std::invalid_argument When `extra_dim` is not a valid dimension name + */ LogicalStore promote(int32_t extra_dim, size_t dim_size) const; + /** + * @brief Projects out a dimension of the store. + * + * Each dimension @f$@f$, where @f$i@f$ > `dim`, is mapped to dimension @f$i-1@f$ in a returned + * store. A returned store provides a view to the input store where the values are on hyperplane + * @f$x_\mathtt{dim} = \mathtt{index}@f$. + * + * For example, if a 2D store `A` contains `[[1, 2], [3, 4]]`, `A.project(0, 1)` yields a store + * equivalent to `[3, 4]`, whereas `A.project(1, 0)` yields `[1, 3]`. + * + * @param dim Dimension to project out + * @param index Index on the chosen dimension + * + * @return A new store with one fewer dimension + * + * @throw std::invalid_argument If `dim` is not a valid dimension name or `index` is out of bounds + */ LogicalStore project(int32_t dim, int64_t index) const; + /** + * @brief Slices a contiguous sub-section of the store. + * + * For example, consider a 2D store `A`: + * + * @code{.unparsed} + * [[1, 2, 3], + * [4, 5, 6], + * [7, 8, 9]] + * @endcode + * + * A slicing `A.slice(0, legate::Slice(1))` yields + * + * @code{.unparsed} + * [[4, 5, 6], + * [7, 8, 9]] + * @endcode + * + * The result store will look like this on a different slicing call + * `A.slice(1, legate::Slice(legate::Slice::OPEN, 2))`: + * + * @code{.unparsed} + * [[1, 2], + * [4, 5], + * [7, 8]] + * @endcode + * + * Finally, chained slicing calls + * + * @code{.cpp} + * A.slice(0, legate::Slice(1)).slice(1, legate::Slice(legate::Slice::OPEN, 2)) + * @endcode + * + * results in: + * + * @code{.unparsed} + * [[4, 5], + * [7, 8]] + * @endcode + * + * @param dim Dimension to slice + * @param sl Slice descriptor + * + * @return A new store that correponds to the sliced section + * + * @throw std::invalid_argument If `dim` is not a valid dimension name + */ LogicalStore slice(int32_t dim, std::slice sl) const; + /** + * @brief Reorders dimensions of the store. + * + * Dimension `i` of the resulting store is mapped to dimension `axes[i]` of the input store. + * + * For example, for a 3D store `A` + * + * @code{.unparsed} + * [[[1, 2], + * [3, 4]], + * [[5, 6], + * [7, 8]]] + * @endcode + * + * transpose calls `A.transpose({1, 2, 0})` and `A.transpose({2, 1, 0})` yield the following + * stores, respectively: + * + * @code{.unparsed} + * [[[1, 5], + * [2, 6]], + * [[3, 7], + * [4, 8]]] + * @endcode + * + * @code(.unparsed} + * [[[1, 5], + * [3, 7]], + * + * [[2, 6], + * [4, 8]]] + * @endcode + * + * @param axes Mapping from dimensions of the resulting store to those of the input + * + * @return A new store with the dimensions transposed + * + * @throw std::invalid_argument If any of the following happens: 1) The length of `axes` doesn't + * match the store's dimension; 2) `axes` has duplicates; 3) Any axis in `axes` is an invalid + * axis name. + */ LogicalStore transpose(std::vector&& axes) const; + /** + * @brief Delinearizes a dimension into multiple dimensions. + * + * Each dimension @f$i@f$ of the store, where @f$i > @f$`dim`, will be mapped to dimension + * @f$i+N@f$ of the resulting store, where @f$N@f$ is the length of `sizes`. A delinearization + * that does not preserve the size of the store is invalid. + * + * For example, consider a 2D store `A` + * + * @code{.unparsed} + * [[1, 2, 3, 4], + * [5, 6, 7, 8]] + * @endcode + * + * A delinearizing call `A.delinearize(1, {2, 2}))` yields: + * + * @code{.unparsed} + * [[[1, 2], + * [3, 4]], + * + * [[5, 6], + * [7, 8]]] + * @endcode + * + * Unlike other transformations, delinearization is not an affine transformation. Due to this + * nature, delinearized stores can raise `legate::NonInvertibleTransformation` in places where + * they cannot be used. + * + * @param dim Dimension to delinearize + * @param sizes Extents for the resulting dimensions + * + * @return A new store with the chosen dimension delinearized + * + * @throw std::invalid_argument If `dim` is invalid for the store or `sizes` does not preserve + * the extent of the chosen dimenison + */ LogicalStore delinearize(int32_t dim, std::vector&& sizes) const; public: + /** + * @brief Creates a tiled partition of the store + * + * @param tile_shape Shape of tiles + * + * @return A store partition + */ LogicalStorePartition partition_by_tiling(std::vector tile_shape) const; public: + /** + * @brief Creates a physical store for this logical store + * + * This call blocks the client's control flow. And it fetches the data for the whole store on + * a single node. + * + * @param context Library within which the physical store is created + * + * @return A physical store of the logical store + */ std::shared_ptr get_physical_store(LibraryContext* context); public: diff --git a/src/core/data/logical_store_detail.cc b/src/core/data/logical_store_detail.cc index e6ce8c9659..2e16ec57b6 100644 --- a/src/core/data/logical_store_detail.cc +++ b/src/core/data/logical_store_detail.cc @@ -17,6 +17,7 @@ #include "core/data/logical_store_detail.h" #include "core/mapping/machine.h" +#include "core/runtime/partition_manager.h" #include "core/runtime/req_analyzer.h" #include "core/type/type_traits.h" #include "core/utilities/buffer_builder.h" diff --git a/src/core/data/store.h b/src/core/data/store.h index a59fe8717e..15613dfcb4 100644 --- a/src/core/data/store.h +++ b/src/core/data/store.h @@ -269,6 +269,7 @@ class FutureWrapper { /** * @ingroup data + * * @brief A multi-dimensional data container storing task data */ class Store { diff --git a/src/core/mapping/machine.cc b/src/core/mapping/machine.cc index 97002824a6..2e89896a89 100644 --- a/src/core/mapping/machine.cc +++ b/src/core/mapping/machine.cc @@ -117,10 +117,10 @@ std::string ProcessorRange::to_string() const std::pair ProcessorRange::get_node_range() const { if (empty()) throw std::runtime_error("Illegal to get a node range of an empty processor range"); - return std::make_pair(low / per_node_count, high / per_node_count); + return std::make_pair(low / per_node_count, (high + per_node_count - 1) / per_node_count); } -ProcessorRange ProcessorRange::slice(const uint32_t& from, const uint32_t& to) const +ProcessorRange ProcessorRange::slice(uint32_t from, uint32_t to) const { uint32_t new_low = std::min(low + from, high); uint32_t new_high = std::min(low + to, high); @@ -144,7 +144,17 @@ MachineDesc::MachineDesc(const std::map& ranges) for (auto& [target, processor_range] : processor_ranges) if (!processor_range.empty()) { preferred_target = target; - break; + return; + } +} + +MachineDesc::MachineDesc(std::map&& ranges) + : processor_ranges(std::move(ranges)) +{ + for (auto& [target, processor_range] : processor_ranges) + if (!processor_range.empty()) { + preferred_target = target; + return; } } @@ -152,12 +162,12 @@ const ProcessorRange& MachineDesc::processor_range() const { return processor_range(preferred_target); } -static const ProcessorRange empty_processor_range = {0, 0, 1}; +static const ProcessorRange EMPTY_RANGE{}; -const ProcessorRange& MachineDesc::processor_range(const TaskTarget& target) const +const ProcessorRange& MachineDesc::processor_range(TaskTarget target) const { auto finder = processor_ranges.find(target); - if (finder == processor_ranges.end()) { return empty_processor_range; } + if (finder == processor_ranges.end()) { return EMPTY_RANGE; } return finder->second; } @@ -179,15 +189,7 @@ std::vector MachineDesc::valid_targets_except( size_t MachineDesc::count() const { return count(preferred_target); } -size_t MachineDesc::count(const TaskTarget& target) const -{ - auto finder = processor_ranges.find(target); - if (finder == processor_ranges.end()) { - throw std::runtime_error( - "There is no requested target in the MachineDesc or MachineDesc is empty"); - } - return finder->second.count(); -} +size_t MachineDesc::count(TaskTarget target) const { return processor_range(target).count(); } std::string MachineDesc::to_string() const { @@ -208,9 +210,9 @@ void MachineDesc::pack(BufferBuilder& buffer) const } } -MachineDesc MachineDesc::only(const TaskTarget& target) const { return only(std::set({target})); } +MachineDesc MachineDesc::only(TaskTarget target) const { return only(std::vector({target})); } -MachineDesc MachineDesc::only(const std::set& targets) const +MachineDesc MachineDesc::only(const std::vector& targets) const { std::map new_processor_ranges; for (auto t : targets) new_processor_ranges.insert({t, processor_range(t)}); @@ -218,23 +220,22 @@ MachineDesc MachineDesc::only(const std::set& targets) const return MachineDesc(new_processor_ranges); } -MachineDesc MachineDesc::slice(const uint32_t& from, - const uint32_t& to, - const TaskTarget& target) const +MachineDesc MachineDesc::slice(uint32_t from, uint32_t to, TaskTarget target) const { return MachineDesc({{target, processor_range(target).slice(from, to)}}); } -MachineDesc MachineDesc::slice(const uint32_t& from, const uint32_t& to) const +MachineDesc MachineDesc::slice(uint32_t from, uint32_t to) const { - if (processor_ranges.size() > 1) - throw std::runtime_error( - "Ambiguous slicing: slicing is not allowed on a machine with more than one processor kind"); - return slice(from, to, preferred_target); } -MachineDesc MachineDesc::operator[](const TaskTarget& target) const { return only(target); } +MachineDesc MachineDesc::operator[](TaskTarget target) const { return only(target); } + +MachineDesc MachineDesc::operator[](const std::vector& targets) const +{ + return only(targets); +} bool MachineDesc::operator==(const MachineDesc& other) const { diff --git a/src/core/mapping/machine.h b/src/core/mapping/machine.h index 455fe1d06c..8ee67d1e46 100644 --- a/src/core/mapping/machine.h +++ b/src/core/mapping/machine.h @@ -23,6 +23,11 @@ #include "legate_defines.h" #include "legion.h" +/** + * @file + * @brief Legate machine interface + */ + namespace legate { namespace mapping { @@ -32,27 +37,110 @@ Processor::Kind to_kind(TaskTarget target); LegateVariantCode to_variant_code(TaskTarget target); +/** + * @ingroup mapping + * @brief A class to represent a range of processors. + * + * `ProcessorRange`s are half-open intervals of logical processors IDs. + */ struct ProcessorRange { + /** + * @brief Creates an empty processor range + */ ProcessorRange() {} + /** + * @brief Creates a processor range + * + * @param low Starting processor ID + * @param high End processor ID + * @param per_node_count Number of per-node processors + */ ProcessorRange(uint32_t low, uint32_t high, uint32_t per_node_count); ProcessorRange operator&(const ProcessorRange&) const; bool operator==(const ProcessorRange&) const; bool operator!=(const ProcessorRange&) const; + /** + * @brief Returns the number of processors in the range + * + * @return Processor count + */ uint32_t count() const; + /** + * @brief Checks if the processor range is empty + * + * @return true The range is empty + * @return false The range is not empty + */ bool empty() const; + /** + * @brief Converts the range to a human-readable string + * + * @return Processor range in a string + */ std::string to_string() const; + /** + * @brief Computes a range of node IDs for this processor range + * + * @return Node range in a pair + */ std::pair get_node_range() const; - ProcessorRange slice(const uint32_t& lo, const uint32_t& hi) const; + /** + * @brief Slices the processor range for a given sub-range + * + * @param from Starting index + * @param to End index + * + * @return Sliced procesor range + */ + ProcessorRange slice(uint32_t from, uint32_t to) const; + /** + * @brief Serializes the processor range to a buffer + * + * @param buffer Buffer to which the processor range should be serialized + */ void pack(BufferBuilder& buffer) const; + /** + * @brief Starting processor ID + */ uint32_t low{0}; + /** + * @brief End processor ID + */ uint32_t high{0}; + /** + * @brief Number of per-node processors + */ uint32_t per_node_count{1}; }; +/** + * @ingroup mapping + * @brief Machine descriptor class + * + * A machine descriptor describes the machine resource that should be used for a given scope of + * execution. By default, the scope is given the entire machine resource configured for this + * process. Then, the client can limit the resource by extracting a portion of the machine + * descriptor and setting it for the scope using `MachineTracker`. Configuring the scope with an + * empty machine descriptor raises a `std::runtime_error` exception. + */ struct MachineDesc { + /** + * @brief Creates an empty machine descriptor + */ MachineDesc() {} + /** + * @brief Creates a machine descriptor with a given set of processor ranges + * + * @param processor_ranges Processor ranges + */ MachineDesc(const std::map& processor_ranges); + /** + * @brief Creates a machine descriptor with a given set of processor ranges + * + * @param processor_ranges Processor ranges + */ + MachineDesc(std::map&& processor_ranges); MachineDesc(const MachineDesc&) = default; MachineDesc& operator=(const MachineDesc&) = default; @@ -60,32 +148,156 @@ struct MachineDesc { MachineDesc(MachineDesc&&) = default; MachineDesc& operator=(MachineDesc&&) = default; + /** + * @brief Returns the processor range for the preferred processor type in this descriptor + */ const ProcessorRange& processor_range() const; - const ProcessorRange& processor_range(const TaskTarget& target) const; - + /** + * @brief Returns the processor range for a given processor type + * + * If the processor type does not exist in the descriptor, an empty range is returned + * + * @target target Processor type to query + */ + const ProcessorRange& processor_range(TaskTarget target) const; + + /** + * @brief Returns the valid task targets within this machine descriptor + * + * @return Task targets + */ std::vector valid_targets() const; + /** + * @brief Returns the valid task targets excluding a given set of targets + * + * @param to_exclude Task targets to exclude from the query + * + * @return Task targets + */ std::vector valid_targets_except(const std::set& to_exclude) const; + /** + * @brief Returns the number of preferred processors + * + * @return Processor count + */ size_t count() const; - size_t count(const TaskTarget& target) const; - + /** + * @brief Returns the number of processors of a given type + * + * @param target Processor type to query + * + * @return Processor count + */ + size_t count(TaskTarget target) const; + + /** + * @brief Converts the machine descriptor to a human-readable string + * + * @return Machine descriptor in a string + */ std::string to_string() const; + /** + * @brief Serializes the machine descriptor to a buffer + * + * @param buffer Buffer to which the machine descriptor should be serialized + */ void pack(BufferBuilder& buffer) const; - MachineDesc only(const TaskTarget& target) const; - MachineDesc only(const std::set& target) const; - MachineDesc slice(const uint32_t& lo, const uint32_t& hi, const TaskTarget& target) const; - MachineDesc slice(const uint32_t& lo, const uint32_t& hi) const; - - MachineDesc operator[](const TaskTarget& target) const; + /** + * @brief Extracts the processor range for a given processor type and creates a fresh machine + * descriptor with it + * + * If the `target` does not exist in the machine descriptor, an empty descriptor is returned. + * + * @param target Processor type to select + * + * @return Machine descriptor with the chosen processor range + */ + MachineDesc only(TaskTarget target) const; + /** + * @brief Extracts the processor ranges for a given set of processor types and creates a fresh + * machine descriptor with them + * + * Any of the `targets` that does not exist will be mapped to an empty processor range in the + * returned machine descriptor + * + * @param targets Processor types to select + * + * @return Machine descriptor with the chosen processor ranges + */ + MachineDesc only(const std::vector& targets) const; + /** + * @brief Slices the processor range for a given processor type + * + * @param from Starting index + * @param to End index + * @param targets Processor type to slice + * + * @return Machine descriptor with the chosen procssor range sliced + */ + MachineDesc slice(uint32_t from, uint32_t to, TaskTarget target) const; + /** + * @brief Slices the processor range for the preferred processor type of this machine descriptor + * + * @param from Starting index + * @param to End index + * + * @return Machine descriptor with the preferred processor range sliced + */ + MachineDesc slice(uint32_t from, uint32_t to) const; + + /** + * @brief Selects the processor range for a given processor type and constructs a machine + * descriptor with it. + * + * This yields the same result as `.only(target)`. + * + * @param targets Processor type to select + * + * @return Machine descriptor with the chosen processor range + */ + MachineDesc operator[](TaskTarget target) const; + /** + * @brief Selects the processor ranges for a given set of processor types and constructs a machine + * descriptor with them. + * + * This yields the same result as `.only(targets)`. + * + * @param targets Processor types to select + * + * @return Machine descriptor with the chosen processor ranges + */ + MachineDesc operator[](const std::vector& targets) const; bool operator==(const MachineDesc& other) const; bool operator!=(const MachineDesc& other) const; + /** + * @brief Computes an intersection between two machine descriptors + * + * @param other Machine descriptor to intersect with this descriptor + * + * @return Machine descriptor + */ MachineDesc operator&(const MachineDesc& other) const; + /** + * @brief Indicates whether the machine descriptor is empty. + * + * A machine descriptor is empty when all its processor ranges are empty + * + * @return true The machine descriptor is empty + * @return false The machine descriptor is non-empty + */ bool empty() const; + /** + * @brief Preferred processor type of this machine descriptor + */ TaskTarget preferred_target{TaskTarget::CPU}; + /** + * @brief Processor ranges in this machine descriptor + */ std::map processor_ranges{}; }; diff --git a/src/core/mapping/mapping.h b/src/core/mapping/mapping.h index ba4fde5189..dadc747cf0 100644 --- a/src/core/mapping/mapping.h +++ b/src/core/mapping/mapping.h @@ -39,6 +39,9 @@ class Task; /** * @ingroup mapping * @brief An enum class for task targets + * + * The enumerators of `TaskTarget` are ordered by their precedence; i.e., `GPU`, if available, is + * chosen over `OMP` or `CPU, `OMP`, if available, is chosen over `CPU`. */ enum class TaskTarget : int32_t { /** diff --git a/src/core/partitioning/constraint.cc b/src/core/partitioning/constraint.cc index 9b16e4d660..f3401548aa 100644 --- a/src/core/partitioning/constraint.cc +++ b/src/core/partitioning/constraint.cc @@ -85,19 +85,19 @@ std::string Broadcast::to_string() const return std::move(ss).str(); } -std::unique_ptr align(const Variable* lhs, const Variable* rhs) +std::unique_ptr align(const Variable* lhs, const Variable* rhs) { // Since an Alignment object owns child nodes, inputs need to be copied return std::make_unique(std::make_unique(*lhs), std::make_unique(*rhs)); } -std::unique_ptr broadcast(const Variable* variable, const tuple& axes) +std::unique_ptr broadcast(const Variable* variable, const tuple& axes) { return broadcast(variable, tuple(axes)); } -std::unique_ptr broadcast(const Variable* variable, tuple&& axes) +std::unique_ptr broadcast(const Variable* variable, tuple&& axes) { return std::make_unique(std::make_unique(*variable), std::forward>(axes)); diff --git a/src/core/partitioning/constraint.h b/src/core/partitioning/constraint.h index cd2cfa573b..75f0d96cca 100644 --- a/src/core/partitioning/constraint.h +++ b/src/core/partitioning/constraint.h @@ -22,33 +22,67 @@ #include "core/utilities/tuple.h" -namespace legate { +/** @defgroup partitioning Partitioning + */ -struct Alignment; -struct Broadcast; -struct Constraint; -struct Literal; -struct Operation; -struct Partition; -struct Variable; +/** + * @file + * @brief Class definitions for partitioning constraint language + */ + +namespace legate { +class Alignment; +class Broadcast; +class Constraint; +class Literal; +class Operation; +class Partition; +class Variable; + +/** + * @ingroup partitioning + * @brief A base class for expressions + */ struct Expr { - public: + enum class Kind : int32_t { + LITERAL = 0, + VARIABLE = 1, + }; + virtual ~Expr() {} - public: virtual void find_partition_symbols(std::vector& partition_symbols) const = 0; - public: - virtual bool closed() const = 0; + /** + * @brief Indicates whether the expression is 'closed', i.e., free of any variables + * + * @return true Expression is closed + * @return false Expression is not closed + */ + virtual bool closed() const = 0; + /** + * @brief Converts the expressions to a human-readable string + * + * @return Expression in a string + */ virtual std::string to_string() const = 0; - public: + /** + * @brief Returns the expression kind + * + * @return Expression kind + */ + virtual Kind kind() const = 0; virtual const Literal* as_literal() const = 0; virtual const Variable* as_variable() const = 0; }; -struct Literal : public Expr { +/** + * @ingroup partitioning + * @brief A class for literals + */ +class Literal : public Expr { public: Literal(const std::shared_ptr& partition); @@ -57,16 +91,16 @@ struct Literal : public Expr { Literal& operator=(const Literal&) = default; public: - virtual void find_partition_symbols( - std::vector& partition_symbols) const override; + void find_partition_symbols(std::vector& partition_symbols) const override; public: - virtual bool closed() const override { return true; } - virtual std::string to_string() const override; + bool closed() const override { return true; } + std::string to_string() const override; public: - virtual const Literal* as_literal() const override { return this; } - virtual const Variable* as_variable() const override { return nullptr; } + Kind kind() const override { return Kind::LITERAL; } + const Literal* as_literal() const override { return this; } + const Variable* as_variable() const override { return nullptr; } public: std::shared_ptr partition() const { return partition_; } @@ -75,7 +109,11 @@ struct Literal : public Expr { std::shared_ptr partition_; }; -struct Variable : public Expr { +/** + * @ingroup partitioning + * @brief Class for partition symbols + */ +class Variable : public Expr { public: Variable(const Operation* op, int32_t id); @@ -88,16 +126,16 @@ struct Variable : public Expr { friend bool operator<(const Variable& lhs, const Variable& rhs); public: - virtual void find_partition_symbols( - std::vector& partition_symbols) const override; + void find_partition_symbols(std::vector& partition_symbols) const override; public: - virtual bool closed() const override { return false; } - virtual std::string to_string() const override; + bool closed() const override { return false; } + std::string to_string() const override; public: - virtual const Literal* as_literal() const override { return nullptr; } - virtual const Variable* as_variable() const override { return this; } + Kind kind() const override { return Kind::VARIABLE; } + const Literal* as_literal() const override { return nullptr; } + const Variable* as_variable() const override { return this; } public: const Operation* operation() const { return op_; } @@ -107,6 +145,10 @@ struct Variable : public Expr { int32_t id_; }; +/** + * @ingroup partitioning + * @brief A base class for partitioning constraints + */ struct Constraint { enum class Kind : int32_t { ALIGNMENT = 0, @@ -115,17 +157,34 @@ struct Constraint { virtual ~Constraint() {} - virtual Kind kind() const = 0; - virtual void find_partition_symbols(std::vector& partition_symbols) const = 0; + /** + * @brief Converts the constraint to a human-readable string + * + * @return Constraint in a string + */ virtual std::string to_string() const = 0; + /** + * @brief Returns the constraint kind + * + * @return Constraint kind + */ + virtual Kind kind() const = 0; + virtual const Alignment* as_alignment() const = 0; virtual const Broadcast* as_broadcast() const = 0; }; -// Constraint AST nodes own their child nodes +/** + * @ingroup partitioning + * @brief A class for alignment constraints + * + * An alignment constraint on stores indicates that the stores should be partitioned in the same + * way. If the stores referred to by an alignment constraint have different shapes, an + * `std::invalid_argument` exception will be raised when the partitioner solves the constraint. + */ class Alignment : public Constraint { public: Alignment(std::unique_ptr&& lhs, std::unique_ptr&& rhs); @@ -144,7 +203,17 @@ class Alignment : public Constraint { const Broadcast* as_broadcast() const override { return nullptr; } public: + /** + * @brief Returns the LHS of the alignment constraint + * + * @return Expression + */ const Expr* lhs() const { return lhs_.get(); } + /** + * @brief Returns the RHS of the alignment constraint + * + * @return Expression + */ const Expr* rhs() const { return rhs_.get(); } private: @@ -152,6 +221,15 @@ class Alignment : public Constraint { std::unique_ptr rhs_; }; +/** + * @ingroup partitioning + * @brief A class for broadcast constraints + * + * A broadcast constraint on a store indicates that some or all dimensions of the store should not + * be partitioned. The dimensions to broadcast must be specified by the constraint as well. If any + * of the dimension names is invalid, an `std::invalid_argument` exception will be raised when the + * auto-partitioner solves the constraint. + */ class Broadcast : public Constraint { public: Broadcast(std::unique_ptr variable, tuple&& axes); @@ -170,7 +248,17 @@ class Broadcast : public Constraint { const Broadcast* as_broadcast() const override { return this; } public: + /** + * @brief Returns the partition symbol to which this constraint is applied + * + * @return Partition symbol + */ const Variable* variable() const { return variable_.get(); } + /** + * @brief Returns the list of axes to broadcast + * + * @return Tuple of integers + */ const tuple& axes() const { return axes_; } private: @@ -178,10 +266,37 @@ class Broadcast : public Constraint { tuple axes_; }; -std::unique_ptr align(const Variable* lhs, const Variable* rhs); +/** + * @ingroup partitioning + * @brief Creates an alignment constraint on two variables + * + * @param lhs LHS variable + * @param rhs RHS variable + * + * @return Alignment constraint + */ +std::unique_ptr align(const Variable* lhs, const Variable* rhs); -std::unique_ptr broadcast(const Variable* variable, const tuple& axes); +/** + * @ingroup partitioning + * @brief Creates a broadcast constraint on a variable + * + * @param variable Partition symbol to constrain + * @param axes List of dimensions to broadcast + * + * @return Broadcast constraint + */ +std::unique_ptr broadcast(const Variable* variable, const tuple& axes); -std::unique_ptr broadcast(const Variable* variable, tuple&& axes); +/** + * @ingroup partitioning + * @brief Creates a broadcast constraint on a variable + * + * @param variable Partition symbol to constrain + * @param axes List of dimensions to broadcast + * + * @return Broadcast constraint + */ +std::unique_ptr broadcast(const Variable* variable, tuple&& axes); } // namespace legate diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index f6895a1c48..84fe952112 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -19,6 +19,7 @@ #include "core/data/logical_store_detail.h" #include "core/partitioning/partition.h" #include "core/runtime/launcher.h" +#include "core/runtime/partition_manager.h" #include "core/runtime/req_analyzer.h" #include "core/runtime/runtime.h" diff --git a/src/core/runtime/context.h b/src/core/runtime/context.h index 6aa25aca89..7f9e5b36fd 100644 --- a/src/core/runtime/context.h +++ b/src/core/runtime/context.h @@ -65,19 +65,8 @@ class InvalidTaskIdException : public std::exception { * @brief A library context that provides APIs for registering components */ class LibraryContext { - public: - /** - * @brief Creates a library context from a library name and a configuration. - * - * A library is registered to the runtime only upon the first construction - * and the `config` object is referred to only when the registration happens. - * All the following constructions of `LibraryContext` only retrieve the - * metadata from the runtime without registration and ignore the `config`. - * - * @param library_name Library name - * @param config Resource configuration for the library. If the library is already - * registered, the value will be ignored. - */ + private: + friend class Runtime; LibraryContext(const std::string& library_name, const ResourceConfig& config, std::unique_ptr mapper); diff --git a/src/core/runtime/field_manager.cc b/src/core/runtime/field_manager.cc new file mode 100644 index 0000000000..9df4d64b8d --- /dev/null +++ b/src/core/runtime/field_manager.cc @@ -0,0 +1,68 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/runtime/field_manager.h" + +#include "core/data/logical_region_field.h" +#include "core/runtime/region_manager.h" + +namespace legate { + +FieldManager::FieldManager(Runtime* runtime, const Domain& shape, uint32_t field_size) + : runtime_(runtime), shape_(shape), field_size_(field_size) +{ +} + +std::shared_ptr FieldManager::allocate_field() +{ + LogicalRegionField* rf = nullptr; + if (!free_fields_.empty()) { + auto field = free_fields_.front(); + log_legate.debug("Field %u recycled in field manager %p", field.second, this); + free_fields_.pop_front(); + rf = new LogicalRegionField(field.first, field.second); + } else { + auto rgn_mgr = runtime_->find_or_create_region_manager(shape_); + auto [lr, fid] = rgn_mgr->allocate_field(field_size_); + rf = new LogicalRegionField(lr, fid); + log_legate.debug("Field %u created in field manager %p", fid, this); + } + assert(rf != nullptr); + return std::shared_ptr(rf, [this](auto* field) { + log_legate.debug("Field %u freed in field manager %p", field->field_id(), this); + this->free_fields_.push_back(FreeField(field->region(), field->field_id())); + delete field; + }); +} + +std::shared_ptr FieldManager::import_field(const Legion::LogicalRegion& region, + Legion::FieldID field_id) +{ + // Import the region only if the region manager is created fresh + auto rgn_mgr = runtime_->find_or_create_region_manager(shape_); + if (!rgn_mgr->has_space()) rgn_mgr->import_region(region); + + log_legate.debug("Field %u imported in field manager %p", field_id, this); + + auto* rf = new LogicalRegionField(region, field_id); + return std::shared_ptr(rf, [this](auto* field) { + log_legate.debug("Field %u freed in field manager %p", field->field_id(), this); + this->free_fields_.push_back(FreeField(field->region(), field->field_id())); + delete field; + }); +} + +} // namespace legate diff --git a/src/core/runtime/field_manager.h b/src/core/runtime/field_manager.h new file mode 100644 index 0000000000..3569ec9b02 --- /dev/null +++ b/src/core/runtime/field_manager.h @@ -0,0 +1,47 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include + +#include "core/utilities/typedefs.h" + +namespace legate { + +class LogicalRegionField; +class Runtime; + +class FieldManager { + public: + FieldManager(Runtime* runtime, const Domain& shape, uint32_t field_size); + + public: + std::shared_ptr allocate_field(); + std::shared_ptr import_field(const Legion::LogicalRegion& region, + Legion::FieldID field_id); + + private: + Runtime* runtime_; + Domain shape_; + uint32_t field_size_; + + private: + using FreeField = std::pair; + std::deque free_fields_; +}; + +} // namespace legate diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/launcher.cc index a9f68f8a96..82ff4a295f 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -31,9 +31,10 @@ namespace legate { TaskLauncher::TaskLauncher(LibraryContext* library, const mapping::MachineDesc& machine, + const std::string& provenance, int64_t task_id, int64_t tag /*= 0*/) - : library_(library), task_id_(task_id), tag_(tag) + : library_(library), task_id_(task_id), tag_(tag), provenance_(provenance) { req_analyzer_ = new RequirementAnalyzer(); out_analyzer_ = new OutputRequirementAnalyzer(); @@ -189,8 +190,7 @@ std::unique_ptr TaskLauncher::build_single_task() buffer_->pack(0); pack_sharding_functor_id(); - auto* runtime = Runtime::get_runtime(); - auto& provenance = runtime->provenance_manager()->get_provenance(); + auto* runtime = Runtime::get_runtime(); auto single_task = std::make_unique(legion_task_id(), buffer_->to_legion_buffer(), @@ -198,7 +198,7 @@ std::unique_ptr TaskLauncher::build_single_task() legion_mapper_id(), tag_, mapper_arg_->to_legion_buffer(), - provenance.c_str()); + provenance_.c_str()); for (auto& future : futures_) single_task->add_future(future); req_analyzer_->populate_launcher(single_task.get()); @@ -227,8 +227,7 @@ std::unique_ptr TaskLauncher::build_index_task( buffer_->pack(0); pack_sharding_functor_id(); - auto* runtime = Runtime::get_runtime(); - auto& provenance = runtime->provenance_manager()->get_provenance(); + auto* runtime = Runtime::get_runtime(); auto index_task = std::make_unique(legion_task_id(), launch_domain, @@ -239,7 +238,7 @@ std::unique_ptr TaskLauncher::build_index_task( legion_mapper_id(), tag_, mapper_arg_->to_legion_buffer(), - provenance.c_str()); + provenance_.c_str()); for (auto& future : futures_) index_task->add_future(future); for (auto& future_map : future_maps_) index_task->point_futures.push_back(future_map); diff --git a/src/core/runtime/launcher.h b/src/core/runtime/launcher.h index ea7593efa3..16d47da941 100644 --- a/src/core/runtime/launcher.h +++ b/src/core/runtime/launcher.h @@ -48,6 +48,7 @@ class TaskLauncher { public: TaskLauncher(LibraryContext* library, const mapping::MachineDesc& machine, + const std::string& provenance, int64_t task_id, int64_t tag = 0); ~TaskLauncher(); @@ -102,6 +103,7 @@ class TaskLauncher { LibraryContext* library_; int64_t task_id_; int64_t tag_; + std::string provenance_; private: std::vector inputs_; diff --git a/src/core/runtime/machine_manager.cc b/src/core/runtime/machine_manager.cc new file mode 100644 index 0000000000..afcb08a14f --- /dev/null +++ b/src/core/runtime/machine_manager.cc @@ -0,0 +1,48 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/runtime/machine_manager.h" + +namespace legate { + +//////////////////////////////////////////// +// legate::MachineManager +//////////////////////////////////////////// +const mapping::MachineDesc& MachineManager::get_machine() const +{ +#ifdef DEBUG_LEGATE + assert(machines_.size() > 0); +#endif + return machines_.back(); +} + +void MachineManager::push_machine(const mapping::MachineDesc& machine) +{ + machines_.push_back(machine); +} + +void MachineManager::push_machine(mapping::MachineDesc&& machine) +{ + machines_.emplace_back(machine); +} + +void MachineManager::pop_machine() +{ + if (machines_.size() <= 1) throw std::underflow_error("can't pop from the empty machine stack"); + machines_.pop_back(); +} + +} // namespace legate diff --git a/src/core/runtime/machine_manager.h b/src/core/runtime/machine_manager.h new file mode 100644 index 0000000000..e4017aa861 --- /dev/null +++ b/src/core/runtime/machine_manager.h @@ -0,0 +1,42 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include "core/mapping/machine.h" + +namespace legate { + +namespace mapping { + +class MachineDesc; + +} // namespace mapping + +class MachineManager { + public: + const mapping::MachineDesc& get_machine() const; + + void push_machine(const mapping::MachineDesc& machine); + void push_machine(mapping::MachineDesc&& machine); + + void pop_machine(); + + private: + std::vector machines_; +}; + +} // namespace legate diff --git a/src/core/runtime/operation.cc b/src/core/runtime/operation.cc index 55c3de8568..507569563f 100644 --- a/src/core/runtime/operation.cc +++ b/src/core/runtime/operation.cc @@ -37,7 +37,10 @@ namespace legate { //////////////////////////////////////////////////// Operation::Operation(LibraryContext* library, uint64_t unique_id, mapping::MachineDesc&& machine) - : library_(library), unique_id_(unique_id), machine_(std::forward(machine)) + : library_(library), + unique_id_(unique_id), + machine_(std::forward(machine)), + provenance_(Runtime::get_runtime()->provenance_manager()->get_provenance()) { } @@ -73,7 +76,7 @@ void Task::add_scalar_arg(const Scalar& scalar) { scalars_.push_back(scalar); } void Task::launch(Strategy* p_strategy) { auto& strategy = *p_strategy; - TaskLauncher launcher(library_, machine_, task_id_); + TaskLauncher launcher(library_, machine_, provenance_, task_id_); auto launch_domain = strategy.launch_domain(this); auto launch_ndim = launch_domain != nullptr ? launch_domain->dim : 0; diff --git a/src/core/runtime/operation.h b/src/core/runtime/operation.h index 83a07835bb..4ac428b91b 100644 --- a/src/core/runtime/operation.h +++ b/src/core/runtime/operation.h @@ -24,6 +24,14 @@ #include "core/partitioning/constraint.h" #include "legion.h" +/** @defgroup op Task and non-task operation descriptors + */ + +/** + * @file + * @brief Class definitions for various operation kinds + */ + namespace legate { class Constraint; @@ -39,6 +47,10 @@ class LogicalStore; } // namespace detail +/** + * @ingroup op + * @brief A base class for all operation kinds + */ class Operation { protected: using StoreArg = std::pair; @@ -53,12 +65,29 @@ class Operation { virtual std::string to_string() const = 0; public: + /** + * @brief Declares partition symbol + * + * @return A new symbol that can be used when passing a store to an operation + */ const Variable* declare_partition(); detail::LogicalStore* find_store(const Variable* variable) const; public: + /** + * @brief Returns the machine of the scope in which this operation is issued + * + * @return The machine of the scope + */ const mapping::MachineDesc& machine() const { return machine_; } + /** + * @brief Returns the provenance information of this operation + * + * @return Provenance + */ + const std::string& provenance() const { return provenance_; } + protected: LibraryContext* library_; uint64_t unique_id_; @@ -74,9 +103,15 @@ class Operation { uint32_t next_part_id_{0}; std::vector> partition_symbols_{}; std::map store_mappings_{}; + std::string provenance_; mapping::MachineDesc machine_; }; +/** + * @ingroup op + * @brief A base class for tasks inherited by two kinds of task descriptors + * (auto-parallelized and manually parallelized task descriptors) + */ class Task : public Operation { protected: Task(LibraryContext* library, @@ -88,6 +123,11 @@ class Task : public Operation { virtual ~Task() {} public: + /** + * @brief Adds a by-value scalar argument to the task + * + * @param scalar A scalar to add to the task + */ void add_scalar_arg(const Scalar& scalar); public: @@ -107,6 +147,10 @@ class Task : public Operation { std::vector scalar_reductions_{}; }; +/** + * @ingroup op + * @brief A class for auto-parallelized task desciptors + */ class AutoTask : public Task { public: friend class Runtime; @@ -119,8 +163,36 @@ class AutoTask : public Task { ~AutoTask() {} public: + /** + * @brief Adds a store to the task as input + * + * Partitioning of the store is controlled by constraints on the partition symbol + * associated with the store + * + * @param store A store to add to the task as input + * @param partition_symbol A partition symbol for the store + */ void add_input(LogicalStore store, const Variable* partition_symbol); + /** + * @brief Adds a store to the task as output + * + * Partitioning of the store is controlled by constraints on the partition symbol + * associated with the store + * + * @param store A store to add to the task as output + * @param partition_symbol A partition symbol for the store + */ void add_output(LogicalStore store, const Variable* partition_symbol); + /** + * @brief Adds a store to the task for reductions + * + * Partitioning of the store is controlled by constraints on the partition symbol + * associated with the store + * + * @param store A store to add to the task for reductions + * @param redop ID of the reduction operator to use + * @param partition_symbol A partition symbol for the store + */ void add_reduction(LogicalStore store, Legion::ReductionOpID redop, const Variable* partition_symbol); @@ -131,6 +203,11 @@ class AutoTask : public Task { const Variable* partition_symbol); public: + /** + * @brief Adds a partitioning constraint to the task + * + * @param constraint A partitioning constraint + */ void add_constraint(std::unique_ptr constraint); void add_to_constraint_graph(ConstraintGraph& constraint_graph) const override; @@ -138,6 +215,10 @@ class AutoTask : public Task { std::vector> constraints_{}; }; +/** + * @ingroup op + * @brief A class for manually parallelized task descriptors + */ class ManualTask : public Task { private: friend class Runtime; @@ -151,13 +232,51 @@ class ManualTask : public Task { ~ManualTask(); public: + /** + * @brief Adds a store to the task as input + * + * The store will be unpartitioned but broadcasted to all the tasks + * + * @param store A store to add to the task as input + */ void add_input(LogicalStore store); + /** + * @brief Adds a store to the task as output + * + * The store will be unpartitioned but broadcasted to all the tasks + * + * @param store A store to add to the task as output + */ void add_output(LogicalStore store); + /** + * @brief Adds a store to the task for reductions + * + * The store will be unpartitioned but broadcasted to all the tasks + * + * @param store A store to add to the task for reductions + * @param redop ID of the reduction operator + */ void add_reduction(LogicalStore store, Legion::ReductionOpID redop); public: + /** + * @brief Adds a store partition to the task as input + * + * @param store_partition A store partition to add to the task as input + */ void add_input(LogicalStorePartition store_partition); + /** + * @brief Adds a store partition to the task as output + * + * @param store_partition A store partition to add to the task as output + */ void add_output(LogicalStorePartition store_partition); + /** + * @brief Adds a store partition to the task for reductions + * + * @param store_partition A store partition to add to the task for reductions + * @param redop ID of the reduction operator + */ void add_reduction(LogicalStorePartition store_partition, Legion::ReductionOpID redop); private: diff --git a/src/core/runtime/partition_manager.cc b/src/core/runtime/partition_manager.cc new file mode 100644 index 0000000000..3b69a128ae --- /dev/null +++ b/src/core/runtime/partition_manager.cc @@ -0,0 +1,217 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/runtime/partition_manager.h" + +#include "core/legate_c.h" +#include "core/mapping/machine.h" +#include "core/partitioning/partition.h" +#include "core/runtime/context.h" +#include "core/runtime/runtime.h" + +namespace legate { + +PartitionManager::PartitionManager(Runtime* runtime, const LibraryContext* context) +{ + auto mapper_id = context->get_mapper_id(); + min_shard_volume_ = + runtime->get_tunable(mapper_id, LEGATE_CORE_TUNABLE_MIN_SHARD_VOLUME); + +#ifdef DEBUG_LEGATE + assert(min_shard_volume_ > 0); +#endif +} + +const std::vector& PartitionManager::get_factors(const mapping::MachineDesc& machine) +{ + uint32_t curr_num_pieces = machine.count(); + + auto finder = all_factors_.find(curr_num_pieces); + if (all_factors_.end() == finder) { + uint32_t remaining_pieces = curr_num_pieces; + std::vector factors; + auto push_factors = [&factors, &remaining_pieces](uint32_t prime) { + while (remaining_pieces % prime == 0) { + factors.push_back(prime); + remaining_pieces /= prime; + } + }; + for (uint32_t factor : {11, 7, 5, 3, 2}) push_factors(factor); + all_factors_.insert({curr_num_pieces, std::move(factors)}); + finder = all_factors_.find(curr_num_pieces); + } + + return finder->second; +} + +Shape PartitionManager::compute_launch_shape(const mapping::MachineDesc& machine, + const Restrictions& restrictions, + const Shape& shape) +{ + uint32_t curr_num_pieces = machine.count(); + // Easy case if we only have one piece: no parallel launch space + if (1 == curr_num_pieces) return {}; + + // If we only have one point then we never do parallel launches + if (shape.all([](auto extent) { return 1 == extent; })) return {}; + + // Prune out any dimensions that are 1 + std::vector temp_shape{}; + std::vector temp_dims{}; + int64_t volume = 1; + for (uint32_t dim = 0; dim < shape.size(); ++dim) { + auto extent = shape[dim]; + if (1 == extent || restrictions[dim] == Restriction::FORBID) continue; + temp_shape.push_back(extent); + temp_dims.push_back(dim); + volume *= extent; + } + + // Figure out how many shards we can make with this array + int64_t max_pieces = (volume + min_shard_volume_ - 1) / min_shard_volume_; + assert(max_pieces > 0); + // If we can only make one piece return that now + if (1 == max_pieces) return {}; + + // Otherwise we need to compute it ourselves + // TODO: a better heuristic here. + // For now if we can make at least two pieces then we will make N pieces. + max_pieces = curr_num_pieces; + + // First compute the N-th root of the number of pieces + uint32_t ndim = temp_shape.size(); + assert(ndim > 0); + std::vector temp_result{}; + + if (1 == ndim) { + // Easy one dimensional case + temp_result.push_back(std::min(temp_shape.front(), static_cast(max_pieces))); + } else if (2 == ndim) { + if (volume < max_pieces) { + // TBD: Once the max_pieces heuristic is fixed, this should never happen + temp_result.swap(temp_shape); + } else { + // Two dimensional so we can use square root to try and generate as square a pieces + // as possible since most often we will be doing matrix operations with these + auto nx = temp_shape[0]; + auto ny = temp_shape[1]; + auto swap = nx > ny; + if (swap) std::swap(nx, ny); + auto n = std::sqrt(static_cast(max_pieces) * nx / ny); + + // Need to constraint n to be an integer with numpcs % n == 0 + // try rounding n both up and down + auto n1 = std::max(1, static_cast(std::floor(n + 1e-12))); + while (max_pieces % n1 != 0) --n1; + auto n2 = std::max(1, static_cast(std::floor(n - 1e-12))); + while (max_pieces % n2 != 0) ++n2; + + // pick whichever of n1 and n2 gives blocks closest to square + // i.e. gives the shortest long side + auto side1 = std::max(nx / n1, ny / (max_pieces / n1)); + auto side2 = std::max(nx / n2, ny / (max_pieces / n2)); + auto px = static_cast(side1 <= side2 ? n1 : n2); + auto py = static_cast(max_pieces / px); + + // we need to trim launch space if it is larger than the + // original shape in one of the dimensions (can happen in + // testing) + if (swap) { + temp_result.push_back(std::min(py, temp_shape[0])); + temp_result.push_back(std::min(px, temp_shape[1])); + } else { + temp_result.push_back(std::min(px, temp_shape[1])); + temp_result.push_back(std::min(py, temp_shape[0])); + } + } + } else { + // For higher dimensions we care less about "square"-ness and more about evenly dividing + // things, compute the prime factors for our number of pieces and then round-robin them + // onto the shape, with the goal being to keep the last dimension >= 32 for good memory + // performance on the GPU + temp_result.resize(ndim); + std::fill(temp_result.begin(), temp_result.end(), 1); + size_t factor_prod = 1; + for (auto factor : get_factors(machine)) { + // Avoid exceeding the maximum number of pieces + if (factor * factor_prod > max_pieces) break; + + factor_prod *= factor; + + std::vector remaining; + for (uint32_t idx = 0; idx < temp_shape.size(); ++idx) + remaining.push_back((temp_shape[idx] + temp_result[idx] - 1) / temp_result[idx]); + uint32_t big_dim = std::max_element(remaining.begin(), remaining.end()) - remaining.begin(); + if (big_dim < ndim - 1) { + // Not the last dimension, so do it + temp_result[big_dim] *= factor; + } else { + // Last dim so see if it still bigger than 32 + if (remaining[big_dim] / factor >= 32) { + // go ahead and do it + temp_result[big_dim] *= factor; + } else { + // Won't be see if we can do it with one of the other dimensions + uint32_t next_big_dim = + std::max_element(remaining.begin(), remaining.end() - 1) - remaining.begin(); + if (remaining[next_big_dim] / factor > 0) + temp_result[next_big_dim] *= factor; + else + // Fine just do it on the last dimension + temp_result[big_dim] *= factor; + } + } + } + } + + // Project back onto the original number of dimensions + assert(temp_result.size() == ndim); + std::vector result(shape.size(), 1); + for (uint32_t idx = 0; idx < ndim; ++idx) result[temp_dims[idx]] = temp_result[idx]; + + return Shape(std::move(result)); +} + +Shape PartitionManager::compute_tile_shape(const Shape& extents, const Shape& launch_shape) +{ + assert(extents.size() == launch_shape.size()); + Shape tile_shape; + for (uint32_t idx = 0; idx < extents.size(); ++idx) { + auto x = extents[idx]; + auto y = launch_shape[idx]; + tile_shape.append_inplace((x + y - 1) / y); + } + return tile_shape; +} + +Legion::IndexPartition PartitionManager::find_index_partition(const Legion::IndexSpace& index_space, + const Tiling& tiling) const +{ + auto finder = tiling_cache_.find(std::make_pair(index_space, tiling)); + if (finder != tiling_cache_.end()) + return finder->second; + else + return Legion::IndexPartition::NO_PART; +} + +void PartitionManager::record_index_partition(const Legion::IndexSpace& index_space, + const Tiling& tiling, + const Legion::IndexPartition& index_partition) +{ + tiling_cache_[std::make_pair(index_space, tiling)] = index_partition; +} + +} // namespace legate diff --git a/src/core/runtime/partition_manager.h b/src/core/runtime/partition_manager.h new file mode 100644 index 0000000000..61763ff6f1 --- /dev/null +++ b/src/core/runtime/partition_manager.h @@ -0,0 +1,67 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include + +#include "legion.h" + +#include "core/data/shape.h" +#include "core/partitioning/restriction.h" + +namespace legate { + +class LibraryContext; +class Runtime; +class Tiling; + +namespace mapping { + +class MachineDesc; + +} // namespace mapping + +class PartitionManager { + public: + PartitionManager(Runtime* runtime, const LibraryContext* context); + + public: + const std::vector& get_factors(const mapping::MachineDesc& machine); + + public: + Shape compute_launch_shape(const mapping::MachineDesc& machine, + const Restrictions& restrictions, + const Shape& shape); + Shape compute_tile_shape(const Shape& extents, const Shape& launch_shape); + + public: + Legion::IndexPartition find_index_partition(const Legion::IndexSpace& index_space, + const Tiling& tiling) const; + void record_index_partition(const Legion::IndexSpace& index_space, + const Tiling& tiling, + const Legion::IndexPartition& index_partition); + + private: + int64_t min_shard_volume_; + std::unordered_map> all_factors_; + + private: + using TilingCacheKey = std::pair; + std::map tiling_cache_; +}; + +} // namespace legate diff --git a/src/core/runtime/provenance_manager.cc b/src/core/runtime/provenance_manager.cc new file mode 100644 index 0000000000..a37768c34e --- /dev/null +++ b/src/core/runtime/provenance_manager.cc @@ -0,0 +1,68 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/runtime/provenance_manager.h" + +#include +#include + +namespace legate { + +static const std::string BOTTOM = ""; + +ProvenanceManager::ProvenanceManager() { provenance_.push_back(BOTTOM); } + +const std::string& ProvenanceManager::get_provenance() +{ +#ifdef DEBUG_LEGATE + assert(provenance_.size() > 0); +#endif + return provenance_.back(); +} + +void ProvenanceManager::set_provenance(const std::string& p) +{ +#ifdef DEBUG_LEGATE + assert(provenance_.size() > 0); +#endif + provenance_.back() = p; +} + +void ProvenanceManager::reset_provenance() +{ +#ifdef DEBUG_LEGATE + assert(provenance_.size() > 0); +#endif + provenance_.back() = BOTTOM; +} + +void ProvenanceManager::push_provenance(const std::string& p) { provenance_.push_back(p); } + +void ProvenanceManager::pop_provenance() +{ + if (provenance_.size() <= 1) { + throw std::underflow_error("can't pop from an empty provenance stack"); + } + provenance_.pop_back(); +} + +void ProvenanceManager::clear_all() +{ + provenance_.clear(); + provenance_.push_back(BOTTOM); +} + +} // namespace legate diff --git a/src/core/runtime/provenance_manager.h b/src/core/runtime/provenance_manager.h new file mode 100644 index 0000000000..b105dc7332 --- /dev/null +++ b/src/core/runtime/provenance_manager.h @@ -0,0 +1,45 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include +#include + +namespace legate { + +class ProvenanceManager { + public: + ProvenanceManager(); + + public: + const std::string& get_provenance(); + + void set_provenance(const std::string& p); + + void reset_provenance(); + + void push_provenance(const std::string& p); + + void pop_provenance(); + + void clear_all(); + + private: + std::vector provenance_; +}; + +} // namespace legate diff --git a/src/core/runtime/region_manager.cc b/src/core/runtime/region_manager.cc new file mode 100644 index 0000000000..1306f95209 --- /dev/null +++ b/src/core/runtime/region_manager.cc @@ -0,0 +1,50 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/runtime/region_manager.h" +#include "core/runtime/runtime.h" + +namespace legate { + +RegionManager::RegionManager(Runtime* runtime, const Domain& shape) + : runtime_(runtime), shape_(shape) +{ +} + +void RegionManager::push_entry() +{ + auto is = runtime_->find_or_create_index_space(shape_); + auto fs = runtime_->create_field_space(); + entries_.emplace_back(runtime_->create_region(is, fs)); +} + +bool RegionManager::has_space() const { return !entries_.empty() && active_entry().has_space(); } + +std::pair RegionManager::allocate_field(size_t field_size) +{ + if (!has_space()) push_entry(); + auto& entry = active_entry(); + auto fid = + runtime_->allocate_field(entry.region.get_field_space(), entry.get_next_field_id(), field_size); + return std::make_pair(entry.region, fid); +} + +void RegionManager::import_region(const Legion::LogicalRegion& region) +{ + entries_.emplace_back(region); +} + +} // namespace legate diff --git a/src/core/runtime/region_manager.h b/src/core/runtime/region_manager.h new file mode 100644 index 0000000000..51362c9c9c --- /dev/null +++ b/src/core/runtime/region_manager.h @@ -0,0 +1,61 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include "core/utilities/typedefs.h" + +namespace legate { + +class Runtime; + +class RegionManager { + private: + struct ManagerEntry { + static constexpr Legion::FieldID FIELD_ID_BASE = 10000; + static constexpr int32_t MAX_NUM_FIELDS = LEGION_MAX_FIELDS - LEGION_DEFAULT_LOCAL_FIELDS; + + ManagerEntry(const Legion::LogicalRegion& _region) + : region(_region), next_field_id(FIELD_ID_BASE) + { + } + bool has_space() const { return next_field_id - FIELD_ID_BASE < MAX_NUM_FIELDS; } + Legion::FieldID get_next_field_id() { return next_field_id++; } + + Legion::LogicalRegion region; + Legion::FieldID next_field_id; + }; + + public: + RegionManager(Runtime* runtime, const Domain& shape); + + private: + const ManagerEntry& active_entry() const { return entries_.back(); } + ManagerEntry& active_entry() { return entries_.back(); } + void push_entry(); + + public: + bool has_space() const; + std::pair allocate_field(size_t field_size); + void import_region(const Legion::LogicalRegion& region); + + private: + Runtime* runtime_; + Domain shape_; + std::vector entries_{}; +}; + +} // namespace legate diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 9406e43244..adc5d0be46 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -27,8 +27,13 @@ #include "core/partitioning/partition.h" #include "core/partitioning/partitioner.h" #include "core/runtime/context.h" +#include "core/runtime/field_manager.h" #include "core/runtime/launcher.h" +#include "core/runtime/machine_manager.h" +#include "core/runtime/partition_manager.h" #include "core/runtime/projection.h" +#include "core/runtime/provenance_manager.h" +#include "core/runtime/region_manager.h" #include "core/runtime/shard.h" #include "core/task/exception.h" #include "core/task/task.h" @@ -56,8 +61,6 @@ static const char* const core_library_name = "legate.core"; /*static*/ bool Core::has_socket_mem = false; -static const std::string EMPTY_STRING = ""; - /*static*/ void Core::parse_config(void) { #ifndef LEGATE_USE_CUDA @@ -299,386 +302,6 @@ void register_builtin_reduction_ops() Core::parse_config(); } -//////////////////////////////////////////////////// -// legate::RegionManager -//////////////////////////////////////////////////// - -class RegionManager { - private: - struct ManagerEntry { - static constexpr Legion::FieldID FIELD_ID_BASE = 10000; - static constexpr int32_t MAX_NUM_FIELDS = LEGION_MAX_FIELDS - LEGION_DEFAULT_LOCAL_FIELDS; - - ManagerEntry(const Legion::LogicalRegion& _region) - : region(_region), next_field_id(FIELD_ID_BASE) - { - } - bool has_space() const { return next_field_id - FIELD_ID_BASE < MAX_NUM_FIELDS; } - Legion::FieldID get_next_field_id() { return next_field_id++; } - - Legion::LogicalRegion region; - Legion::FieldID next_field_id; - }; - - public: - RegionManager(Runtime* runtime, const Domain& shape); - - private: - const ManagerEntry& active_entry() const { return entries_.back(); } - ManagerEntry& active_entry() { return entries_.back(); } - void push_entry(); - - public: - bool has_space() const; - std::pair allocate_field(size_t field_size); - void import_region(const Legion::LogicalRegion& region); - - private: - Runtime* runtime_; - Domain shape_; - std::vector entries_{}; -}; - -RegionManager::RegionManager(Runtime* runtime, const Domain& shape) - : runtime_(runtime), shape_(shape) -{ -} - -void RegionManager::push_entry() -{ - auto is = runtime_->find_or_create_index_space(shape_); - auto fs = runtime_->create_field_space(); - entries_.emplace_back(runtime_->create_region(is, fs)); -} - -bool RegionManager::has_space() const { return !entries_.empty() && active_entry().has_space(); } - -std::pair RegionManager::allocate_field(size_t field_size) -{ - if (!has_space()) push_entry(); - auto& entry = active_entry(); - auto fid = - runtime_->allocate_field(entry.region.get_field_space(), entry.get_next_field_id(), field_size); - return std::make_pair(entry.region, fid); -} - -void RegionManager::import_region(const Legion::LogicalRegion& region) -{ - entries_.emplace_back(region); -} - -//////////////////////////////////////////////////// -// legate::FieldManager -//////////////////////////////////////////////////// - -class FieldManager { - public: - FieldManager(Runtime* runtime, const Domain& shape, uint32_t field_size); - - public: - std::shared_ptr allocate_field(); - std::shared_ptr import_field(const Legion::LogicalRegion& region, - Legion::FieldID field_id); - - private: - Runtime* runtime_; - Domain shape_; - uint32_t field_size_; - - private: - using FreeField = std::pair; - std::deque free_fields_; -}; - -FieldManager::FieldManager(Runtime* runtime, const Domain& shape, uint32_t field_size) - : runtime_(runtime), shape_(shape), field_size_(field_size) -{ -} - -std::shared_ptr FieldManager::allocate_field() -{ - LogicalRegionField* rf = nullptr; - if (!free_fields_.empty()) { - auto field = free_fields_.front(); - log_legate.debug("Field %u recycled in field manager %p", field.second, this); - free_fields_.pop_front(); - rf = new LogicalRegionField(field.first, field.second); - } else { - auto rgn_mgr = runtime_->find_or_create_region_manager(shape_); - auto [lr, fid] = rgn_mgr->allocate_field(field_size_); - rf = new LogicalRegionField(lr, fid); - log_legate.debug("Field %u created in field manager %p", fid, this); - } - assert(rf != nullptr); - return std::shared_ptr(rf, [this](auto* field) { - log_legate.debug("Field %u freed in field manager %p", field->field_id(), this); - this->free_fields_.push_back(FreeField(field->region(), field->field_id())); - delete field; - }); -} - -std::shared_ptr FieldManager::import_field(const Legion::LogicalRegion& region, - Legion::FieldID field_id) -{ - // Import the region only if the region manager is created fresh - auto rgn_mgr = runtime_->find_or_create_region_manager(shape_); - if (!rgn_mgr->has_space()) rgn_mgr->import_region(region); - - log_legate.debug("Field %u imported in field manager %p", field_id, this); - - auto* rf = new LogicalRegionField(region, field_id); - return std::shared_ptr(rf, [this](auto* field) { - log_legate.debug("Field %u freed in field manager %p", field->field_id(), this); - this->free_fields_.push_back(FreeField(field->region(), field->field_id())); - delete field; - }); -} - -//////////////////////////////////////////////////// -// legate::PartitionManager -//////////////////////////////////////////////////// - -PartitionManager::PartitionManager(Runtime* runtime, const LibraryContext* context) -{ - auto mapper_id = context->get_mapper_id(); - min_shard_volume_ = - runtime->get_tunable(mapper_id, LEGATE_CORE_TUNABLE_MIN_SHARD_VOLUME); - -#ifdef DEBUG_LEGATE - assert(min_shard_volume_ > 0); -#endif -} - -const std::vector& PartitionManager::get_factors(const mapping::MachineDesc& machine) -{ - uint32_t curr_num_pieces = machine.count(); - - auto finder = all_factors_.find(curr_num_pieces); - if (all_factors_.end() == finder) { - uint32_t remaining_pieces = curr_num_pieces; - std::vector factors; - auto push_factors = [&factors, &remaining_pieces](uint32_t prime) { - while (remaining_pieces % prime == 0) { - factors.push_back(prime); - remaining_pieces /= prime; - } - }; - for (uint32_t factor : {11, 7, 5, 3, 2}) push_factors(factor); - all_factors_.insert({curr_num_pieces, std::move(factors)}); - finder = all_factors_.find(curr_num_pieces); - } - - return finder->second; -} - -Shape PartitionManager::compute_launch_shape(const mapping::MachineDesc& machine, - const Restrictions& restrictions, - const Shape& shape) -{ - uint32_t curr_num_pieces = machine.count(); - // Easy case if we only have one piece: no parallel launch space - if (1 == curr_num_pieces) return {}; - - // If we only have one point then we never do parallel launches - if (shape.all([](auto extent) { return 1 == extent; })) return {}; - - // Prune out any dimensions that are 1 - std::vector temp_shape{}; - std::vector temp_dims{}; - int64_t volume = 1; - for (uint32_t dim = 0; dim < shape.size(); ++dim) { - auto extent = shape[dim]; - if (1 == extent || restrictions[dim] == Restriction::FORBID) continue; - temp_shape.push_back(extent); - temp_dims.push_back(dim); - volume *= extent; - } - - // Figure out how many shards we can make with this array - int64_t max_pieces = (volume + min_shard_volume_ - 1) / min_shard_volume_; - assert(max_pieces > 0); - // If we can only make one piece return that now - if (1 == max_pieces) return {}; - - // Otherwise we need to compute it ourselves - // TODO: a better heuristic here. - // For now if we can make at least two pieces then we will make N pieces. - max_pieces = curr_num_pieces; - - // First compute the N-th root of the number of pieces - uint32_t ndim = temp_shape.size(); - assert(ndim > 0); - std::vector temp_result{}; - - if (1 == ndim) { - // Easy one dimensional case - temp_result.push_back(std::min(temp_shape.front(), static_cast(max_pieces))); - } else if (2 == ndim) { - if (volume < max_pieces) { - // TBD: Once the max_pieces heuristic is fixed, this should never happen - temp_result.swap(temp_shape); - } else { - // Two dimensional so we can use square root to try and generate as square a pieces - // as possible since most often we will be doing matrix operations with these - auto nx = temp_shape[0]; - auto ny = temp_shape[1]; - auto swap = nx > ny; - if (swap) std::swap(nx, ny); - auto n = std::sqrt(static_cast(max_pieces) * nx / ny); - - // Need to constraint n to be an integer with numpcs % n == 0 - // try rounding n both up and down - auto n1 = std::max(1, static_cast(std::floor(n + 1e-12))); - while (max_pieces % n1 != 0) --n1; - auto n2 = std::max(1, static_cast(std::floor(n - 1e-12))); - while (max_pieces % n2 != 0) ++n2; - - // pick whichever of n1 and n2 gives blocks closest to square - // i.e. gives the shortest long side - auto side1 = std::max(nx / n1, ny / (max_pieces / n1)); - auto side2 = std::max(nx / n2, ny / (max_pieces / n2)); - auto px = static_cast(side1 <= side2 ? n1 : n2); - auto py = static_cast(max_pieces / px); - - // we need to trim launch space if it is larger than the - // original shape in one of the dimensions (can happen in - // testing) - if (swap) { - temp_result.push_back(std::min(py, temp_shape[0])); - temp_result.push_back(std::min(px, temp_shape[1])); - } else { - temp_result.push_back(std::min(px, temp_shape[1])); - temp_result.push_back(std::min(py, temp_shape[0])); - } - } - } else { - // For higher dimensions we care less about "square"-ness and more about evenly dividing - // things, compute the prime factors for our number of pieces and then round-robin them - // onto the shape, with the goal being to keep the last dimension >= 32 for good memory - // performance on the GPU - temp_result.resize(ndim); - std::fill(temp_result.begin(), temp_result.end(), 1); - size_t factor_prod = 1; - for (auto factor : get_factors(machine)) { - // Avoid exceeding the maximum number of pieces - if (factor * factor_prod > max_pieces) break; - - factor_prod *= factor; - - std::vector remaining; - for (uint32_t idx = 0; idx < temp_shape.size(); ++idx) - remaining.push_back((temp_shape[idx] + temp_result[idx] - 1) / temp_result[idx]); - uint32_t big_dim = std::max_element(remaining.begin(), remaining.end()) - remaining.begin(); - if (big_dim < ndim - 1) { - // Not the last dimension, so do it - temp_result[big_dim] *= factor; - } else { - // Last dim so see if it still bigger than 32 - if (remaining[big_dim] / factor >= 32) { - // go ahead and do it - temp_result[big_dim] *= factor; - } else { - // Won't be see if we can do it with one of the other dimensions - uint32_t next_big_dim = - std::max_element(remaining.begin(), remaining.end() - 1) - remaining.begin(); - if (remaining[next_big_dim] / factor > 0) - temp_result[next_big_dim] *= factor; - else - // Fine just do it on the last dimension - temp_result[big_dim] *= factor; - } - } - } - } - - // Project back onto the original number of dimensions - assert(temp_result.size() == ndim); - std::vector result(shape.size(), 1); - for (uint32_t idx = 0; idx < ndim; ++idx) result[temp_dims[idx]] = temp_result[idx]; - - return Shape(std::move(result)); -} - -Shape PartitionManager::compute_tile_shape(const Shape& extents, const Shape& launch_shape) -{ - assert(extents.size() == launch_shape.size()); - Shape tile_shape; - for (uint32_t idx = 0; idx < extents.size(); ++idx) { - auto x = extents[idx]; - auto y = launch_shape[idx]; - tile_shape.append_inplace((x + y - 1) / y); - } - return std::move(tile_shape); -} - -Legion::IndexPartition PartitionManager::find_index_partition(const Legion::IndexSpace& index_space, - const Tiling& tiling) const -{ - auto finder = tiling_cache_.find(std::make_pair(index_space, tiling)); - if (finder != tiling_cache_.end()) - return finder->second; - else - return Legion::IndexPartition::NO_PART; -} - -void PartitionManager::record_index_partition(const Legion::IndexSpace& index_space, - const Tiling& tiling, - const Legion::IndexPartition& index_partition) -{ - tiling_cache_[std::make_pair(index_space, tiling)] = index_partition; -} - -//////////////////////////////////////////// -// legate::MachineManager -//////////////////////////////////////////// -const mapping::MachineDesc& MachineManager::get_machine() const -{ -#ifdef DEBUG_LEGATE - assert(machines_.size() > 0); -#endif - return machines_.back(); -} - -void MachineManager::push_machine(const mapping::MachineDesc& machine) -{ - machines_.push_back(machine); -} - -void MachineManager::push_machine(mapping::MachineDesc&& machine) -{ - machines_.emplace_back(machine); -} - -void MachineManager::pop_machine() -{ - if (machines_.size() <= 1) throw std::underflow_error("can't pop from the empty machine stack"); - machines_.pop_back(); -} - -//////////////////////////////////////////// -// legate::MachineTracker -//////////////////////////////////////////// - -MachineTracker::MachineTracker(const mapping::MachineDesc& machine) -{ - auto* runtime = Runtime::get_runtime(); - auto result = machine & Runtime::get_runtime()->get_machine(); - if (result.count() == 0) - throw std::runtime_error("Empty machines cannot be used for resource scoping"); - runtime->machine_manager()->push_machine(std::move(result)); -} - -MachineTracker::~MachineTracker() -{ - auto* runtime = Runtime::get_runtime(); - runtime->machine_manager()->pop_machine(); -} - -const mapping::MachineDesc& MachineTracker::get_current_machine() const -{ - return Runtime::get_runtime()->get_machine(); -} - //////////////////////////////////////////////////// // legate::Runtime //////////////////////////////////////////////////// @@ -706,11 +329,8 @@ LibraryContext* Runtime::find_library(const std::string& library_name, { auto finder = libraries_.find(library_name); if (libraries_.end() == finder) { - if (!can_fail) { - log_legate.error("Library %s does not exist", library_name.c_str()); - LEGATE_ABORT; - } else - return nullptr; + if (!can_fail) throw std::out_of_range("Library " + library_name + " does not exist"); + return nullptr; } return finder->second; } @@ -719,10 +339,8 @@ LibraryContext* Runtime::create_library(const std::string& library_name, const ResourceConfig& config, std::unique_ptr mapper) { - if (libraries_.find(library_name) != libraries_.end()) { - log_legate.error("Library %s already exists", library_name.c_str()); - LEGATE_ABORT; - } + if (libraries_.find(library_name) != libraries_.end()) + throw std::invalid_argument("Library " + library_name + " already exists"); log_legate.debug("Library %s is created", library_name.c_str()); if (nullptr == mapper) mapper = std::make_unique(); @@ -796,14 +414,13 @@ mapping::MachineDesc Runtime::slice_machine_for_task(LibraryContext* library, in throw std::invalid_argument(std::move(ss).str()); } - std::set task_targets; + std::vector task_targets; auto& machine = machine_manager_->get_machine(); for (const auto& t : machine.valid_targets()) { - auto variant = mapping::to_variant_code(t); - if (task_info->has_variant(variant)) task_targets.insert(t); + if (task_info->has_variant(mapping::to_variant_code(t))) task_targets.push_back(t); } - auto sliced = machine.only(task_targets); + if (sliced.empty()) { std::stringstream ss; ss << "Task " << task_id << " (" << task_info->name() << ") of library " @@ -811,7 +428,7 @@ mapping::MachineDesc Runtime::slice_machine_for_task(LibraryContext* library, in << "the current machine configuration."; throw std::invalid_argument(ss.str()); } - return std::move(sliced); + return sliced; } // This function should be moved to the library context @@ -1051,9 +668,11 @@ Legion::FutureMap Runtime::dispatch(Legion::IndexTaskLauncher* launcher, Legion::Future Runtime::extract_scalar(const Legion::Future& result, uint32_t idx) const { - auto& machine = get_machine(); - auto variant = mapping::to_variant_code(machine.preferred_target); - TaskLauncher launcher(core_context_, machine, LEGATE_CORE_EXTRACT_SCALAR_TASK_ID, variant); + auto& machine = get_machine(); + auto& provenance = provenance_manager()->get_provenance(); + auto variant = mapping::to_variant_code(machine.preferred_target); + TaskLauncher launcher( + core_context_, machine, provenance, LEGATE_CORE_EXTRACT_SCALAR_TASK_ID, variant); launcher.add_future(result); launcher.add_scalar(Scalar(idx)); return launcher.execute_single(); @@ -1063,9 +682,11 @@ Legion::FutureMap Runtime::extract_scalar(const Legion::FutureMap& result, uint32_t idx, const Legion::Domain& launch_domain) const { - auto& machine = get_machine(); - auto variant = mapping::to_variant_code(machine.preferred_target); - TaskLauncher launcher(core_context_, machine, LEGATE_CORE_EXTRACT_SCALAR_TASK_ID, variant); + auto& machine = get_machine(); + auto& provenance = provenance_manager()->get_provenance(); + auto variant = mapping::to_variant_code(machine.preferred_target); + TaskLauncher launcher( + core_context_, machine, provenance, LEGATE_CORE_EXTRACT_SCALAR_TASK_ID, variant); launcher.add_future_map(result); launcher.add_scalar(Scalar(idx)); return launcher.execute(launch_domain); @@ -1083,6 +704,8 @@ Legion::Future Runtime::reduce_future_map(const Legion::FutureMap& future_map, void Runtime::issue_execution_fence(bool block /*=false*/) { + flush_scheduling_window(); + // FIXME: This needs to be a Legate operation auto future = legion_runtime_->issue_execution_fence(legion_context_); if (block) future.wait(); } @@ -1220,73 +843,6 @@ int32_t start(int32_t argc, char** argv) { return Runtime::start(argc, argv); } int32_t wait_for_shutdown() { return Runtime::get_runtime()->wait_for_shutdown(); } -////////////////////////////////////////////// -// ProvenanceManager -////////////////////////////////////////////// - -ProvenanceManager::ProvenanceManager() { provenance_.push_back(EMPTY_STRING); } - -const std::string& ProvenanceManager::get_provenance() -{ -#ifdef DEBUG_LEGATE - assert(provenance_.size() > 0); -#endif - return provenance_.back(); -} - -void ProvenanceManager::set_provenance(const std::string& p) -{ -#ifdef DEBUG_LEGATE - assert(provenance_.size() > 0); -#endif - provenance_.back() = p; -} - -void ProvenanceManager::reset_provenance() -{ -#ifdef DEBUG_LEGATE - assert(provenance_.size() > 0); -#endif - provenance_.back() = EMPTY_STRING; -} - -void ProvenanceManager::push_provenance(const std::string& p) { provenance_.push_back(p); } - -void ProvenanceManager::pop_provenance() -{ - if (provenance_.size() <= 1) { - throw std::underflow_error("can't pop from an empty provenance stack"); - } - provenance_.pop_back(); -} - -void ProvenanceManager::clear_all() -{ - provenance_.clear(); - provenance_.push_back(EMPTY_STRING); -} - -////////////////////////////////////// -// ProvenanceTracker -////////////////////////////////////// - -ProvenanceTracker::ProvenanceTracker(const std::string& p) -{ - auto* runtime = Runtime::get_runtime(); - runtime->provenance_manager()->push_provenance(p); -} - -ProvenanceTracker::~ProvenanceTracker() -{ - auto* runtime = Runtime::get_runtime(); - runtime->provenance_manager()->pop_provenance(); -} - -const std::string& ProvenanceTracker::get_current_provenance() const -{ - return Runtime::get_runtime()->provenance_manager()->get_provenance(); -} - } // namespace legate extern "C" { diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 0677f55a60..321dcbafa0 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -22,14 +22,11 @@ #include -#include "core/data/scalar.h" #include "core/data/shape.h" #include "core/data/store.h" #include "core/legate_c.h" #include "core/mapping/machine.h" -#include "core/partitioning/restriction.h" #include "core/runtime/resource.h" -#include "core/task/exception.h" #include "core/type/type_info.h" #include "core/utilities/typedefs.h" @@ -39,6 +36,8 @@ namespace legate { class LibraryContext; +class Scalar; +class TaskException; namespace mapping { @@ -61,7 +60,7 @@ struct Core { static void show_progress(const Legion::Task* task, Legion::Context ctx, Legion::Runtime* runtime); - static void report_unexpected_exception(const Legion::Task* task, const legate::TaskException& e); + static void report_unexpected_exception(const Legion::Task* task, const TaskException& e); static void retrieve_tunable(Legion::Context legion_context, Legion::Runtime* legion_runtime, LibraryContext* context); @@ -94,88 +93,23 @@ class AutoTask; class FieldManager; class LogicalRegionField; class LogicalStore; +class MachineManager; class ManualTask; class Operation; -class PartitioningFunctor; +class PartitionManager; +class ProvenanceManager; class RegionManager; -class ResourceConfig; -class Runtime; class Tiling; -class ProvenanceManager { - public: - ProvenanceManager(); - - public: - const std::string& get_provenance(); - - void set_provenance(const std::string& p); - - void reset_provenance(); - - void push_provenance(const std::string& p); - - void pop_provenance(); - - void clear_all(); - - private: - std::vector provenance_; -}; - -class PartitionManager { - public: - PartitionManager(Runtime* runtime, const LibraryContext* context); - - public: - const std::vector& get_factors(const mapping::MachineDesc& machine); - - public: - Shape compute_launch_shape(const mapping::MachineDesc& machine, - const Restrictions& restrictions, - const Shape& shape); - Shape compute_tile_shape(const Shape& extents, const Shape& launch_shape); - - public: - Legion::IndexPartition find_index_partition(const Legion::IndexSpace& index_space, - const Tiling& tiling) const; - void record_index_partition(const Legion::IndexSpace& index_space, - const Tiling& tiling, - const Legion::IndexPartition& index_partition); - - private: - int64_t min_shard_volume_; - std::unordered_map> all_factors_; - - private: - using TilingCacheKey = std::pair; - std::map tiling_cache_; -}; - -class MachineManager { - public: - MachineManager(){}; - - public: - const mapping::MachineDesc& get_machine() const; - - void push_machine(const mapping::MachineDesc& machine); - void push_machine(mapping::MachineDesc&& machine); - - void pop_machine(); - - private: - std::vector machines_; -}; - -struct MachineTracker { - MachineTracker(const mapping::MachineDesc& machine); - - ~MachineTracker(); - - const mapping::MachineDesc& get_current_machine() const; -}; - +/** + * @ingroup runtime + * @brief Class that implements the Legate runtime + * + * The legate runtime provides common services, including as library registration, + * store creation, operator creation and submission, resource management and scoping, + * and communicator management. Legate libraries are free of all these details about + * distribute programming and can focus on their domain logics. + */ class Runtime { public: Runtime(Legion::Runtime* legion_runtime); @@ -186,7 +120,34 @@ class Runtime { friend int32_t start(int32_t argc, char** argv); public: + /** + * @brief Find a library + * + * @param library_name Library name + * @param can_fail Optional flag indicating that the query can fail. When it's true and no + * library is found for a given name, `nullptr` is returned. + * + * @return Context object for the library + * + * @throw std::out_of_range If no library is found for a given name and `can_fail` is `false` + */ LibraryContext* find_library(const std::string& library_name, bool can_fail = false) const; + /** + * @brief Create a library + * + * A library is a collection of tasks and custom reduction operators. The maximum number of + * tasks and reduction operators can be optionally specified with a `ResourceConfig` object. + * Each library can optionally have a mapper that specifies mapping policies for its tasks. + * When no mapper is given, the default mapper is used. + * + * @param library_name Library name. Must be unique to this library + * @param config Optional configuration object + * @param mapper Optional mapper object + * + * @throw std::invalid_argument If a library already exists for a given name + * + * @return Context object for the library + */ LibraryContext* create_library(const std::string& library_name, const ResourceConfig& config = ResourceConfig{}, std::unique_ptr mapper = nullptr); @@ -210,18 +171,68 @@ class Runtime { public: mapping::MachineDesc slice_machine_for_task(LibraryContext* library, int64_t task_id); + /** + * @brief Create an AutoTask + * + * @param library Library to query the task + * @param task_id Library-local Task ID + * + * @return Task object + */ std::unique_ptr create_task(LibraryContext* library, int64_t task_id); + /** + * @brief Create a ManualTask + * + * @param library Library to query the task + * @param task_id Library-local Task ID + * @param launch_shape Launch domain for the task + * + * @return Task object + */ std::unique_ptr create_task(LibraryContext* library, int64_t task_id, const Shape& launch_shape); void flush_scheduling_window(); + /** + * @brief Submits an operation for execution + * + * Each submitted operation goes through multiple pipeline steps to eventually get scheduled + * for execution. It's not guaranteed that the submitted operation starts executing immediately. + * + * @param op Operation to execute + */ void submit(std::unique_ptr op); public: + /** + * @brief Creates an unbound store + * + * @param type Element type + * @param dim Number of dimensions of the store + * + * @return Logical store + */ LogicalStore create_store(std::unique_ptr type, int32_t dim = 1); + /** + * @brief Creates a normal store + * + * @param extents Shape of the store + * @param type Element type + * @param optimize_scalar When true, the runtime internally uses futures optimized for storing + * scalars + * + * @return Logical store + */ LogicalStore create_store(std::vector extents, std::unique_ptr type, bool optimize_scalar = false); + /** + * @brief Creates a normal store out of a `Scalar` object + * + * @param scalar Value of the scalar to create a store with + * + * @return Logical store + */ LogicalStore create_store(const Scalar& scalar); uint64_t get_unique_store_id(); @@ -237,6 +248,7 @@ class Runtime { public: RegionManager* find_or_create_region_manager(const Legion::Domain& shape); FieldManager* find_or_create_field_manager(const Legion::Domain& shape, uint32_t field_size); + MachineManager* machine_manager() const; PartitionManager* partition_manager() const; ProvenanceManager* provenance_manager() const; @@ -273,10 +285,24 @@ class Runtime { Legion::Future reduce_future_map(const Legion::FutureMap& future_map, int32_t reduction_op) const; public: + /** + * @brief Issues an execution fence + * + * An execution fence is a join point in the task graph. All operations prior to a fence must + * finish before any of the subsequent operations start. + * + * @param block When `true`, the control code blocks on the fence and all operations that have + * been submitted prior to this fence. + */ void issue_execution_fence(bool block = false); public: void initialize_toplevel_machine(); + /** + * @brief Returns the machine of the current scope + * + * @return Machine object + */ const mapping::MachineDesc& get_machine() const; public: @@ -291,6 +317,11 @@ class Runtime { static int32_t start(int32_t argc, char** argv); public: + /** + * @brief Returns a singleton runtime object + * + * @return The runtime object + */ static Runtime* get_runtime(); static void create_runtime(Legion::Runtime* legion_runtime); int32_t wait_for_shutdown(); @@ -307,6 +338,7 @@ class Runtime { using FieldManagerKey = std::pair; std::map field_managers_; std::map region_managers_; + MachineManager* machine_manager_{nullptr}; PartitionManager* partition_manager_{nullptr}; ProvenanceManager* provenance_manager_{nullptr}; @@ -337,32 +369,37 @@ class Runtime { private: uint32_t next_type_uid_; std::map, int32_t> reduction_ops_{}; - - private: - MachineManager* machine_manager_{nullptr}; - - public: - MachineManager* machine_manager() const; }; +/** + * @brief Initializes the Legate runtime + * + * @param argc Number of command-line flags + * @param argv Command-line flags + */ void initialize(int32_t argc, char** argv); +/** + * @brief Starts the Legate runtime + * + * This makes the runtime ready to accept requests made via its APIs + * + * @param argc Number of command-line flags + * @param argv Command-line flags + * + * @return Non-zero value when the runtime start-up failed, 0 otherwise + */ int32_t start(int32_t argc, char** argv); +/** + * @brief Waits for the runtime to finish + * + * The client code must call this to make sure all Legate tasks run + * + * @return Non-zero value when the runtime encountered a failure, 0 otherwise + */ int32_t wait_for_shutdown(); -struct ProvenanceTracker { - ProvenanceTracker(const std::string& p); - ~ProvenanceTracker(); - const std::string& get_current_provenance() const; -}; - } // namespace legate -#define TRACK_PROVENANCE(STMT) \ - do { \ - legate::ProvenanceTracker track(std::string(__FILE__) + ":" + std::to_string(__LINE__)); \ - STMT; \ - } while (false) - #include "core/runtime/runtime.inl" diff --git a/src/core/runtime/tracker.cc b/src/core/runtime/tracker.cc new file mode 100644 index 0000000000..1a96e914a9 --- /dev/null +++ b/src/core/runtime/tracker.cc @@ -0,0 +1,70 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/runtime/tracker.h" + +#include "core/runtime/machine_manager.h" +#include "core/runtime/provenance_manager.h" +#include "core/runtime/runtime.h" + +namespace legate { + +////////////////////////////////////// +// ProvenanceTracker +////////////////////////////////////// + +ProvenanceTracker::ProvenanceTracker(const std::string& p) +{ + auto* runtime = Runtime::get_runtime(); + runtime->provenance_manager()->push_provenance(p); +} + +ProvenanceTracker::~ProvenanceTracker() +{ + auto* runtime = Runtime::get_runtime(); + runtime->provenance_manager()->pop_provenance(); +} + +const std::string& ProvenanceTracker::get_current_provenance() const +{ + return Runtime::get_runtime()->provenance_manager()->get_provenance(); +} + +//////////////////////////////////////////// +// legate::MachineTracker +//////////////////////////////////////////// + +MachineTracker::MachineTracker(const mapping::MachineDesc& machine) +{ + auto* runtime = Runtime::get_runtime(); + auto result = machine & Runtime::get_runtime()->get_machine(); + if (result.count() == 0) + throw std::runtime_error("Empty machines cannot be used for resource scoping"); + runtime->machine_manager()->push_machine(std::move(result)); +} + +MachineTracker::~MachineTracker() +{ + auto* runtime = Runtime::get_runtime(); + runtime->machine_manager()->pop_machine(); +} + +const mapping::MachineDesc& MachineTracker::get_current_machine() const +{ + return Runtime::get_runtime()->get_machine(); +} + +} // namespace legate diff --git a/src/core/runtime/tracker.h b/src/core/runtime/tracker.h new file mode 100644 index 0000000000..29b616e1a8 --- /dev/null +++ b/src/core/runtime/tracker.h @@ -0,0 +1,91 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "core/mapping/machine.h" + +/** + * @file + * @brief Class definitions for legate::ProvenanceTracker and legate::MachineTracker + */ + +namespace legate { + +/** + * @ingroup util + * @brief A helper class to set provenance information for the scope + * + * Client programs often want to attach provenance information to each of their operations and hvae + * it rendered in profiling outputs. In such scenarios, client programs can create a + * `ProvenanceTracker` object with the provenance string in a scope and issue operations. All the + * issued operations then will be associated with the provenance information, which will be attached + * to them in profiling outputs. + */ +struct ProvenanceTracker { + /** + * @brief Creates a trakcer that sets a given provenance string for the scope + * + * @param provenance Provenance information in string + */ + ProvenanceTracker(const std::string& provenance); + + /** + * @brief Pops out the provenance string set by this tracker + */ + ~ProvenanceTracker(); + + /** + * @brief Returns the current provenance string + * + * @return Provenance string + */ + const std::string& get_current_provenance() const; +}; + +/** + * @ingroup util + * @brief A helper class to configure machine for the scope + * + * By default, Legate operations target the entire machine available for the program. When a client + * program wants to assign a subset of the machine to its operations, it can subdivide the machine + * using the machine API (see `MachineDesc` for details) and set a sub-machine for the scope using + * `MachineTracker`. All operations within the scope where the `MachineTracker` object is alive will + * only target that sub-machine, instead of the entire machine. + * + */ +struct MachineTracker { + /** + * @brief Creates a trakcer that sets a given machine for the scope + * + * @param machine Machine to use for the scope + */ + MachineTracker(const mapping::MachineDesc& machine); + + /** + * @brief Pops out the machine set by this tracker + */ + ~MachineTracker(); + + /** + * @brief Returns the current machine + * + * @return Machine + */ + const mapping::MachineDesc& get_current_machine() const; +}; + +} // namespace legate diff --git a/src/core/type/type_info.h b/src/core/type/type_info.h index cb9ca9c9c2..d327930d16 100644 --- a/src/core/type/type_info.h +++ b/src/core/type/type_info.h @@ -346,9 +346,7 @@ class StructType : public ExtensionType { * @ingroup types * @brief Creates a metadata object for a primitive type * - * @param uid Unique ID - * @param element_type Type of the array elements - * @param N Size of the array + * @param code Type code * * @return Type object */ @@ -379,6 +377,7 @@ std::unique_ptr fixed_array_type(std::unique_ptr element_type, * @brief Creates a metadata object for a struct type * * @param field_types A vector of field types + * @param align If true, fields in the struct are aligned * * @return Type object */ diff --git a/src/core/utilities/buffer_builder.h b/src/core/utilities/buffer_builder.h index 4c7bef538c..bba3133761 100644 --- a/src/core/utilities/buffer_builder.h +++ b/src/core/utilities/buffer_builder.h @@ -22,20 +22,56 @@ namespace legate { +/** + * @ingroup util + * @brief A helper class to serialize values into a contiguous buffer + */ class BufferBuilder { public: + /** + * @brief Creates an empty buffer builder + */ BufferBuilder(); public: + /** + * @brief Serializes a value + * + * @param value Value to serialize + */ template void pack(const T& value); + /** + * @brief Serializes multiple values + * + * @param values Values to serialize in a vector + */ template void pack(const std::vector& values); + /** + * @brief Serializes multiple values + * + * @param values Values to serialize in a tuple + */ template void pack(const tuple& values); + /** + * @brief Serializes an arbitrary allocation + * + * The caller should make sure that `(char*)buffer + (size - 1)` is a valid address. + * + * @param buffer Buffer to serialize + * @param size Size of the buffer + */ void pack_buffer(const void* buffer, size_t size); public: + /** + * @brief Wraps the `BufferBuilder`'s internal allocation with a Legion `UntypedBuffer`. + * + * Since `UntypedBuffer` does not make a copy of the input allocation, the returned buffer + * is good to use only as long as this buffer builder is alive. + */ Legion::UntypedBuffer to_legion_buffer() const; private: diff --git a/src/legate.h b/src/legate.h index e1d92fee50..5da162f91c 100644 --- a/src/legate.h +++ b/src/legate.h @@ -29,10 +29,15 @@ #include "core/data/scalar.h" #include "core/data/store.h" #include "core/legate_c.h" +#include "core/mapping/mapping.h" #include "core/partitioning/constraint.h" #include "core/partitioning/partition.h" +#include "core/runtime/machine_manager.h" #include "core/runtime/operation.h" +#include "core/runtime/provenance_manager.h" #include "core/runtime/runtime.h" +#include "core/runtime/tracker.h" +#include "core/task/exception.h" #include "core/task/registrar.h" #include "core/task/task.h" #include "core/type/type_traits.h" diff --git a/tests/cpp/integration/provenance.cc b/tests/cpp/integration/provenance.cc index 4641c767c1..cc4241bcfa 100644 --- a/tests/cpp/integration/provenance.cc +++ b/tests/cpp/integration/provenance.cc @@ -106,18 +106,25 @@ void test_clear_provenance(legate::LibraryContext* context) void test_provenance_tracker(legate::LibraryContext* context) { + legate::ProvenanceTracker track(std::string(__FILE__) + ":" + std::to_string(__LINE__)); auto runtime = legate::Runtime::get_runtime(); // auto task auto task = runtime->create_task(context, PROVENANCE); - std::string provenance = "provenance.cc:114"; + std::string provenance = "provenance.cc:109"; task->add_scalar_arg(legate::Scalar(provenance)); - TRACK_PROVENANCE(runtime->submit(std::move(task))); + runtime->submit(std::move(task)); } void test_nested_provenance_tracker(legate::LibraryContext* context) { - legate::ProvenanceTracker track(std::string(__FILE__) + std::to_string(__LINE__)); + legate::ProvenanceTracker track(std::string(__FILE__) + ":" + std::to_string(__LINE__)); test_provenance_tracker(context); + // The provenance string used by test_provenance_tracker should be popped out at this point + auto runtime = legate::Runtime::get_runtime(); + auto task = runtime->create_task(context, PROVENANCE); + std::string provenance = "provenance.cc:120"; + task->add_scalar_arg(legate::Scalar(provenance)); + runtime->submit(std::move(task)); } void test_manual_tracker(legate::LibraryContext* context) diff --git a/tests/cpp/unit/machine.cc b/tests/cpp/unit/machine.cc index 69ea5e3a78..dd67cdc4ca 100644 --- a/tests/cpp/unit/machine.cc +++ b/tests/cpp/unit/machine.cc @@ -62,7 +62,7 @@ TEST(Machine, ProcessorRange) // get_node_range { legate::mapping::ProcessorRange range(0, 7, 2); - EXPECT_EQ(range.get_node_range(), std::make_pair(uint32_t(0), uint32_t(3))); + EXPECT_EQ(range.get_node_range(), std::make_pair(uint32_t(0), uint32_t(4))); } // intersection nonempty @@ -110,8 +110,8 @@ TEST(Machine, MachineDesc) { legate::mapping::MachineDesc machine; EXPECT_EQ(machine.preferred_target, legate::mapping::TaskTarget::CPU); - EXPECT_THROW(machine.count(), std::runtime_error); - EXPECT_THROW(machine.count(legate::mapping::TaskTarget::GPU), std::runtime_error); + EXPECT_EQ(machine.count(), 0); + EXPECT_EQ(machine.count(legate::mapping::TaskTarget::GPU), 0); EXPECT_EQ(machine.processor_range(), legate::mapping::ProcessorRange(0, 0, 1)); EXPECT_EQ(machine.processor_range(legate::mapping::TaskTarget::GPU), legate::mapping::ProcessorRange(0, 0, 1)); @@ -254,7 +254,9 @@ TEST(Machine, MachineDesc) { legate::mapping::MachineDesc machine1({{legate::mapping::TaskTarget::CPU, cpu_range}, {legate::mapping::TaskTarget::GPU, gpu_range}}); - EXPECT_THROW(machine1.slice(0, 1), std::runtime_error); + legate::mapping::MachineDesc expected( + {{legate::mapping::TaskTarget::GPU, gpu_range.slice(0, 1)}}); + EXPECT_EQ(machine1.slice(0, 1), expected); auto new_machine1 = machine1.slice(0, 2, legate::mapping::TaskTarget::GPU); EXPECT_EQ(new_machine1.preferred_target, legate::mapping::TaskTarget::GPU); From 5f4fa4d0cdb1c8cce4f4b82a88271267463585a5 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 24 May 2023 16:04:09 -0700 Subject: [PATCH 0128/1425] Remove unnecessary std::move and std::forward * Remove unnecessary uses of std::move and std::forward See merge request legate/legate.core.internal!37 --- src/core/data/logical_store.cc | 11 ++++------- src/core/data/logical_store_detail.cc | 8 ++++---- src/core/data/store.cc | 16 ++++++++-------- src/core/data/transform.cc | 12 +++++------- src/core/mapping/instance_manager.cc | 8 ++++---- src/core/mapping/machine.cc | 4 ++-- src/core/mapping/mapping.cc | 14 +++++++------- src/core/mapping/store.cc | 4 ++-- src/core/partitioning/constraint.cc | 7 +++---- src/core/partitioning/constraint_graph.cc | 4 ++-- src/core/partitioning/partition.cc | 11 +++++------ src/core/partitioning/partitioner.cc | 5 ++--- src/core/partitioning/restriction.cc | 2 +- src/core/runtime/context.cc | 2 +- src/core/runtime/launcher.cc | 4 ++-- src/core/runtime/operation.cc | 9 ++++----- src/core/task/task.cc | 2 +- src/core/task/task.inl | 2 +- src/core/type/type_info.cc | 7 +++---- src/core/utilities/deserializer.cc | 2 +- src/core/utilities/deserializer.h | 2 +- src/core/utilities/tuple.inl | 2 +- 22 files changed, 64 insertions(+), 74 deletions(-) diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index a16542f4da..07d43ba692 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -43,10 +43,7 @@ Domain LogicalRegionField::domain() const return Runtime::get_runtime()->get_index_space_domain(lr_.get_index_space()); } -LogicalStore::LogicalStore(std::shared_ptr&& impl) - : impl_(std::forward(impl)) -{ -} +LogicalStore::LogicalStore(std::shared_ptr&& impl) : impl_(std::move(impl)) {} int32_t LogicalStore::dim() const { return impl_->dim(); } @@ -82,12 +79,12 @@ LogicalStore LogicalStore::slice(int32_t dim, std::slice sl) const LogicalStore LogicalStore::transpose(std::vector&& axes) const { - return LogicalStore(impl_->transpose(std::forward(axes))); + return LogicalStore(impl_->transpose(std::move(axes))); } LogicalStore LogicalStore::delinearize(int32_t dim, std::vector&& sizes) const { - return LogicalStore(impl_->delinearize(dim, std::forward(sizes))); + return LogicalStore(impl_->delinearize(dim, std::move(sizes))); } std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) @@ -101,7 +98,7 @@ void LogicalStore::set_key_partition(const Partition* partition) } LogicalStorePartition::LogicalStorePartition(std::shared_ptr&& impl) - : impl_(std::forward(impl)) + : impl_(std::move(impl)) { } diff --git a/src/core/data/logical_store_detail.cc b/src/core/data/logical_store_detail.cc index 2e16ec57b6..561a8f3dc9 100644 --- a/src/core/data/logical_store_detail.cc +++ b/src/core/data/logical_store_detail.cc @@ -122,7 +122,7 @@ Partition* Storage::find_or_create_key_partition(const mapping::MachineDesc& mac void Storage::set_key_partition(std::unique_ptr&& key_partition) { - key_partition_ = std::forward(key_partition); + key_partition_ = std::move(key_partition); } void Storage::reset_key_partition() { key_partition_ = nullptr; } @@ -158,7 +158,7 @@ StoragePartition::StoragePartition(std::shared_ptr parent, LogicalStore::LogicalStore(std::shared_ptr&& storage) : store_id_(Runtime::get_runtime()->get_unique_store_id()), - storage_(std::forward(storage)), + storage_(std::move(storage)), transform_(std::make_shared()) { if (!unbound()) extents_ = storage_->extents(); @@ -173,9 +173,9 @@ LogicalStore::LogicalStore(Shape&& extents, const std::shared_ptr& storage, std::shared_ptr&& transform) : store_id_(Runtime::get_runtime()->get_unique_store_id()), - extents_(std::forward(extents)), + extents_(std::move(extents)), storage_(storage), - transform_(std::forward(transform)) + transform_(std::move(transform)) { #ifdef DEBUG_LEGATE assert(transform_ != nullptr); diff --git a/src/core/data/store.cc b/src/core/data/store.cc index e8bc01e0aa..898c8b8770 100644 --- a/src/core/data/store.cc +++ b/src/core/data/store.cc @@ -219,7 +219,7 @@ Store::Store(int32_t dim, type_(std::move(type)), redop_id_(redop_id), future_(future), - transform_(std::forward(transform)), + transform_(std::move(transform)), readable_(true) { } @@ -234,8 +234,8 @@ Store::Store(int32_t dim, dim_(dim), type_(std::move(type)), redop_id_(redop_id), - region_field_(std::forward(region_field)), - transform_(std::forward(transform)) + region_field_(std::move(region_field)), + transform_(std::move(transform)) { readable_ = region_field_.is_readable(); writable_ = region_field_.is_writable(); @@ -251,8 +251,8 @@ Store::Store(int32_t dim, dim_(dim), type_(std::move(type)), redop_id_(-1), - unbound_field_(std::forward(unbound_field)), - transform_(std::forward(transform)) + unbound_field_(std::move(unbound_field)), + transform_(std::move(transform)) { } @@ -282,7 +282,7 @@ Store::Store(int32_t dim, dim_(dim), type_(std::move(type)), redop_id_(redop_id), - region_field_(std::forward(region_field)), + region_field_(std::move(region_field)), transform_(transform) { readable_ = region_field_.is_readable(); @@ -297,8 +297,8 @@ Store::Store(Store&& other) noexcept type_(std::move(other.type_)), redop_id_(other.redop_id_), future_(other.future_), - region_field_(std::forward(other.region_field_)), - unbound_field_(std::forward(other.unbound_field_)), + region_field_(std::move(other.region_field_)), + unbound_field_(std::move(other.unbound_field_)), transform_(std::move(other.transform_)), readable_(other.readable_), writable_(other.writable_), diff --git a/src/core/data/transform.cc b/src/core/data/transform.cc index c8498b5b35..b56e1cd069 100644 --- a/src/core/data/transform.cc +++ b/src/core/data/transform.cc @@ -34,14 +34,13 @@ Legion::DomainAffineTransform combine(const Legion::DomainAffineTransform& lhs, TransformStack::TransformStack(std::unique_ptr&& transform, const std::shared_ptr& parent) - : transform_(std::forward(transform)), parent_(parent) + : transform_(std::move(transform)), parent_(parent) { } TransformStack::TransformStack(std::unique_ptr&& transform, std::shared_ptr&& parent) - : transform_(std::forward(transform)), - parent_(std::forward(parent)) + : transform_(std::move(transform)), parent_(std::move(parent)) { } @@ -139,13 +138,12 @@ std::unique_ptr TransformStack::pop() transform_ = std::move(parent_->transform_); parent_ = std::move(parent_->parent_); } - return std::move(result); + return result; } std::shared_ptr TransformStack::push(std::unique_ptr&& transform) { - return std::make_shared(std::forward(transform), - shared_from_this()); + return std::make_shared(std::move(transform), shared_from_this()); } void TransformStack::dump() const { std::cerr << *this << std::endl; } @@ -596,7 +594,7 @@ Restrictions Delinearize::invert(const Restrictions& restrictions) const { auto result = restrictions; for (uint32_t dim = 1; dim < sizes_.size(); ++dim) result.remove_inplace(dim + 1); - return std::move(result); + return result; } void Delinearize::pack(BufferBuilder& buffer) const diff --git a/src/core/mapping/instance_manager.cc b/src/core/mapping/instance_manager.cc index e7d6bc0252..8b9296200d 100644 --- a/src/core/mapping/instance_manager.cc +++ b/src/core/mapping/instance_manager.cc @@ -30,7 +30,7 @@ RegionGroup::RegionGroup(const std::set& rs, const Domain bound) } RegionGroup::RegionGroup(std::set&& rs, const Domain bound) - : regions(std::forward(rs)), bounding_box(bound) + : regions(std::move(rs)), bounding_box(bound) { } @@ -38,7 +38,7 @@ std::vector RegionGroup::get_regions() const { std::vector result; result.insert(result.end(), regions.begin(), regions.end()); - return std::move(result); + return result; } bool RegionGroup::subsumes(const RegionGroup* other) @@ -233,7 +233,7 @@ std::set InstanceSet::record_instance(RegionGroupP group, dump_and_sanity_check(); #endif - return std::move(replaced); + return replaced; } bool InstanceSet::erase(Instance inst) @@ -372,7 +372,7 @@ RegionGroupP InstanceManager::find_region_group(const Region& region, << memory << "," << exact << ") ~> " << *result; #endif - return std::move(result); + return result; } std::set InstanceManager::record_instance( diff --git a/src/core/mapping/machine.cc b/src/core/mapping/machine.cc index 2e89896a89..2ce7c54b36 100644 --- a/src/core/mapping/machine.cc +++ b/src/core/mapping/machine.cc @@ -175,7 +175,7 @@ std::vector MachineDesc::valid_targets() const { std::vector result; for (auto& [target, _] : processor_ranges) result.push_back(target); - return std::move(result); + return result; } std::vector MachineDesc::valid_targets_except( @@ -184,7 +184,7 @@ std::vector MachineDesc::valid_targets_except( std::vector result; for (auto& [target, _] : processor_ranges) if (to_exclude.find(target) == to_exclude.end()) result.push_back(target); - return std::move(result); + return result; } size_t MachineDesc::count() const { return count(preferred_target); } diff --git a/src/core/mapping/mapping.cc b/src/core/mapping/mapping.cc index d8c4ce6193..ed0ef8f3d1 100644 --- a/src/core/mapping/mapping.cc +++ b/src/core/mapping/mapping.cc @@ -25,7 +25,7 @@ namespace legate { namespace mapping { DimOrdering::DimOrdering(Kind _kind, std::vector&& _dims) - : kind(_kind), dims(std::forward(_dims)) + : kind(_kind), dims(std::move(_dims)) { } @@ -35,7 +35,7 @@ DimOrdering::DimOrdering(Kind _kind, std::vector&& _dims) /*static*/ DimOrdering DimOrdering::custom_order(std::vector&& dims) { - return DimOrdering(Kind::CUSTOM, std::forward(dims)); + return DimOrdering(Kind::CUSTOM, std::move(dims)); } Memory::Kind get_memory_kind(StoreTarget target) @@ -88,7 +88,7 @@ void DimOrdering::set_fortran_order() { kind = Kind::FORTRAN; } void DimOrdering::set_custom_order(std::vector&& dims) { kind = Kind::CUSTOM; - dims = std::forward&&>(dims); + dims = std::move(dims); } bool InstanceMappingPolicy::operator==(const InstanceMappingPolicy& other) const @@ -130,7 +130,7 @@ void InstanceMappingPolicy::populate_layout_constraints( InstanceMappingPolicy policy{}; policy.target = target; policy.exact = exact; - return std::move(policy); + return policy; } bool StoreMapping::for_future() const @@ -180,7 +180,7 @@ std::set StoreMapping::requirement_indices() const if (store.get().is_future()) continue; indices.insert(store.get().region_field().index()); } - return std::move(indices); + return indices; } std::set StoreMapping::requirements() const @@ -192,7 +192,7 @@ std::set StoreMapping::requirements() const if (!req->region.exists()) continue; reqs.insert(req); } - return std::move(reqs); + return reqs; } void StoreMapping::populate_layout_constraints( @@ -223,7 +223,7 @@ void StoreMapping::populate_layout_constraints( StoreMapping mapping{}; mapping.policy = InstanceMappingPolicy::default_policy(target, exact); mapping.stores.emplace_back(store); - return std::move(mapping); + return mapping; } } // namespace mapping diff --git a/src/core/mapping/store.cc b/src/core/mapping/store.cc index 4acfe1e8bd..23429cc0fb 100644 --- a/src/core/mapping/store.cc +++ b/src/core/mapping/store.cc @@ -56,7 +56,7 @@ Store::Store(int32_t dim, type_(std::move(type)), redop_id_(-1), future_(future), - transform_(std::forward(transform)) + transform_(std::move(transform)) { } @@ -74,7 +74,7 @@ Store::Store(Legion::Mapping::MapperRuntime* runtime, type_(std::move(type)), redop_id_(redop_id), region_field_(region_field), - transform_(std::forward(transform)), + transform_(std::move(transform)), runtime_(runtime), context_(context) { diff --git a/src/core/partitioning/constraint.cc b/src/core/partitioning/constraint.cc index f3401548aa..60e9fb7bfc 100644 --- a/src/core/partitioning/constraint.cc +++ b/src/core/partitioning/constraint.cc @@ -51,7 +51,7 @@ std::string Variable::to_string() const } Alignment::Alignment(std::unique_ptr&& lhs, std::unique_ptr&& rhs) - : lhs_(std::forward(lhs)), rhs_(std::forward(rhs)) + : lhs_(std::move(lhs)), rhs_(std::move(rhs)) { } @@ -69,7 +69,7 @@ std::string Alignment::to_string() const } Broadcast::Broadcast(std::unique_ptr variable, tuple&& axes) - : variable_(std::move(variable)), axes_(std::forward>(axes)) + : variable_(std::move(variable)), axes_(std::move(axes)) { } @@ -99,8 +99,7 @@ std::unique_ptr broadcast(const Variable* variable, const tuple broadcast(const Variable* variable, tuple&& axes) { - return std::make_unique(std::make_unique(*variable), - std::forward>(axes)); + return std::make_unique(std::make_unique(*variable), std::move(axes)); } } // namespace legate diff --git a/src/core/partitioning/constraint_graph.cc b/src/core/partitioning/constraint_graph.cc index 2d5bbb8965..1ab8a13c35 100644 --- a/src/core/partitioning/constraint_graph.cc +++ b/src/core/partitioning/constraint_graph.cc @@ -115,7 +115,7 @@ std::vector ConstraintGraph::find_equivalence_class( result.push_back(equiv_class->partition_symbol); equiv_class = equiv_class->next; } - return std::move(result); + return result; } Restrictions ConstraintGraph::find_restrictions(const Variable* partition_symbol) const @@ -127,7 +127,7 @@ Restrictions ConstraintGraph::find_restrictions(const Variable* partition_symbol join_inplace(result, equiv_class->restrictions); equiv_class = equiv_class->next; } - return std::move(result); + return result; } void ConstraintGraph::dump() diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index 84fe952112..54304a23d7 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -58,9 +58,9 @@ std::string NoPartition::to_string() const { return "NoPartition"; } Tiling::Tiling(Shape&& tile_shape, Shape&& color_shape, Shape&& offsets) : Partition(), - tile_shape_(std::forward(tile_shape)), - color_shape_(std::forward(color_shape)), - offsets_(std::forward(offsets)) + tile_shape_(std::move(tile_shape)), + color_shape_(std::move(color_shape)), + offsets_(std::move(offsets)) { if (offsets_.empty()) offsets_ = Shape(tile_shape_.size(), 0); assert(tile_shape_.size() == color_shape_.size()); @@ -156,9 +156,8 @@ std::unique_ptr create_tiling(Shape&& tile_shape, Shape&& color_shape, Shape&& offsets /*= {}*/) { - return std::make_unique(std::forward(tile_shape), - std::forward(color_shape), - std::forward(offsets)); + return std::make_unique( + std::move(tile_shape), std::move(color_shape), std::move(offsets)); } std::ostream& operator<<(std::ostream& out, const Partition& partition) diff --git a/src/core/partitioning/partitioner.cc b/src/core/partitioning/partitioner.cc index f86024704f..c83f7367af 100644 --- a/src/core/partitioning/partitioner.cc +++ b/src/core/partitioning/partitioner.cc @@ -187,8 +187,7 @@ void Strategy::compute_launch_domains() // legate::Partitioner //////////////////////////////////////////////////// -Partitioner::Partitioner(std::vector&& operations) - : operations_(std::forward>(operations)) +Partitioner::Partitioner(std::vector&& operations) : operations_(std::move(operations)) { } @@ -240,7 +239,7 @@ std::unique_ptr Partitioner::solve() strategy->compute_launch_domains(); - return std::move(strategy); + return strategy; } void Partitioner::solve_for_unbound_stores(std::vector& partition_symbols, diff --git a/src/core/partitioning/restriction.cc b/src/core/partitioning/restriction.cc index ac31687161..568c3fe34b 100644 --- a/src/core/partitioning/restriction.cc +++ b/src/core/partitioning/restriction.cc @@ -24,7 +24,7 @@ tuple join(const tuple& lhs, const tuple& { auto result = lhs; join_inplace(result, rhs); - return std::move(result); + return result; } void join_inplace(Restrictions& lhs, const Restrictions& rhs) diff --git a/src/core/runtime/context.cc b/src/core/runtime/context.cc index 7c6addb6a6..d6a9268581 100644 --- a/src/core/runtime/context.cc +++ b/src/core/runtime/context.cc @@ -293,7 +293,7 @@ std::vector TaskContext::get_return_values() const } } - return std::move(return_values); + return return_values; } const std::string& TaskContext::get_provenance() const { return task_->get_provenance_string(); } diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/launcher.cc index 82ff4a295f..1a4d4aa10c 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -204,7 +204,7 @@ std::unique_ptr TaskLauncher::build_single_task() req_analyzer_->populate_launcher(single_task.get()); out_analyzer_->populate_output_requirements(output_requirements_); - return std::move(single_task); + return single_task; } std::unique_ptr TaskLauncher::build_index_task( @@ -245,7 +245,7 @@ std::unique_ptr TaskLauncher::build_index_task( req_analyzer_->populate_launcher(index_task.get()); out_analyzer_->populate_output_requirements(output_requirements_); - return std::move(index_task); + return index_task; } void TaskLauncher::bind_region_fields_to_unbound_stores() diff --git a/src/core/runtime/operation.cc b/src/core/runtime/operation.cc index 507569563f..99060b09c5 100644 --- a/src/core/runtime/operation.cc +++ b/src/core/runtime/operation.cc @@ -39,7 +39,7 @@ namespace legate { Operation::Operation(LibraryContext* library, uint64_t unique_id, mapping::MachineDesc&& machine) : library_(library), unique_id_(unique_id), - machine_(std::forward(machine)), + machine_(std::move(machine)), provenance_(Runtime::get_runtime()->provenance_manager()->get_provenance()) { } @@ -67,7 +67,7 @@ Task::Task(LibraryContext* library, int64_t task_id, uint64_t unique_id, mapping::MachineDesc&& machine) - : Operation(library, unique_id, std::forward(machine)), task_id_(task_id) + : Operation(library, unique_id, std::move(machine)), task_id_(task_id) { } @@ -201,7 +201,7 @@ AutoTask::AutoTask(LibraryContext* library, int64_t task_id, uint64_t unique_id, mapping::MachineDesc&& machine) - : Task(library, task_id, unique_id, std::forward(machine)) + : Task(library, task_id, unique_id, std::move(machine)) { } @@ -259,8 +259,7 @@ ManualTask::ManualTask(LibraryContext* library, const Shape& launch_shape, uint64_t unique_id, mapping::MachineDesc&& machine) - : Task(library, task_id, unique_id, std::forward(machine)), - strategy_(std::make_unique()) + : Task(library, task_id, unique_id, std::move(machine)), strategy_(std::make_unique()) { strategy_->set_launch_shape(this, launch_shape); } diff --git a/src/core/task/task.cc b/src/core/task/task.cc index a8f6a58736..83901b210c 100644 --- a/src/core/task/task.cc +++ b/src/core/task/task.cc @@ -39,7 +39,7 @@ std::string generate_task_name(const std::type_info& ti) char* demangled = abi::__cxa_demangle(ti.name(), 0, 0, &status); result = demangled; free(demangled); - return std::move(result); + return result; } void task_wrapper(VariantImpl variant_impl, diff --git a/src/core/task/task.inl b/src/core/task/task.inl index 34cc9f89ea..67f8895da5 100644 --- a/src/core/task/task.inl +++ b/src/core/task/task.inl @@ -70,7 +70,7 @@ template detail::VariantHelper::record(task_info.get(), all_options); detail::VariantHelper::record(task_info.get(), all_options); detail::VariantHelper::record(task_info.get(), all_options); - return std::move(task_info); + return task_info; } template diff --git a/src/core/type/type_info.cc b/src/core/type/type_info.cc index 20cc340f37..420d70f5fc 100644 --- a/src/core/type/type_info.cc +++ b/src/core/type/type_info.cc @@ -153,7 +153,7 @@ StructType::StructType(int32_t uid, aligned_(align), alignment_(1), size_(0), - field_types_(std::forward(field_types)) + field_types_(std::move(field_types)) { offsets_.reserve(field_types_.size()); if (aligned_) { @@ -261,9 +261,8 @@ std::unique_ptr fixed_array_type(std::unique_ptr element_type, std::unique_ptr struct_type(std::vector>&& field_types, bool align) noexcept(false) { - return std::make_unique(Runtime::get_runtime()->get_type_uid(), - std::forward>>(field_types), - align); + return std::make_unique( + Runtime::get_runtime()->get_type_uid(), std::move(field_types), align); } std::ostream& operator<<(std::ostream& ostream, const Type::Code& code) diff --git a/src/core/utilities/deserializer.cc b/src/core/utilities/deserializer.cc index 3eba5120de..c70fef9b28 100644 --- a/src/core/utilities/deserializer.cc +++ b/src/core/utilities/deserializer.cc @@ -202,7 +202,7 @@ CopyDeserializer::CopyDeserializer(const Legion::Copy* copy, Legion::Mapping::MapperRuntime* runtime, Legion::Mapping::MapperContext context) : BaseDeserializer(copy->mapper_data, copy->mapper_data_size), - all_reqs_(std::forward>(all_requirements)), + all_reqs_(std::move(all_requirements)), curr_reqs_(all_reqs_.begin()), runtime_(runtime), context_(context), diff --git a/src/core/utilities/deserializer.h b/src/core/utilities/deserializer.h index 1592204fad..ad53f9c11e 100644 --- a/src/core/utilities/deserializer.h +++ b/src/core/utilities/deserializer.h @@ -43,7 +43,7 @@ class BaseDeserializer { { T value; static_cast(this)->_unpack(value); - return std::move(value); + return value; } public: diff --git a/src/core/utilities/tuple.inl b/src/core/utilities/tuple.inl index 50ce78635a..13b3272844 100644 --- a/src/core/utilities/tuple.inl +++ b/src/core/utilities/tuple.inl @@ -31,7 +31,7 @@ tuple::tuple(const std::vector& values) : data_(values) } template -tuple::tuple(std::vector&& values) : data_(std::forward>(values)) +tuple::tuple(std::vector&& values) : data_(std::move(values)) { } From 57984834440df8963847b97a320a0253be8d86b7 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 24 May 2023 18:49:57 -0700 Subject: [PATCH 0129/1425] Safety checks for value accessors in `legate::Scalar` * Add safety checks to value accessors in legate::Scalar * Use a template magic to provide a nicer way to create struct types See merge request legate/legate.core.internal!38 --- src/core/data/scalar.h | 10 ++++++++++ src/core/data/scalar.inl | 37 +++++++++++++++++++++++++++++++++++-- src/core/type/type_info.h | 21 +++++++++++++++++++++ 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/src/core/data/scalar.h b/src/core/data/scalar.h index 8b3207d49e..0fb62fad67 100644 --- a/src/core/data/scalar.h +++ b/src/core/data/scalar.h @@ -67,6 +67,16 @@ class Scalar { */ template Scalar(T value); + /** + * @brief Creates an owned scalar of a specified type from a scalar value + * + * @tparam T The scalar type to wrap + * + * @param type A type of the scalar + * @param value A scalar value to create a `Scalar` with + */ + template + Scalar(T value, std::unique_ptr type); /** * @brief Creates an owned scalar from a string. The value from the * original string will be copied. diff --git a/src/core/data/scalar.inl b/src/core/data/scalar.inl index 7d61d93063..88799f2dc9 100644 --- a/src/core/data/scalar.inl +++ b/src/core/data/scalar.inl @@ -19,6 +19,23 @@ namespace legate { template Scalar::Scalar(T value) : own_(true), type_(primitive_type(legate_type_code_of)) { + static_assert(legate_type_code_of != Type::Code::FIXED_ARRAY); + static_assert(legate_type_code_of != Type::Code::STRUCT); + static_assert(legate_type_code_of != Type::Code::STRING); + static_assert(legate_type_code_of != Type::Code::INVALID); + + auto buffer = malloc(sizeof(T)); + memcpy(buffer, &value, sizeof(T)); + data_ = buffer; +} + +template +Scalar::Scalar(T value, std::unique_ptr type) : own_(true), type_(std::move(type)) +{ + if (type_->code == Type::Code::INVALID) + throw std::invalid_argument("Invalid type cannot be used"); + if (type_->size() != sizeof(T)) + throw std::invalid_argument("Size of the value doesn't match with the type"); auto buffer = malloc(sizeof(T)); memcpy(buffer, &value, sizeof(T)); data_ = buffer; @@ -37,12 +54,17 @@ Scalar::Scalar(const std::vector& values) template VAL Scalar::value() const { + if (sizeof(VAL) != type_->size()) + throw std::invalid_argument("Size of the scalar is " + std::to_string(type_->size()) + + ", but the requested type has size " + std::to_string(sizeof(VAL))); return *static_cast(data_); } template <> inline std::string Scalar::value() const { + if (type_->code != Type::Code::STRING) + throw std::invalid_argument("Type of the scalar is not string"); // Getting a span of a temporary scalar is illegal in general, // but we know this is safe as the span's pointer is held by this object. auto len = *static_cast(data_); @@ -55,10 +77,21 @@ template Span Scalar::values() const { if (type_->code == Type::Code::FIXED_ARRAY) { - auto size = static_cast(type_.get())->num_elements(); + auto arr_type = static_cast(type_.get()); + const auto& elem_type = arr_type->element_type(); + if (sizeof(VAL) != elem_type.size()) + throw std::invalid_argument( + "The scalar's element type has size " + std::to_string(elem_type.size()) + + ", but the requested element type has size " + std::to_string(sizeof(VAL))); + auto size = arr_type->num_elements(); return Span(reinterpret_cast(data_), size); - } else + } else { + if (sizeof(VAL) != type_->size()) + throw std::invalid_argument("Size of the scalar is " + std::to_string(type_->size()) + + ", but the requested element type has size " + + std::to_string(sizeof(VAL))); return Span(static_cast(data_), 1); + } } template <> diff --git a/src/core/type/type_info.h b/src/core/type/type_info.h index d327930d16..a200cb6096 100644 --- a/src/core/type/type_info.h +++ b/src/core/type/type_info.h @@ -384,6 +384,27 @@ std::unique_ptr fixed_array_type(std::unique_ptr element_type, std::unique_ptr struct_type(std::vector>&& field_types, bool align = false) noexcept(false); +/** + * @ingroup types + * @brief Creates a metadata object for a struct type + * + * @param align If true, fields in the struct are aligned + * @param field_types Field types + * + * @return Type object + */ +template +std::enable_if_t...>, std::unique_ptr> +struct_type(bool align, std::unique_ptr... field_types) noexcept(false) +{ + std::vector> vec_field_types; + auto move_field_type = [&vec_field_types](auto&& field_type) { + vec_field_types.push_back(std::move(field_type)); + }; + (move_field_type(std::move(field_types)), ...); + return struct_type(std::move(vec_field_types), align); +} + std::ostream& operator<<(std::ostream&, const Type::Code&); std::ostream& operator<<(std::ostream&, const Type&); From fa4cb4751ebd903adda03e3d4e37e22c275832d9 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Thu, 25 May 2023 17:15:42 -0700 Subject: [PATCH 0130/1425] Constraint solver enhancements and complete storage partition implementation * Constraint solver clean-up and complete greedy heuristics * Clean up the constraint solver code * Clean up distjoiness and completeness handling * Consolidate the legion partition creation in LogicalRegionField * Make sure we pass the right parent regions for single tasks * Move restriction checks to partitions * Fix for unit tests * Remove dead code * Put the complete tiling optimization back * Make sure inline mapping the root of a LogicalRegionField tree * Fix the key partition computation for transformed stores * Complete slicing implementation * Extend storage partition with slicing * Partition inversion and conversion in store transformations * Pack the right extents for the future-backed storage * Add a utility to legate::tuple * Add a constructor taking a custom type to Scalar See merge request legate/legate.core.internal!39 --- legate_core_cpp.cmake | 4 +- src/core/data/logical_region_field.cc | 58 +++ src/core/data/logical_region_field.h | 22 +- src/core/data/logical_store.cc | 19 +- src/core/data/logical_store.h | 11 +- src/core/data/logical_store_detail.cc | 408 +++++++++++++----- src/core/data/logical_store_detail.h | 108 +++-- src/core/data/scalar.cc | 7 + src/core/data/scalar.h | 2 + src/core/data/scalar.inl | 1 - src/core/data/shape.cc | 11 +- src/core/data/shape.h | 2 + src/core/data/slice.h | 47 ++ src/core/data/transform.cc | 215 ++++++++- src/core/data/transform.h | 32 ++ src/core/partitioning/constraint.cc | 1 + src/core/partitioning/constraint_graph.cc | 151 ------- src/core/partitioning/constraint_solver.cc | 207 +++++++++ ...constraint_graph.h => constraint_solver.h} | 44 +- src/core/partitioning/partition.cc | 87 ++-- src/core/partitioning/partition.h | 78 ++-- src/core/partitioning/partitioner.cc | 84 ++-- src/core/partitioning/partitioner.h | 17 +- src/core/runtime/field_manager.cc | 1 + src/core/runtime/launcher.cc | 10 +- src/core/runtime/launcher.h | 4 +- src/core/runtime/launcher_arg.cc | 2 +- src/core/runtime/operation.cc | 61 +-- src/core/runtime/operation.h | 12 +- src/core/runtime/partition_manager.cc | 10 + src/core/runtime/partition_manager.h | 1 + src/core/runtime/req_analyzer.cc | 17 +- src/core/runtime/runtime.cc | 29 +- src/core/runtime/runtime.h | 7 +- src/core/utilities/tuple.h | 36 +- src/core/utilities/tuple.inl | 126 +++++- tests/cpp/unit/store.cc | 7 +- 37 files changed, 1420 insertions(+), 519 deletions(-) create mode 100644 src/core/data/logical_region_field.cc create mode 100644 src/core/data/slice.h delete mode 100644 src/core/partitioning/constraint_graph.cc create mode 100644 src/core/partitioning/constraint_solver.cc rename src/core/partitioning/{constraint_graph.h => constraint_solver.h} (53%) diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index ff6ae28226..6de0a7b548 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -193,6 +193,7 @@ list(APPEND legate_core_SOURCES src/core/comm/comm_cpu.cc src/core/comm/coll.cc src/core/data/allocator.cc + src/core/data/logical_region_field.cc src/core/data/logical_store.cc src/core/data/logical_store_detail.cc src/core/data/scalar.cc @@ -208,7 +209,7 @@ list(APPEND legate_core_SOURCES src/core/mapping/operation.cc src/core/mapping/store.cc src/core/partitioning/constraint.cc - src/core/partitioning/constraint_graph.cc + src/core/partitioning/constraint_solver.cc src/core/partitioning/partition.cc src/core/partitioning/partitioner.cc src/core/partitioning/restriction.cc @@ -433,6 +434,7 @@ install( src/core/data/scalar.h src/core/data/scalar.inl src/core/data/shape.h + src/core/data/slice.h src/core/data/store.h src/core/data/store.inl src/core/data/transform.h diff --git a/src/core/data/logical_region_field.cc b/src/core/data/logical_region_field.cc new file mode 100644 index 0000000000..0829a3d682 --- /dev/null +++ b/src/core/data/logical_region_field.cc @@ -0,0 +1,58 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/data/logical_region_field.h" +#include "core/partitioning/partition.h" +#include "core/runtime/runtime.h" + +namespace legate { + +LogicalRegionField::LogicalRegionField(const Legion::LogicalRegion& lr, + Legion::FieldID fid, + std::shared_ptr parent) + : lr_(lr), fid_(fid), parent_(std::move(parent)) +{ +} + +int32_t LogicalRegionField::dim() const { return lr_.get_dim(); } + +const LogicalRegionField& LogicalRegionField::get_root() const +{ + return parent_ != nullptr ? parent_->get_root() : *this; +} + +Domain LogicalRegionField::domain() const +{ + return Runtime::get_runtime()->get_index_space_domain(lr_.get_index_space()); +} + +std::shared_ptr LogicalRegionField::get_child(const Tiling* tiling, + const Shape& color, + bool complete) +{ + auto legion_partition = get_legion_partition(tiling, complete); + auto color_point = to_domain_point(color); + return std::make_shared( + Runtime::get_runtime()->get_subregion(legion_partition, color_point), fid_, shared_from_this()); +} + +Legion::LogicalPartition LogicalRegionField::get_legion_partition(const Partition* partition, + bool complete) +{ + return partition->construct(lr_, complete); +} + +} // namespace legate diff --git a/src/core/data/logical_region_field.h b/src/core/data/logical_region_field.h index ba5c0a299a..8abc0b1c60 100644 --- a/src/core/data/logical_region_field.h +++ b/src/core/data/logical_region_field.h @@ -16,16 +16,24 @@ #pragma once +#include +#include + #include "legion.h" -#include "core/runtime/runtime.h" +#include "core/data/shape.h" namespace legate { -class LogicalRegionField { +class Partition; +class Tiling; + +class LogicalRegionField : public std::enable_shared_from_this { public: LogicalRegionField() {} - LogicalRegionField(const Legion::LogicalRegion& lr, Legion::FieldID fid); + LogicalRegionField(const Legion::LogicalRegion& lr, + Legion::FieldID fid, + std::shared_ptr parent = nullptr); public: LogicalRegionField(const LogicalRegionField& other) = default; @@ -35,13 +43,21 @@ class LogicalRegionField { int32_t dim() const; const Legion::LogicalRegion& region() const { return lr_; } Legion::FieldID field_id() const { return fid_; } + const LogicalRegionField& get_root() const; public: Legion::Domain domain() const; + public: + std::shared_ptr get_child(const Tiling* tiling, + const Shape& color, + bool complete); + Legion::LogicalPartition get_legion_partition(const Partition* partition, bool complete); + private: Legion::LogicalRegion lr_{}; Legion::FieldID fid_{-1U}; + std::shared_ptr parent_{nullptr}; }; } // namespace legate diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index 07d43ba692..9c81f77fd9 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -16,7 +16,6 @@ #include -#include "core/data/logical_region_field.h" #include "core/data/logical_store.h" #include "core/data/logical_store_detail.h" #include "core/data/store.h" @@ -28,21 +27,10 @@ #include "core/utilities/dispatch.h" #include "legate_defines.h" -using namespace Legion; - namespace legate { extern Logger log_legate; -LogicalRegionField::LogicalRegionField(const LogicalRegion& lr, FieldID fid) : lr_(lr), fid_(fid) {} - -int32_t LogicalRegionField::dim() const { return lr_.get_dim(); } - -Domain LogicalRegionField::domain() const -{ - return Runtime::get_runtime()->get_index_space_domain(lr_.get_index_space()); -} - LogicalStore::LogicalStore(std::shared_ptr&& impl) : impl_(std::move(impl)) {} int32_t LogicalStore::dim() const { return impl_->dim(); } @@ -72,7 +60,7 @@ LogicalStorePartition LogicalStore::partition_by_tiling(std::vector tile return LogicalStorePartition(impl_->partition_by_tiling(Shape(std::move(tile_shape)))); } -LogicalStore LogicalStore::slice(int32_t dim, std::slice sl) const +LogicalStore LogicalStore::slice(int32_t dim, Slice sl) const { return LogicalStore(impl_->slice(dim, sl)); } @@ -92,9 +80,10 @@ std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) return impl_->get_physical_store(context); } -void LogicalStore::set_key_partition(const Partition* partition) +void LogicalStore::set_key_partition(const mapping::MachineDesc& machine, + const Partition* partition) { - impl_->set_key_partition(partition); + impl_->set_key_partition(machine, partition); } LogicalStorePartition::LogicalStorePartition(std::shared_ptr&& impl) diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index 85bc6af1ee..41daaba160 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -22,6 +22,7 @@ #include "legion.h" #include "core/data/shape.h" +#include "core/data/slice.h" #include "core/data/transform.h" #include "core/type/type_info.h" #include "core/utilities/typedefs.h" @@ -42,6 +43,12 @@ class Projection; class Runtime; class Store; +namespace mapping { + +class MachineDesc; + +} + namespace detail { class LogicalStore; @@ -230,7 +237,7 @@ class LogicalStore { * * @throw std::invalid_argument If `dim` is not a valid dimension name */ - LogicalStore slice(int32_t dim, std::slice sl) const; + LogicalStore slice(int32_t dim, Slice sl) const; /** * @brief Reorders dimensions of the store. * @@ -334,7 +341,7 @@ class LogicalStore { std::shared_ptr get_physical_store(LibraryContext* context); public: - void set_key_partition(const Partition* partition); + void set_key_partition(const mapping::MachineDesc& machine, const Partition* partition); public: std::shared_ptr impl() const { return impl_; } diff --git a/src/core/data/logical_store_detail.cc b/src/core/data/logical_store_detail.cc index 561a8f3dc9..1bbd1eac84 100644 --- a/src/core/data/logical_store_detail.cc +++ b/src/core/data/logical_store_detail.cc @@ -35,35 +35,135 @@ namespace detail { //////////////////////////////////////////////////// Storage::Storage(int32_t dim, std::unique_ptr type) - : unbound_(true), dim_(dim), type_(std::move(type)) + : storage_id_(Runtime::get_runtime()->get_unique_storage_id()), + unbound_(true), + dim_(dim), + type_(std::move(type)), + offsets_(dim_, 0) { +#ifdef DEBUG_LEGATE + log_legate.debug() << "Create " << to_string(); +#endif } -Storage::Storage(Shape extents, std::unique_ptr type, bool optimize_scalar) - : dim_(extents.size()), extents_(extents), type_(std::move(type)), volume_(extents.volume()) +Storage::Storage(const Shape& extents, std::unique_ptr type, bool optimize_scalar) + : storage_id_(Runtime::get_runtime()->get_unique_storage_id()), + dim_(extents.size()), + extents_(extents), + type_(std::move(type)), + volume_(extents.volume()), + offsets_(dim_, 0) { if (optimize_scalar && volume_ == 1) kind_ = Kind::FUTURE; +#ifdef DEBUG_LEGATE + log_legate.debug() << "Create " << to_string(); +#endif } -Storage::Storage(Shape extents, std::unique_ptr type, const Legion::Future& future) - : dim_(extents.size()), +Storage::Storage(const Shape& extents, std::unique_ptr type, const Legion::Future& future) + : storage_id_(Runtime::get_runtime()->get_unique_storage_id()), + dim_(extents.size()), extents_(extents), type_(std::move(type)), kind_(Kind::FUTURE), future_(future), - volume_(extents.volume()) + volume_(extents.volume()), + offsets_(dim_, 0) +{ +#ifdef DEBUG_LEGATE + log_legate.debug() << "Create " << to_string(); +#endif +} + +Storage::Storage(Shape&& extents, + std::unique_ptr type, + std::shared_ptr parent, + Shape&& color, + Shape&& offsets) + : storage_id_(Runtime::get_runtime()->get_unique_storage_id()), + dim_(extents.size()), + extents_(std::move(extents)), + type_(std::move(type)), + volume_(extents_.volume()), + level_(parent->level() + 1), + parent_(std::move(parent)), + color_(std::move(color)), + offsets_(std::move(offsets)) +{ +#ifdef DEBUG_LEGATE + log_legate.debug() << "Create " << to_string(); +#endif +} + +const Shape& Storage::extents() const +{ + if (unbound_) { + Runtime::get_runtime()->flush_scheduling_window(); + if (unbound_) throw std::invalid_argument("Illegal to access an uninitialized unbound store"); + } + return extents_; +} + +const Shape& Storage::offsets() const +{ +#ifdef DEBUG_LEGATE + assert(!unbound_); +#endif + return offsets_; +} + +std::shared_ptr Storage::slice(Shape tile_shape, Shape offsets) +{ + if (Kind::FUTURE == kind_) return shared_from_this(); + + auto root = get_root(); + auto shape = root->extents(); + + auto can_tile_completely = + (shape % tile_shape).sum() == 0 && (offsets % tile_shape).sum() == 0 && + Runtime::get_runtime()->partition_manager()->use_complete_tiling(shape, tile_shape); + + Shape color_shape, color; + tuple signed_offsets; + if (can_tile_completely) { + color_shape = shape / tile_shape; + color = offsets / tile_shape; + signed_offsets = tuple(0, shape.size()); + } else { + color_shape = Shape(shape.size(), 1); + color = Shape(shape.size(), 0); + signed_offsets = apply([](size_t v) { return static_cast(v); }, offsets); + } + + auto tiling = + create_tiling(std::move(tile_shape), std::move(color_shape), std::move(signed_offsets)); + auto* p_tiling = static_cast(tiling.get()); + auto storage_partition = root->create_partition(std::move(tiling), can_tile_completely); + return storage_partition->get_child_storage(color); +} + +std::shared_ptr Storage::get_root() const { + return nullptr == parent_ ? shared_from_this() : parent_->get_root(); } -int32_t Storage::dim() { return dim_; } +std::shared_ptr Storage::get_root() +{ + return nullptr == parent_ ? shared_from_this() : parent_->get_root(); +} LogicalRegionField* Storage::get_region_field() { #ifdef DEBUG_LEGATE assert(Kind::REGION_FIELD == kind_); #endif - if (nullptr == region_field_) + if (region_field_ != nullptr) return region_field_.get(); + + if (nullptr == parent_) region_field_ = Runtime::get_runtime()->create_region_field(extents_, type_->size()); + else + region_field_ = parent_->get_child_data(color_); + return region_field_.get(); } @@ -96,50 +196,57 @@ RegionField Storage::map(LibraryContext* context) #ifdef DEBUG_LEGATE assert(Kind::REGION_FIELD == kind_); #endif - return Runtime::get_runtime()->map_region_field(context, region_field_.get()); + return Runtime::get_runtime()->map_region_field(context, region_field_->get_root()); +} + +Restrictions Storage::compute_restrictions() const +{ + return Restrictions(dim_, Restriction::ALLOW); } -Partition* Storage::find_or_create_key_partition(const mapping::MachineDesc& machine, - const Restrictions& restrictions) +Partition* Storage::find_key_partition(const mapping::MachineDesc& machine, + const Restrictions& restrictions) const { uint32_t new_num_pieces = machine.count(); - if (num_pieces_ == new_num_pieces && key_partition_ != nullptr && restrictions_ == restrictions) + if (num_pieces_ == new_num_pieces && key_partition_ != nullptr && + key_partition_->satisfies_restrictions(restrictions)) return key_partition_.get(); - - auto part_mgr = Runtime::get_runtime()->partition_manager(); - auto launch_shape = part_mgr->compute_launch_shape(machine, restrictions, extents_); - num_pieces_ = new_num_pieces; - restrictions_ = restrictions; - if (launch_shape.empty()) - key_partition_ = create_no_partition(); - else { - auto tile_shape = part_mgr->compute_tile_shape(extents_, launch_shape); - key_partition_ = create_tiling(std::move(tile_shape), std::move(launch_shape)); - } - - return key_partition_.get(); + else if (parent_ != nullptr) + return parent_->find_key_partition(machine, restrictions); + else + return nullptr; } -void Storage::set_key_partition(std::unique_ptr&& key_partition) +void Storage::set_key_partition(const mapping::MachineDesc& machine, + std::unique_ptr&& key_partition) { + num_pieces_ = machine.count(); key_partition_ = std::move(key_partition); } void Storage::reset_key_partition() { key_partition_ = nullptr; } -Legion::LogicalPartition Storage::find_or_create_legion_partition(const Partition* partition) +std::shared_ptr Storage::create_partition(std::shared_ptr partition, + std::optional complete) { -#ifdef DEBUG_LEGATE - assert(Kind::REGION_FIELD == kind_); -#endif - auto region = get_region_field()->region(); - return partition->construct( - region, partition->is_disjoint_for(nullptr), partition->is_complete_for(nullptr)); + if (!complete.has_value()) complete = partition->is_complete_for(this); + return std::make_shared( + shared_from_this(), std::move(partition), complete.value()); } -std::shared_ptr Storage::create_partition(std::shared_ptr partition) +std::string Storage::to_string() const { - return std::make_shared(shared_from_this(), std::move(partition)); + std::stringstream ss; + + ss << "Storage(" << storage_id_ << ") {shape: "; + if (unbound_) + ss << "(unbound)"; + else + ss << extents_; + ss << ", dim: " << dim_ << ", kind: " << (kind_ == Kind::REGION_FIELD ? "Region" : "Future") + << ", type: " << type_->to_string() << ", level: " << level_; + + return std::move(ss).str(); } //////////////////////////////////////////////////// @@ -147,11 +254,57 @@ std::shared_ptr Storage::create_partition(std::shared_ptr parent, - std::shared_ptr partition) - : parent_(std::move(parent)), partition_(std::move(partition)) + std::shared_ptr partition, + bool complete) + : complete_(complete), + level_(parent->level() + 1), + parent_(std::move(parent)), + partition_(std::move(partition)) { } +std::shared_ptr StoragePartition::get_root() const { return parent_->get_root(); } + +std::shared_ptr StoragePartition::get_root() { return parent_->get_root(); } + +std::shared_ptr StoragePartition::get_child_storage(const Shape& color) +{ + if (partition_->kind() != Partition::Kind::TILING) + throw std::runtime_error("Sub-storage is implemented only for tiling"); + auto tiling = static_cast(partition_.get()); + auto child_extents = tiling->get_child_extents(parent_->extents(), color); + auto child_offsets = tiling->get_child_offsets(color); + return std::make_shared(std::move(child_extents), + parent_->type().clone(), + shared_from_this(), + Shape(color), + std::move(child_offsets)); +} + +std::shared_ptr StoragePartition::get_child_data(const Shape& color) +{ + if (partition_->kind() != Partition::Kind::TILING) + throw std::runtime_error("Sub-storage is implemented only for tiling"); + auto tiling = static_cast(partition_.get()); + return parent_->get_region_field()->get_child(tiling, color, complete_); +} + +Partition* StoragePartition::find_key_partition(const mapping::MachineDesc& machine, + const Restrictions& restrictions) const +{ + return parent_->find_key_partition(machine, restrictions); +} + +Legion::LogicalPartition StoragePartition::get_legion_partition() +{ + return parent_->get_region_field()->get_legion_partition(partition_.get(), complete_); +} + +bool StoragePartition::is_disjoint_for(const Domain* launch_domain) const +{ + return partition_->is_disjoint_for(launch_domain); +} + //////////////////////////////////////////////////// // legate::detail::LogicalStore //////////////////////////////////////////////////// @@ -193,11 +346,9 @@ bool LogicalStore::unbound() const { return storage_->unbound(); } const Shape& LogicalStore::extents() const { - if (extents_.empty()) { + if (unbound()) { Runtime::get_runtime()->flush_scheduling_window(); - if (extents_.empty()) { - throw std::invalid_argument("Illegal to access an uninitialized unbound store"); - } + if (unbound()) throw std::invalid_argument("Illegal to access an uninitialized unbound store"); } return extents_; } @@ -217,6 +368,8 @@ const Type& LogicalStore::type() const { return storage_->type(); } bool LogicalStore::transformed() const { return !transform_->identity(); } +const Storage* LogicalStore::get_storage() const { return storage_.get(); } + LogicalRegionField* LogicalStore::get_region_field() { return storage_->get_region_field(); } Legion::Future LogicalStore::get_future() { return storage_->get_future(); } @@ -238,7 +391,7 @@ void LogicalStore::set_future(Legion::Future future) storage_->set_future(future); } -std::shared_ptr LogicalStore::promote(int32_t extra_dim, size_t dim_size) const +std::shared_ptr LogicalStore::promote(int32_t extra_dim, size_t dim_size) { if (extra_dim < 0 || extra_dim > dim()) { throw std::invalid_argument("Invalid promotion on dimension " + std::to_string(extra_dim) + @@ -250,19 +403,26 @@ std::shared_ptr LogicalStore::promote(int32_t extra_dim, size_t di return std::make_shared(std::move(new_extents), storage_, std::move(transform)); } -std::shared_ptr LogicalStore::project(int32_t d, int64_t index) const +std::shared_ptr LogicalStore::project(int32_t d, int64_t index) { + auto old_extents = extents(); + if (d < 0 || d >= dim()) { throw std::invalid_argument("Invalid projection on dimension " + std::to_string(d) + " for a " + std::to_string(dim()) + "-D store"); - } else if (index < 0 || index >= extents()[d]) { + } else if (index < 0 || index >= old_extents[d]) { throw std::invalid_argument("Projection index " + std::to_string(index) + - " is out of bounds [0, " + std::to_string(extents()[d]) + ")"); + " is out of bounds [0, " + std::to_string(old_extents[d]) + ")"); } - auto new_extents = extents().remove(d); + auto new_extents = old_extents.remove(d); auto transform = transform_->push(std::make_unique(d, index)); - return std::make_shared(std::move(new_extents), storage_, std::move(transform)); + auto substorage = volume() == 0 + ? storage_ + : storage_->slice(transform->invert_extents(new_extents), + transform->invert_point(Shape(new_extents.size(), 0))); + return std::make_shared( + std::move(new_extents), std::move(substorage), std::move(transform)); } std::shared_ptr LogicalStore::partition_by_tiling(Shape tile_shape) @@ -272,47 +432,44 @@ std::shared_ptr LogicalStore::partition_by_tiling(Shape t std::to_string(extents().size()) + "-tuple, got a " + std::to_string(tile_shape.size()) + "-tuple"); } - Shape color_shape(extents()); - // TODO: This better use std::transform - for (size_t idx = 0; idx < tile_shape.size(); ++idx) - color_shape[idx] = (color_shape[idx] + tile_shape[idx] - 1) / tile_shape[idx]; - auto partition = create_tiling(std::move(tile_shape), std::move(color_shape)); - return create_partition(std::move(partition)); + auto color_shape = apply([](auto c, auto t) { return (c + t - 1) / t; }, extents(), tile_shape); + auto partition = create_tiling(std::move(tile_shape), std::move(color_shape)); + return create_partition(std::move(partition), true); } -std::shared_ptr LogicalStore::slice(int32_t idx, std::slice sl) const +std::shared_ptr LogicalStore::slice(int32_t idx, Slice slice) { if (idx < 0 || idx >= dim()) { throw std::invalid_argument("Invalid slicing of dimension " + std::to_string(idx) + " for a " + std::to_string(dim()) + "-D store"); } - auto start = sl.start(); - auto stop = sl.size(); - auto step = sl.stride(); - auto size = extents()[idx]; - if (start < 0) { start = start + size; } - if (stop < 0) { start = start + size; } - - if (step != 1) { - throw std::invalid_argument("Unsupported slicing step: " + std::to_string(step)); - } else if (start >= size || stop > size) { - throw std::invalid_argument("Out-of-bounds slicing on dimension " + std::to_string(idx) + - " for a store"); - } + auto sanitize_slice = [](const Slice& slice, size_t extent) { + int64_t start = slice.start.value_or(0); + int64_t stop = slice.stop.value_or(extent); - auto old_extents = extents(); - auto new_extents = Shape(); - for (int i = 0; i < idx; i++) { new_extents.append_inplace(old_extents[i]); } - new_extents.append_inplace(stop - start); - for (int i = idx + 1; i < old_extents.size(); i++) { new_extents.append_inplace(old_extents[i]); } + if (start < 0) start += extent; + if (stop < 0) stop += extent; + + return std::make_pair(std::max(0, start), std::max(0, stop)); + }; + + auto exts = extents(); + auto [start, stop] = sanitize_slice(slice, exts[idx]); + exts[idx] = stop - start; + + if (exts[idx] == extents()[idx]) return shared_from_this(); auto transform = (start == 0) ? transform_ : transform_->push(std::make_unique(idx, -start)); - return std::make_shared(std::move(new_extents), storage_, std::move(transform)); + auto substorage = volume() == 0 ? storage_ + : storage_->slice(transform->invert_extents(exts), + transform->invert_point(Shape(exts.size(), 0))); + return std::make_shared( + std::move(exts), std::move(substorage), std::move(transform)); } -std::shared_ptr LogicalStore::transpose(std::vector&& axes) const +std::shared_ptr LogicalStore::transpose(std::vector&& axes) { if (axes.size() != dim()) { throw std::invalid_argument("Dimension Mismatch: expected " + std::to_string(dim()) + @@ -336,8 +493,7 @@ std::shared_ptr LogicalStore::transpose(std::vector&& axe return std::make_shared(std::move(new_extents), storage_, std::move(transform)); } -std::shared_ptr LogicalStore::delinearize(int32_t idx, - std::vector&& sizes) const +std::shared_ptr LogicalStore::delinearize(int32_t idx, std::vector&& sizes) { if (idx < 0 || idx >= dim()) { throw std::invalid_argument("Invalid delinearization on dimension " + std::to_string(idx) + @@ -385,6 +541,11 @@ std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) return mapped_; } +Restrictions LogicalStore::compute_restrictions() const +{ + return transform_->convert(storage_->compute_restrictions()); +} + Legion::ProjectionID LogicalStore::compute_projection(int32_t launch_ndim) const { if (transform_->identity()) { @@ -403,70 +564,68 @@ Legion::ProjectionID LogicalStore::compute_projection(int32_t launch_ndim) const return Runtime::get_runtime()->get_projection(ndim, point); } -std::unique_ptr LogicalStore::create_projection(const Partition* partition, - int32_t launch_ndim) -{ - if (has_scalar_storage()) return std::make_unique(); - - // We're about to create a legion partition for this store, so the store should have its region - // created. - auto proj_id = compute_projection(launch_ndim); - auto* orig_partition = partition; - std::unique_ptr inverted = nullptr; - if (!transform_->identity()) { - inverted = transform_->invert(partition); - partition = inverted.get(); - } - -#ifdef DEBUG_LEGATE - log_legate.debug() << "Partition Store(" << store_id_ << ") {partition: " << *orig_partition - << ", inverted: " << *partition << ", projection: " << proj_id << "}"; -#endif - - auto region_field = get_region_field(); - auto region = region_field->region(); - auto legion_partition = partition->construct( - region, partition->is_disjoint_for(nullptr), partition->is_complete_for(nullptr)); - return std::make_unique(legion_partition, proj_id); -} - std::shared_ptr LogicalStore::find_or_create_key_partition( const mapping::MachineDesc& machine, const Restrictions& restrictions) { uint32_t new_num_pieces = machine.count(); - if (num_pieces_ == new_num_pieces && key_partition_ != nullptr && restrictions_ == restrictions) + if (num_pieces_ == new_num_pieces && key_partition_ != nullptr && + key_partition_->satisfies_restrictions(restrictions)) return key_partition_; if (has_scalar_storage()) { num_pieces_ = new_num_pieces; - restrictions_ = restrictions; key_partition_ = create_no_partition(); return key_partition_; } - Partition* storage_part = - storage_->find_or_create_key_partition(machine, transform_->invert(restrictions)); - auto store_part = transform_->convert(storage_part); - num_pieces_ = new_num_pieces; - restrictions_ = restrictions; - key_partition_ = std::move(store_part); + Partition* storage_part = storage_->find_key_partition(machine, transform_->invert(restrictions)); + std::unique_ptr store_part = nullptr; + if (nullptr == storage_part) { + auto part_mgr = Runtime::get_runtime()->partition_manager(); + auto launch_shape = part_mgr->compute_launch_shape(machine, restrictions, extents_); + if (launch_shape.empty()) + store_part = create_no_partition(); + else { + auto tile_shape = part_mgr->compute_tile_shape(extents_, launch_shape); + store_part = create_tiling(std::move(tile_shape), std::move(launch_shape)); + } + } else + store_part = transform_->convert(storage_part); +#ifdef DEBUG_LEGATE + assert(store_part != nullptr); +#endif + num_pieces_ = new_num_pieces; + key_partition_ = std::move(store_part); return key_partition_; } -void LogicalStore::set_key_partition(const Partition* partition) +bool LogicalStore::has_key_partition(const mapping::MachineDesc& machine, + const Restrictions& restrictions) const +{ + uint32_t new_num_pieces = machine.count(); + if (key_partition_ != nullptr && new_num_pieces == num_pieces_ && + key_partition_->satisfies_restrictions(restrictions)) + return true; + else + return storage_->find_key_partition(machine, transform_->invert(restrictions)) != nullptr; +} + +void LogicalStore::set_key_partition(const mapping::MachineDesc& machine, + const Partition* partition) { + num_pieces_ = machine.count(); auto inverted = transform_->invert(partition); - storage_->set_key_partition(std::move(inverted)); + storage_->set_key_partition(machine, std::move(inverted)); } void LogicalStore::reset_key_partition() { storage_->reset_key_partition(); } std::shared_ptr LogicalStore::create_partition( - std::shared_ptr partition) + std::shared_ptr partition, std::optional complete) { if (unbound()) { throw std::invalid_argument("Unbound store cannot be manually partitioned"); } - // TODO: the partition here should be inverted by the transform - auto storage_partition = storage_->create_partition(partition); + auto storage_partition = + storage_->create_partition(transform_->invert(partition.get()), complete); return std::make_shared(std::move(storage_partition), shared_from_this()); } @@ -503,5 +662,22 @@ LogicalStorePartition::LogicalStorePartition(std::shared_ptr s { } +std::unique_ptr LogicalStorePartition::create_projection(const Domain* launch_domain) +{ + if (nullptr == launch_domain || store_->has_scalar_storage()) + return std::make_unique(); + + // We're about to create a legion partition for this store, so the store should have its region + // created. + auto legion_partition = storage_partition_->get_legion_partition(); + auto proj_id = store_->compute_projection(launch_domain->dim); + return std::make_unique(legion_partition, proj_id); +} + +bool LogicalStorePartition::is_disjoint_for(const Domain* launch_domain) const +{ + return storage_partition_->is_disjoint_for(launch_domain); +} + } // namespace detail } // namespace legate diff --git a/src/core/data/logical_store_detail.h b/src/core/data/logical_store_detail.h index 4aa0be7eaa..5e28f50101 100644 --- a/src/core/data/logical_store_detail.h +++ b/src/core/data/logical_store_detail.h @@ -16,9 +16,10 @@ #pragma once -#include +#include #include "core/data/logical_region_field.h" +#include "core/data/slice.h" #include "core/partitioning/partition.h" #include "core/partitioning/restriction.h" #include "core/runtime/runtime.h" @@ -44,18 +45,31 @@ class Storage : public std::enable_shared_from_this { public: // Create a RegionField-backed storage whose size is unbound. Initialized lazily. Storage(int32_t dim, std::unique_ptr type); - // Create a RegionField-backed or a Future-backedstorage. Initialized lazily. - Storage(Shape extents, std::unique_ptr type, bool optimize_scalar); + // Create a RegionField-backed or a Future-backed storage. Initialized lazily. + Storage(const Shape& extents, std::unique_ptr type, bool optimize_scalar); // Create a Future-backed storage. Initialized eagerly. - Storage(Shape extents, std::unique_ptr type, const Legion::Future& future); + Storage(const Shape& extents, std::unique_ptr type, const Legion::Future& future); + // Create a RegionField-bakced sub-storage. Initialized lazily. + Storage(Shape&& extents, + std::unique_ptr type, + std::shared_ptr parent, + Shape&& color, + Shape&& offsets); public: bool unbound() const { return unbound_; } - const Shape& extents() const { return extents_; } + const Shape& extents() const; + const Shape& offsets() const; size_t volume() const { return volume_; } - int32_t dim(); + int32_t dim() { return dim_; } const Type& type() const { return *type_; } Kind kind() const { return kind_; } + int32_t level() const { return level_; } + + public: + std::shared_ptr slice(Shape tile_shape, Shape offsets); + std::shared_ptr get_root() const; + std::shared_ptr get_root(); public: LogicalRegionField* get_region_field(); @@ -67,39 +81,73 @@ class Storage : public std::enable_shared_from_this { RegionField map(LibraryContext* context); public: - Partition* find_or_create_key_partition(const mapping::MachineDesc& machine, - const Restrictions& restrictions); - void set_key_partition(std::unique_ptr&& key_partition); + Restrictions compute_restrictions() const; + Partition* find_key_partition(const mapping::MachineDesc& machine, + const Restrictions& restrictions) const; + void set_key_partition(const mapping::MachineDesc& machine, + std::unique_ptr&& key_partition); void reset_key_partition(); - Legion::LogicalPartition find_or_create_legion_partition(const Partition* partition); public: - std::shared_ptr create_partition(std::shared_ptr partition); + std::shared_ptr create_partition(std::shared_ptr partition, + std::optional complete = std::nullopt); + + public: + std::string to_string() const; private: + uint64_t storage_id_{0}; bool unbound_{false}; int32_t dim_{-1}; Shape extents_; size_t volume_; std::unique_ptr type_{nullptr}; Kind kind_{Kind::REGION_FIELD}; + + private: std::shared_ptr region_field_{nullptr}; Legion::Future future_{}; + private: + int32_t level_{0}; + std::shared_ptr parent_{nullptr}; + Shape color_{}; + // Unlike offsets in a tiling, these offsets can never be negative, as a slicing always selects a + // sub-rectangle of its parent + Shape offsets_{}; + private: uint32_t num_pieces_{0}; - Restrictions restrictions_{}; std::unique_ptr key_partition_{nullptr}; }; -class StoragePartition { +class StoragePartition : public std::enable_shared_from_this { public: - StoragePartition(std::shared_ptr parent, std::shared_ptr partition); + StoragePartition(std::shared_ptr parent, + std::shared_ptr partition, + bool complete); public: std::shared_ptr partition() const { return partition_; } + std::shared_ptr get_root() const; + std::shared_ptr get_root(); + std::shared_ptr get_child_storage(const Shape& color); + std::shared_ptr get_child_data(const Shape& color); + + public: + Partition* find_key_partition(const mapping::MachineDesc& machine, + const Restrictions& restrictions) const; + Legion::LogicalPartition get_legion_partition(); + + public: + int32_t level() const { return level_; } + + public: + bool is_disjoint_for(const Domain* launch_domain) const; private: + bool complete_; + int32_t level_; std::shared_ptr parent_; std::shared_ptr partition_; }; @@ -137,17 +185,18 @@ class LogicalStore : public std::enable_shared_from_this { bool transformed() const; public: + const Storage* get_storage() const; LogicalRegionField* get_region_field(); Legion::Future get_future(); void set_region_field(std::shared_ptr&& region_field); void set_future(Legion::Future future); public: - std::shared_ptr promote(int32_t extra_dim, size_t dim_size) const; - std::shared_ptr project(int32_t dim, int64_t index) const; - std::shared_ptr slice(int32_t dim, std::slice sl) const; - std::shared_ptr transpose(std::vector&& axes) const; - std::shared_ptr delinearize(int32_t dim, std::vector&& sizes) const; + std::shared_ptr promote(int32_t extra_dim, size_t dim_size); + std::shared_ptr project(int32_t dim, int64_t index); + std::shared_ptr slice(int32_t dim, Slice sl); + std::shared_ptr transpose(std::vector&& axes); + std::shared_ptr delinearize(int32_t dim, std::vector&& sizes); public: std::shared_ptr partition_by_tiling(Shape tile_shape); @@ -156,16 +205,18 @@ class LogicalStore : public std::enable_shared_from_this { std::shared_ptr get_physical_store(LibraryContext* context); public: + Restrictions compute_restrictions() const; std::unique_ptr create_projection(const Partition* partition, int32_t launch_ndim); std::shared_ptr find_or_create_key_partition(const mapping::MachineDesc& machine, const Restrictions& restrictions); - void set_key_partition(const Partition* partition); + bool has_key_partition(const mapping::MachineDesc& machine, + const Restrictions& restrictions) const; + void set_key_partition(const mapping::MachineDesc& machine, const Partition* partition); void reset_key_partition(); - private: - std::shared_ptr create_partition(std::shared_ptr partition); - - private: + public: + std::shared_ptr create_partition( + std::shared_ptr partition, std::optional complete = std::nullopt); Legion::ProjectionID compute_projection(int32_t launch_ndim) const; public: @@ -181,13 +232,12 @@ class LogicalStore : public std::enable_shared_from_this { std::shared_ptr transform_; private: - uint32_t num_pieces_; - Restrictions restrictions_{}; - std::shared_ptr key_partition_; + uint32_t num_pieces_{0}; + std::shared_ptr key_partition_{nullptr}; std::shared_ptr mapped_{nullptr}; }; -class LogicalStorePartition { +class LogicalStorePartition : public std::enable_shared_from_this { public: LogicalStorePartition(std::shared_ptr storage_partition, std::shared_ptr store); @@ -195,6 +245,8 @@ class LogicalStorePartition { public: std::shared_ptr storage_partition() const { return storage_partition_; } std::shared_ptr store() const { return store_; } + std::unique_ptr create_projection(const Domain* launch_domain); + bool is_disjoint_for(const Domain* launch_domain) const; private: std::shared_ptr storage_partition_; diff --git a/src/core/data/scalar.cc b/src/core/data/scalar.cc index a5b792261b..04d5bbc739 100644 --- a/src/core/data/scalar.cc +++ b/src/core/data/scalar.cc @@ -21,6 +21,13 @@ namespace legate { Scalar::Scalar(const Scalar& other) : own_(other.own_), type_(other.type_->clone()) { copy(other); } +Scalar::Scalar(Scalar&& other) : own_(other.own_), type_(std::move(other.type_)), data_(other.data_) +{ + other.own_ = false; + other.type_ = nullptr; + other.data_ = nullptr; +} + Scalar::Scalar(std::unique_ptr type, const void* data) : type_(std::move(type)), data_(data) { } diff --git a/src/core/data/scalar.h b/src/core/data/scalar.h index 0fb62fad67..b3c5d4c8c8 100644 --- a/src/core/data/scalar.h +++ b/src/core/data/scalar.h @@ -47,6 +47,8 @@ class Scalar { public: Scalar() = default; Scalar(const Scalar& other); + Scalar(Scalar&& other); + /** * @brief Creates a shared `Scalar` with an existing allocation. The caller is responsible * for passing in a sufficiently big allocation. diff --git a/src/core/data/scalar.inl b/src/core/data/scalar.inl index 88799f2dc9..8f201d5ba4 100644 --- a/src/core/data/scalar.inl +++ b/src/core/data/scalar.inl @@ -23,7 +23,6 @@ Scalar::Scalar(T value) : own_(true), type_(primitive_type(legate_type_code_of != Type::Code::STRUCT); static_assert(legate_type_code_of != Type::Code::STRING); static_assert(legate_type_code_of != Type::Code::INVALID); - auto buffer = malloc(sizeof(T)); memcpy(buffer, &value, sizeof(T)); data_ = buffer; diff --git a/src/core/data/shape.cc b/src/core/data/shape.cc index 9b72f3752b..44733859a6 100644 --- a/src/core/data/shape.cc +++ b/src/core/data/shape.cc @@ -25,9 +25,18 @@ Legion::Domain to_domain(const tuple& shape) domain.dim = ndim; for (int32_t idx = 0; idx < ndim; ++idx) { domain.rect_data[idx] = 0; - domain.rect_data[idx + ndim] = shape[idx] - 1; + domain.rect_data[idx + ndim] = static_cast(shape[idx]) - 1; } return domain; } +Legion::DomainPoint to_domain_point(const Shape& shape) +{ + Legion::DomainPoint point; + auto ndim = static_cast(shape.size()); + point.dim = ndim; + for (int32_t idx = 0; idx < ndim; ++idx) point[idx] = static_cast(shape[idx]); + return point; +} + } // namespace legate diff --git a/src/core/data/shape.h b/src/core/data/shape.h index 6bd37175e5..06e0dcd084 100644 --- a/src/core/data/shape.h +++ b/src/core/data/shape.h @@ -26,4 +26,6 @@ using Shape = tuple; Legion::Domain to_domain(const Shape& shape); +Legion::DomainPoint to_domain_point(const Shape& shape); + } // namespace legate diff --git a/src/core/data/slice.h b/src/core/data/slice.h new file mode 100644 index 0000000000..05915e0b1d --- /dev/null +++ b/src/core/data/slice.h @@ -0,0 +1,47 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include + +/** + * @file + * @brief A simple slice class that has the same semantics as Python's + */ + +namespace legate { + +/** + * @ingroup data + * @brief A slice descriptor + * + * legate::Slice behaves similarly to how the slice in Python does, and has different semantics + * from std::slice. + */ +struct Slice { + static constexpr std::optional OPEN = std::nullopt; + + Slice(std::optional _start = OPEN, std::optional _stop = OPEN) + : start(_start), stop(_stop) + { + } + + std::optional start{OPEN}; + std::optional stop{OPEN}; +}; + +} // namespace legate diff --git a/src/core/data/transform.cc b/src/core/data/transform.cc index b56e1cd069..747b464301 100644 --- a/src/core/data/transform.cc +++ b/src/core/data/transform.cc @@ -72,6 +72,18 @@ proj::SymbolicPoint TransformStack::invert(const proj::SymbolicPoint& point) con return parent_->identity() ? result : parent_->invert(result); } +Restrictions TransformStack::convert(const Restrictions& restrictions) const +{ + if (identity()) return restrictions; + + if (parent_->identity()) + return transform_->convert(restrictions); + else { + auto result = parent_->convert(restrictions); + return transform_->convert(result); + } +} + Restrictions TransformStack::invert(const Restrictions& restrictions) const { if (identity()) return restrictions; @@ -80,6 +92,22 @@ Restrictions TransformStack::invert(const Restrictions& restrictions) const return parent_->identity() ? std::move(result) : parent_->invert(result); } +Shape TransformStack::invert_extents(const Shape& extents) const +{ + if (identity()) return extents; + + auto result = transform_->invert_extents(extents); + return parent_->identity() ? std::move(result) : parent_->invert_extents(result); +} + +Shape TransformStack::invert_point(const Shape& point) const +{ + if (identity()) return point; + + auto result = transform_->invert_point(point); + return parent_->identity() ? std::move(result) : parent_->invert_point(result); +} + void TransformStack::pack(BufferBuilder& buffer) const { if (identity()) @@ -180,15 +208,57 @@ Legion::DomainAffineTransform Shift::inverse_transform(int32_t in_dim) const return result; } -std::unique_ptr Shift::convert(const Partition* partition) const { return nullptr; } +std::unique_ptr Shift::convert(const Partition* partition) const +{ + switch (partition->kind()) { + case Partition::Kind::NO_PARTITION: { + return create_no_partition(); + } + case Partition::Kind::TILING: { + auto tiling = static_cast(partition); + return create_tiling(Shape(tiling->tile_shape()), + Shape(tiling->color_shape()), + tiling->offsets().update(dim_, offset_)); + } + } + assert(false); + return nullptr; +} -std::unique_ptr Shift::invert(const Partition* partition) const { return nullptr; } +std::unique_ptr Shift::invert(const Partition* partition) const +{ + switch (partition->kind()) { + case Partition::Kind::NO_PARTITION: { + return create_no_partition(); + } + case Partition::Kind::TILING: { + auto tiling = static_cast(partition); + auto new_offset = tiling->offsets()[dim_] - offset_; + return create_tiling(Shape(tiling->tile_shape()), + Shape(tiling->color_shape()), + tiling->offsets().update(dim_, new_offset)); + } + } + assert(false); + return nullptr; +} // the shift transform makes no change on the store's dimensions proj::SymbolicPoint Shift::invert(const proj::SymbolicPoint& point) const { return point; } +Restrictions Shift::convert(const Restrictions& restrictions) const { return restrictions; } + Restrictions Shift::invert(const Restrictions& restrictions) const { return restrictions; } +Shape Shift::invert_extents(const Shape& extents) const { return extents; } + +Shape Shift::invert_point(const Shape& point) const +{ + auto result = point; + result[dim_] -= offset_; + return std::move(result); +} + void Shift::pack(BufferBuilder& buffer) const { buffer.pack(LEGATE_CORE_TRANSFORM_SHIFT); @@ -290,11 +360,20 @@ proj::SymbolicPoint Promote::invert(const proj::SymbolicPoint& point) const return point.remove(extra_dim_); } +Restrictions Promote::convert(const Restrictions& restrictions) const +{ + return restrictions.insert(extra_dim_, Restriction::AVOID); +} + Restrictions Promote::invert(const Restrictions& restrictions) const { return restrictions.remove(extra_dim_); } +Shape Promote::invert_extents(const Shape& extents) const { return extents.remove(extra_dim_); } + +Shape Promote::invert_point(const Shape& point) const { return point.remove(extra_dim_); } + void Promote::pack(BufferBuilder& buffer) const { buffer.pack(LEGATE_CORE_TRANSFORM_PROMOTE); @@ -395,11 +474,20 @@ proj::SymbolicPoint Project::invert(const proj::SymbolicPoint& point) const return point.insert(dim_, proj::SymbolicExpr()); } +Restrictions Project::convert(const Restrictions& restrictions) const +{ + return restrictions.remove(dim_); +} + Restrictions Project::invert(const Restrictions& restrictions) const { return restrictions.insert(dim_, Restriction::ALLOW); } +Shape Project::invert_extents(const Shape& extents) const { return extents.insert(dim_, 1); } + +Shape Project::invert_point(const Shape& point) const { return point.insert(dim_, coord_); } + void Project::pack(BufferBuilder& buffer) const { buffer.pack(LEGATE_CORE_TRANSFORM_PROJECT); @@ -457,9 +545,39 @@ Legion::DomainAffineTransform Transpose::inverse_transform(int32_t in_dim) const return result; } -std::unique_ptr Transpose::convert(const Partition* partition) const { return nullptr; } +std::unique_ptr Transpose::convert(const Partition* partition) const +{ + switch (partition->kind()) { + case Partition::Kind::NO_PARTITION: { + return create_no_partition(); + } + case Partition::Kind::TILING: { + auto tiling = static_cast(partition); + return create_tiling(tiling->tile_shape().map(axes_), + tiling->color_shape().map(axes_), + tiling->offsets().map(axes_)); + } + } + assert(false); + return nullptr; +} -std::unique_ptr Transpose::invert(const Partition* partition) const { return nullptr; } +std::unique_ptr Transpose::invert(const Partition* partition) const +{ + switch (partition->kind()) { + case Partition::Kind::NO_PARTITION: { + return create_no_partition(); + } + case Partition::Kind::TILING: { + auto tiling = static_cast(partition); + return create_tiling(tiling->tile_shape().map(inverse_), + tiling->color_shape().map(inverse_), + tiling->offsets().map(inverse_)); + } + } + assert(false); + return nullptr; +} proj::SymbolicPoint Transpose::invert(const proj::SymbolicPoint& point) const { @@ -472,6 +590,13 @@ proj::SymbolicPoint Transpose::invert(const proj::SymbolicPoint& point) const return proj::SymbolicPoint(std::move(exprs)); } +Restrictions Transpose::convert(const Restrictions& restrictions) const +{ + std::vector result; + for (int32_t dim : axes_) result.push_back(restrictions[dim]); + return Restrictions(std::move(result)); +} + Restrictions Transpose::invert(const Restrictions& restrictions) const { std::vector result; @@ -479,6 +604,10 @@ Restrictions Transpose::invert(const Restrictions& restrictions) const return Restrictions(std::move(result)); } +Shape Transpose::invert_extents(const Shape& extents) const { return extents.map(inverse_); } + +Shape Transpose::invert_point(const Shape& point) const { return point.map(inverse_); } + namespace { // anonymous template void print_vector(std::ostream& out, const std::vector& vec) @@ -577,10 +706,56 @@ Legion::DomainAffineTransform Delinearize::inverse_transform(int32_t in_dim) con std::unique_ptr Delinearize::convert(const Partition* partition) const { + throw NonInvertibleTransformation("Delinearize transform cannot be used in conversion"); return nullptr; } -std::unique_ptr Delinearize::invert(const Partition* partition) const { return nullptr; } +std::unique_ptr Delinearize::invert(const Partition* partition) const +{ + switch (partition->kind()) { + case Partition::Kind::NO_PARTITION: { + return create_no_partition(); + } + case Partition::Kind::TILING: { + auto tiling = static_cast(partition); + auto& tile_shape = tiling->tile_shape(); + auto& color_shape = tiling->color_shape(); + auto& offsets = tiling->offsets(); + + auto invertible = [&](const Tiling* tiling) { + size_t volume = 1; + size_t sum_offset = 0; + for (uint32_t idx = 1; idx < sizes_.size(); ++idx) { + volume *= color_shape[dim_ + idx]; + sum_offset += offsets[dim_ + idx]; + } + return 1 == volume && 0 == sum_offset; + }; + + if (!invertible(tiling)) + throw NonInvertibleTransformation("Delinearize transform cannot invert this partition: " + + tiling->to_string()); + + auto new_tile_shape = tile_shape; + auto new_color_shape = color_shape; + auto new_offsets = offsets; + + for (uint32_t idx = 1; idx < sizes_.size(); ++idx) { + new_tile_shape.remove_inplace(dim_ + 1); + new_color_shape.remove_inplace(dim_ + 1); + new_offsets.remove_inplace(dim_ + 1); + } + + new_tile_shape[dim_] *= strides_[dim_]; + new_offsets[dim_] *= strides_[dim_]; + + return create_tiling( + std::move(new_tile_shape), std::move(new_color_shape), std::move(new_offsets)); + } + } + assert(false); + return nullptr; +} proj::SymbolicPoint Delinearize::invert(const proj::SymbolicPoint& point) const { @@ -590,11 +765,35 @@ proj::SymbolicPoint Delinearize::invert(const proj::SymbolicPoint& point) const return proj::SymbolicPoint(std::move(exprs)); } +Restrictions Delinearize::convert(const Restrictions& restrictions) const +{ + std::vector result; + for (uint32_t dim = 0; dim <= dim_; ++dim) result.push_back(restrictions[dim]); + for (uint32_t idx = 1; idx < sizes_.size(); ++idx) result.push_back(Restriction::FORBID); + for (uint32_t dim = dim_ + 1; dim < restrictions.size(); ++dim) + result.push_back(restrictions[dim]); + return Restrictions(std::move(result)); +} + Restrictions Delinearize::invert(const Restrictions& restrictions) const { - auto result = restrictions; - for (uint32_t dim = 1; dim < sizes_.size(); ++dim) result.remove_inplace(dim + 1); - return result; + std::vector result; + for (uint32_t dim = 0; dim <= dim_; ++dim) result.push_back(restrictions[dim]); + for (uint32_t dim = dim_ + sizes_.size(); dim < restrictions.size(); ++dim) + result.push_back(restrictions[dim]); + return Restrictions(std::move(result)); +} + +Shape Delinearize::invert_extents(const Shape& extents) const +{ + throw NonInvertibleTransformation(); + return Shape(); +} + +Shape Delinearize::invert_point(const Shape& point) const +{ + throw NonInvertibleTransformation(); + return Shape(); } void Delinearize::pack(BufferBuilder& buffer) const diff --git a/src/core/data/transform.h b/src/core/data/transform.h index 725bda6566..3e92cdf3db 100644 --- a/src/core/data/transform.h +++ b/src/core/data/transform.h @@ -18,6 +18,7 @@ #include +#include "core/data/shape.h" #include "core/partitioning/restriction.h" #include "core/runtime/projection.h" #include "core/utilities/typedefs.h" @@ -27,13 +28,26 @@ namespace legate { class BufferBuilder; class Partition; +class NonInvertibleTransformation : public std::exception { + public: + NonInvertibleTransformation() : error_message_("Non-invertible transformation") {} + NonInvertibleTransformation(const std::string& error_message) : error_message_(error_message) {} + const char* what() const throw() { return error_message_.c_str(); } + + private: + std::string error_message_; +}; + struct Transform { virtual Domain transform(const Domain& input) const = 0; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const = 0; virtual std::unique_ptr convert(const Partition* partition) const = 0; virtual std::unique_ptr invert(const Partition* partition) const = 0; virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const = 0; + virtual Restrictions convert(const Restrictions& restrictions) const = 0; virtual Restrictions invert(const Restrictions& restrictions) const = 0; + virtual Shape invert_extents(const Shape& extents) const = 0; + virtual Shape invert_point(const Shape& point) const = 0; virtual void pack(BufferBuilder& buffer) const = 0; virtual void print(std::ostream& out) const = 0; }; @@ -57,7 +71,10 @@ struct TransformStack : public Transform, std::enable_shared_from_this convert(const Partition* partition) const override; virtual std::unique_ptr invert(const Partition* partition) const override; virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; + virtual Restrictions convert(const Restrictions& restrictions) const override; virtual Restrictions invert(const Restrictions& restrictions) const override; + virtual Shape invert_extents(const Shape& extents) const override; + virtual Shape invert_point(const Shape& point) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; @@ -84,7 +101,10 @@ class Shift : public StoreTransform { virtual std::unique_ptr convert(const Partition* partition) const override; virtual std::unique_ptr invert(const Partition* partition) const override; virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; + virtual Restrictions convert(const Restrictions& restrictions) const override; virtual Restrictions invert(const Restrictions& restrictions) const override; + virtual Shape invert_extents(const Shape& extents) const override; + virtual Shape invert_point(const Shape& point) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; @@ -106,7 +126,10 @@ class Promote : public StoreTransform { virtual std::unique_ptr convert(const Partition* partition) const override; virtual std::unique_ptr invert(const Partition* partition) const override; virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; + virtual Restrictions convert(const Restrictions& restrictions) const override; virtual Restrictions invert(const Restrictions& restrictions) const override; + virtual Shape invert_extents(const Shape& extents) const override; + virtual Shape invert_point(const Shape& point) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; @@ -129,7 +152,10 @@ class Project : public StoreTransform { virtual std::unique_ptr convert(const Partition* partition) const override; virtual std::unique_ptr invert(const Partition* partition) const override; virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; + virtual Restrictions convert(const Restrictions& restrictions) const override; virtual Restrictions invert(const Restrictions& restrictions) const override; + virtual Shape invert_extents(const Shape& extents) const override; + virtual Shape invert_point(const Shape& point) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; @@ -151,7 +177,10 @@ class Transpose : public StoreTransform { virtual std::unique_ptr convert(const Partition* partition) const override; virtual std::unique_ptr invert(const Partition* partition) const override; virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; + virtual Restrictions convert(const Restrictions& restrictions) const override; virtual Restrictions invert(const Restrictions& restrictions) const override; + virtual Shape invert_extents(const Shape& extents) const override; + virtual Shape invert_point(const Shape& point) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; @@ -173,7 +202,10 @@ class Delinearize : public StoreTransform { virtual std::unique_ptr convert(const Partition* partition) const override; virtual std::unique_ptr invert(const Partition* partition) const override; virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; + virtual Restrictions convert(const Restrictions& restrictions) const override; virtual Restrictions invert(const Restrictions& restrictions) const override; + virtual Shape invert_extents(const Shape& extents) const override; + virtual Shape invert_point(const Shape& point) const override; virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; diff --git a/src/core/partitioning/constraint.cc b/src/core/partitioning/constraint.cc index 60e9fb7bfc..6a7a295b0a 100644 --- a/src/core/partitioning/constraint.cc +++ b/src/core/partitioning/constraint.cc @@ -87,6 +87,7 @@ std::string Broadcast::to_string() const std::unique_ptr align(const Variable* lhs, const Variable* rhs) { + if (*lhs == *rhs) throw std::invalid_argument("Alignment needs two distinct variables"); // Since an Alignment object owns child nodes, inputs need to be copied return std::make_unique(std::make_unique(*lhs), std::make_unique(*rhs)); diff --git a/src/core/partitioning/constraint_graph.cc b/src/core/partitioning/constraint_graph.cc deleted file mode 100644 index 1ab8a13c35..0000000000 --- a/src/core/partitioning/constraint_graph.cc +++ /dev/null @@ -1,151 +0,0 @@ -/* Copyright 2021 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#include - -#include "legion.h" - -#include "core/data/logical_store_detail.h" -#include "core/partitioning/constraint.h" -#include "core/partitioning/constraint_graph.h" -#include "core/runtime/operation.h" - -namespace legate { - -extern Legion::Logger log_legate; - -void ConstraintGraph::add_partition_symbol(const Variable* partition_symbol) -{ - partition_symbols_.insert(partition_symbol); -} - -void ConstraintGraph::add_constraint(const Constraint* constraint) -{ - constraints_.push_back(constraint); -} - -void ConstraintGraph::compute_equivalence_classes() -{ - for (auto& part_symb : partition_symbols()) { - // TODO: partition symbols can be independent of any stores of the operation - // (e.g., when a symbol subsumes a union of two other symbols) - int32_t ndim = part_symb->operation()->find_store(part_symb)->dim(); - all_equiv_class_entries_.emplace_back(new EquivClass(part_symb, ndim)); - equiv_classes_.insert({*part_symb, all_equiv_class_entries_.back().get()}); - } - - auto update_table = [&](auto* new_cls, auto* orig_cls) { - while (orig_cls != nullptr) { - equiv_classes_[*orig_cls->partition_symbol] = new_cls; - orig_cls = orig_cls->next; - } - }; - - auto handle_alignment = [&](const Alignment* alignment) { -#ifdef DEBUG_LEGATE - assert(alignment->lhs()->as_variable() != nullptr && - alignment->rhs()->as_variable() != nullptr); -#endif - - std::vector part_symbs_to_unify; - alignment->find_partition_symbols(part_symbs_to_unify); -#ifdef DEBUG_LEGATE - assert(!part_symbs_to_unify.empty()); -#endif - - EquivClass* equiv_class = equiv_classes_[*part_symbs_to_unify[0]]; -#ifdef DEBUG_LEGATE - assert(equiv_class != nullptr); -#endif - for (size_t idx = 1; idx < part_symbs_to_unify.size(); ++idx) { - auto class_to_unify = equiv_classes_[*part_symbs_to_unify[idx]]; - auto result = equiv_class->unify(class_to_unify); - - if (result != equiv_class) update_table(result, equiv_class); - if (result != class_to_unify) update_table(result, class_to_unify); - equiv_class = result; - } - }; - - auto handle_broadcast = [&](const Broadcast* broadcast) { - auto* variable = broadcast->variable(); - auto& axes = broadcast->axes(); - EquivClass* equiv_class = equiv_classes_.at(*variable); - for (uint32_t idx = 0; idx < axes.size(); ++idx) { - uint32_t axis = axes[idx]; - // TODO: We want to check the axis eagerly and raise an exception - // if it is out of bounds - if (axis >= equiv_class->restrictions.size()) continue; - equiv_class->restrictions[axes[idx]] = Restriction::FORBID; - } - }; - - for (auto& constraint : constraints_) switch (constraint->kind()) { - case Constraint::Kind::ALIGNMENT: { - handle_alignment(constraint->as_alignment()); - break; - } - case Constraint::Kind::BROADCAST: { - handle_broadcast(constraint->as_broadcast()); - break; - } - } -} - -std::vector ConstraintGraph::find_equivalence_class( - const Variable* partition_symbol) const -{ - auto equiv_class = equiv_classes_.at(*partition_symbol); - std::vector result; - result.reserve(equiv_class->size); - while (equiv_class != nullptr) { - result.push_back(equiv_class->partition_symbol); - equiv_class = equiv_class->next; - } - return result; -} - -Restrictions ConstraintGraph::find_restrictions(const Variable* partition_symbol) const -{ - auto equiv_class = equiv_classes_.at(*partition_symbol); - Restrictions result = equiv_class->restrictions; - equiv_class = equiv_class->next; - while (equiv_class != nullptr) { - join_inplace(result, equiv_class->restrictions); - equiv_class = equiv_class->next; - } - return result; -} - -void ConstraintGraph::dump() -{ - log_legate.debug("===== Constraint Graph ====="); - log_legate.debug() << "Variables:"; - for (auto& symbol : partition_symbols_.elements()) - log_legate.debug() << " " << symbol->to_string(); - log_legate.debug() << "Constraints:"; - for (auto& constraint : constraints_) log_legate.debug() << " " << constraint->to_string(); - log_legate.debug("============================"); -} - -const std::vector& ConstraintGraph::partition_symbols() const -{ - return partition_symbols_.elements(); -} - -const std::vector& ConstraintGraph::constraints() const { return constraints_; } - -} // namespace legate diff --git a/src/core/partitioning/constraint_solver.cc b/src/core/partitioning/constraint_solver.cc new file mode 100644 index 0000000000..86f5fef8a7 --- /dev/null +++ b/src/core/partitioning/constraint_solver.cc @@ -0,0 +1,207 @@ +/* Copyright 2021 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "legion.h" + +#include "core/data/logical_store_detail.h" +#include "core/partitioning/constraint.h" +#include "core/partitioning/constraint_solver.h" +#include "core/runtime/operation.h" + +namespace legate { + +extern Legion::Logger log_legate; + +namespace { + +struct UnionFindEntry { + UnionFindEntry(const Variable* symb, const detail::LogicalStore* store) + : partition_symbol(symb), restrictions(store->compute_restrictions()), next(nullptr), size(1) + { + } + + UnionFindEntry* unify(UnionFindEntry* other) + { + UnionFindEntry* self = this; + if (self->size < other->size) std::swap(self, other); + + auto end = self; + while (end->next != nullptr) end = end->next; + end->next = other; + end->size += other->size; + return self; + } + + const Variable* partition_symbol; + Restrictions restrictions; + UnionFindEntry* next; + size_t size; +}; + +} // namespace + +struct ConstraintSolver::EquivClass { + EquivClass(const UnionFindEntry* entry) + { + partition_symbols.reserve(entry->size); + partition_symbols.push_back(entry->partition_symbol); + restrictions = entry->restrictions; + + auto* next = entry->next; + while (next != nullptr) { + partition_symbols.push_back(next->partition_symbol); + join_inplace(restrictions, next->restrictions); + next = next->next; + } + } + + std::vector partition_symbols; + Restrictions restrictions; +}; + +ConstraintSolver::ConstraintSolver() {} + +ConstraintSolver::~ConstraintSolver() {} + +void ConstraintSolver::add_partition_symbol(const Variable* partition_symbol) +{ + partition_symbols_.insert(partition_symbol); +} + +void ConstraintSolver::add_constraint(const Constraint* constraint) +{ + constraints_.push_back(constraint); +} + +void ConstraintSolver::solve_constraints() +{ + std::vector entries; + std::map table; + + // Initialize the table by creating singleton equivalence classes + const auto& all_symbols = partition_symbols(); + entries.reserve(all_symbols.size()); + + auto initialize = [&entries, &table](const auto& all_symbols) { + for (auto& symb : all_symbols) { + // TODO: partition symbols can be independent of any stores of the operation + // (e.g., when a symbol subsumes a union of two other symbols) + auto* store = symb->operation()->find_store(symb); + entries.emplace_back(symb, store); + table.insert({*symb, &entries.back()}); + } + }; + initialize(all_symbols); + + // Unify equivalence classes based on alignment constraints + auto handle_alignment = [&table](const Alignment* alignment) { + auto update_table = [&table](UnionFindEntry* old_cls, UnionFindEntry* new_cls) { + while (old_cls != nullptr) { + table[*old_cls->partition_symbol] = new_cls; + old_cls = old_cls->next; + } + }; + + std::vector part_symbs_to_unify; + alignment->find_partition_symbols(part_symbs_to_unify); +#ifdef DEBUG_LEGATE + assert(!part_symbs_to_unify.empty()); +#endif + + auto it = part_symbs_to_unify.begin(); + auto* equiv_class = table[**it++]; +#ifdef DEBUG_LEGATE + assert(equiv_class != nullptr); +#endif + for (; it != part_symbs_to_unify.end(); ++it) { + auto* to_unify = table[**it]; + auto* result = equiv_class->unify(to_unify); + + if (result != equiv_class) update_table(equiv_class, result); + if (result != to_unify) update_table(to_unify, result); + } + }; + + // Set the restrictions according to broadcasting constraints + auto handle_broadcast = [&table](const Broadcast* broadcast) { + auto* variable = broadcast->variable(); + auto& axes = broadcast->axes(); + auto* equiv_class = table.at(*variable); + for (uint32_t idx = 0; idx < axes.size(); ++idx) { + uint32_t axis = axes[idx]; + // TODO: We want to check the axis eagerly and raise an exception + // if it is out of bounds + if (axis >= equiv_class->restrictions.size()) continue; + equiv_class->restrictions[axes[idx]] = Restriction::FORBID; + } + }; + + // Reflect each constraint to the solver state + for (auto& constraint : constraints_) switch (constraint->kind()) { + case Constraint::Kind::ALIGNMENT: { + handle_alignment(constraint->as_alignment()); + break; + } + case Constraint::Kind::BROADCAST: { + handle_broadcast(constraint->as_broadcast()); + break; + } + } + + // Combine states of each union of equivalence classes into one + std::unordered_set distinct_entries; + for (auto& [_, entry] : table) distinct_entries.insert(entry); + + for (auto* entry : distinct_entries) { + auto equiv_class = std::make_unique(entry); + for (auto* symb : equiv_class->partition_symbols) + equiv_class_map_.insert({*symb, equiv_class.get()}); + equiv_classes_.push_back(std::move(equiv_class)); + } +} + +const std::vector& ConstraintSolver::find_equivalence_class( + const Variable* partition_symbol) const +{ + return equiv_class_map_.at(*partition_symbol)->partition_symbols; +} + +const Restrictions& ConstraintSolver::find_restrictions(const Variable* partition_symbol) const +{ + return equiv_class_map_.at(*partition_symbol)->restrictions; +} + +void ConstraintSolver::dump() +{ + log_legate.debug("===== Constraint Graph ====="); + log_legate.debug() << "Variables:"; + for (auto& symbol : partition_symbols_.elements()) + log_legate.debug() << " " << symbol->to_string(); + log_legate.debug() << "Constraints:"; + for (auto& constraint : constraints_) log_legate.debug() << " " << constraint->to_string(); + log_legate.debug("============================"); +} + +const std::vector& ConstraintSolver::partition_symbols() const +{ + return partition_symbols_.elements(); +} + +const std::vector& ConstraintSolver::constraints() const { return constraints_; } + +} // namespace legate diff --git a/src/core/partitioning/constraint_graph.h b/src/core/partitioning/constraint_solver.h similarity index 53% rename from src/core/partitioning/constraint_graph.h rename to src/core/partitioning/constraint_solver.h index 5bc737812a..08064ce57f 100644 --- a/src/core/partitioning/constraint_graph.h +++ b/src/core/partitioning/constraint_solver.h @@ -26,31 +26,11 @@ namespace legate { -struct EquivClass { - EquivClass(const Variable* symb, int32_t ndim) - : partition_symbol(symb), next(nullptr), size(1), restrictions(ndim, Restriction::ALLOW) - { - } - - EquivClass* unify(EquivClass* other) - { - EquivClass* self = this; - if (self->size < other->size) std::swap(self, other); - - auto end = self; - while (end->next != nullptr) end = end->next; - end->next = other; - end->size += other->size; - return self; - } - - const Variable* partition_symbol; - EquivClass* next; - size_t size; - Restrictions restrictions; -}; +struct ConstraintSolver { + public: + ConstraintSolver(); + ~ConstraintSolver(); -struct ConstraintGraph { public: void add_partition_symbol(const Variable* partition_symbol); void add_constraint(const Constraint* constraint); @@ -63,17 +43,19 @@ struct ConstraintGraph { const std::vector& constraints() const; public: - void compute_equivalence_classes(); - std::vector find_equivalence_class(const Variable* partition_symbol) const; - Restrictions find_restrictions(const Variable* partition_symbol) const; + void solve_constraints(); + const std::vector& find_equivalence_class( + const Variable* partition_symbol) const; + const Restrictions& find_restrictions(const Variable* partition_symbol) const; private: - ordered_set partition_symbols_; - std::vector constraints_; + ordered_set partition_symbols_{}; + std::vector constraints_{}; private: - std::map equiv_classes_; - std::vector> all_equiv_class_entries_; + class EquivClass; + std::map equiv_class_map_{}; + std::vector> equiv_classes_{}; }; } // namespace legate diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index 54304a23d7..3b18ef1d9f 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -27,21 +27,18 @@ namespace legate { NoPartition::NoPartition() : Partition() {} -bool NoPartition::is_complete_for(const detail::LogicalStore* store) const { return false; } +bool NoPartition::is_complete_for(const detail::Storage* storage) const { return true; } -bool NoPartition::is_disjoint_for(const detail::LogicalStore* store) const { return false; } - -Legion::LogicalPartition NoPartition::construct(Legion::LogicalRegion region, - bool disjoint, - bool complete) const +bool NoPartition::is_disjoint_for(const Domain* launch_domain) const { - return Legion::LogicalPartition::NO_PART; + return nullptr == launch_domain || launch_domain->get_volume() == 1; } -std::unique_ptr NoPartition::get_projection(detail::LogicalStore* store, - int32_t launch_ndim) const +bool NoPartition::satisfies_restrictions(const Restrictions& restrictions) const { return true; } + +Legion::LogicalPartition NoPartition::construct(Legion::LogicalRegion region, bool complete) const { - return std::make_unique(); + return Legion::LogicalPartition::NO_PART; } bool NoPartition::has_launch_domain() const { return false; } @@ -56,13 +53,13 @@ std::unique_ptr NoPartition::clone() const { return create_no_partiti std::string NoPartition::to_string() const { return "NoPartition"; } -Tiling::Tiling(Shape&& tile_shape, Shape&& color_shape, Shape&& offsets) +Tiling::Tiling(Shape&& tile_shape, Shape&& color_shape, tuple&& offsets) : Partition(), tile_shape_(std::move(tile_shape)), color_shape_(std::move(color_shape)), offsets_(std::move(offsets)) { - if (offsets_.empty()) offsets_ = Shape(tile_shape_.size(), 0); + if (offsets_.empty()) offsets_ = tuple(tile_shape_.size(), 0); assert(tile_shape_.size() == color_shape_.size()); assert(tile_shape_.size() == offsets_.size()); } @@ -89,13 +86,42 @@ bool Tiling::operator<(const Tiling& other) const return false; } -bool Tiling::is_complete_for(const detail::LogicalStore* store) const { return false; } +bool Tiling::is_complete_for(const detail::Storage* storage) const +{ + const auto& storage_exts = storage->extents(); + const auto& storage_offs = storage->offsets(); + +#ifdef DEBUG_LEGATE + assert(storage_exts.size() == storage_offs.size()); + assert(storage_offs.size() == offsets_.size()); +#endif + + uint32_t ndim = storage_exts.size(); + + for (uint32_t dim = 0; dim < ndim; ++dim) { + int64_t my_lo = offsets_[dim]; + int64_t my_hi = my_lo + static_cast(tile_shape_[dim] * color_shape_[dim]); + if (static_cast(storage_offs[dim]) < my_lo && + my_hi < static_cast(storage_offs[dim] + storage_exts[dim])) + return false; + } + return true; +} -bool Tiling::is_disjoint_for(const detail::LogicalStore* store) const { return true; } +bool Tiling::is_disjoint_for(const Domain* launch_domain) const +{ + return nullptr == launch_domain || launch_domain->get_volume() <= color_shape_.volume(); +} -Legion::LogicalPartition Tiling::construct(Legion::LogicalRegion region, - bool disjoint, - bool complete) const +bool Tiling::satisfies_restrictions(const Restrictions& restrictions) const +{ + static auto satisfies_restriction = [](Restriction r, size_t ext) { + return r != Restriction::FORBID || ext == 1; + }; + return apply(satisfies_restriction, restrictions, color_shape_).all(); +} + +Legion::LogicalPartition Tiling::construct(Legion::LogicalRegion region, bool complete) const { auto index_space = region.get_index_space(); auto runtime = Runtime::get_runtime(); @@ -121,8 +147,7 @@ Legion::LogicalPartition Tiling::construct(Legion::LogicalRegion region, auto color_domain = to_domain(color_shape_); auto color_space = runtime->find_or_create_index_space(color_domain); - auto kind = complete ? (disjoint ? LEGION_DISJOINT_COMPLETE_KIND : LEGION_ALIASED_COMPLETE_KIND) - : (disjoint ? LEGION_DISJOINT_KIND : LEGION_ALIASED_KIND); + auto kind = complete ? LEGION_DISJOINT_COMPLETE_KIND : LEGION_DISJOINT_KIND; index_partition = runtime->create_restricted_partition(index_space, color_space, kind, transform, extent); @@ -130,12 +155,6 @@ Legion::LogicalPartition Tiling::construct(Legion::LogicalRegion region, return runtime->create_logical_partition(region, index_partition); } -std::unique_ptr Tiling::get_projection(detail::LogicalStore* store, - int32_t launch_ndim) const -{ - return store->create_projection(this, launch_ndim); -} - bool Tiling::has_launch_domain() const { return true; } Legion::Domain Tiling::launch_domain() const { return to_domain(color_shape_); } @@ -150,11 +169,27 @@ std::string Tiling::to_string() const return std::move(ss).str(); } +Shape Tiling::get_child_extents(const Shape& extents, const Shape& color) +{ + auto lo = apply(std::plus{}, tile_shape_ * color, offsets_); + auto hi = apply(std::plus{}, tile_shape_ * (color + 1), offsets_); + lo = apply([](int64_t v) { return std::max(0, v); }, lo); + hi = apply([](size_t a, int64_t b) { return std::min(a, b); }, extents, hi); + return apply([](int64_t h, int64_t l) { return static_cast(h - l); }, hi, lo); +} + +Shape Tiling::get_child_offsets(const Shape& color) +{ + return apply([](size_t a, int64_t b) { return static_cast(static_cast(a) + b); }, + tile_shape_ * color, + offsets_); +} + std::unique_ptr create_no_partition() { return std::make_unique(); } std::unique_ptr create_tiling(Shape&& tile_shape, Shape&& color_shape, - Shape&& offsets /*= {}*/) + tuple&& offsets /*= {}*/) { return std::make_unique( std::move(tile_shape), std::move(color_shape), std::move(offsets)); diff --git a/src/core/partitioning/partition.h b/src/core/partitioning/partition.h index 859f0f89d6..0a0640b433 100644 --- a/src/core/partitioning/partition.h +++ b/src/core/partitioning/partition.h @@ -18,9 +18,12 @@ #include -#include "core/data/shape.h" #include "legion.h" +#include "core/data/shape.h" +#include "core/partitioning/restriction.h" +#include "core/utilities/typedefs.h" + namespace legate { class Projection; @@ -28,6 +31,7 @@ class Projection; namespace detail { class LogicalStore; +class Storage; } // namespace detail @@ -46,15 +50,13 @@ struct Partition { virtual Kind kind() const = 0; public: - virtual bool is_complete_for(const detail::LogicalStore* store) const = 0; - virtual bool is_disjoint_for(const detail::LogicalStore* store) const = 0; + virtual bool is_complete_for(const detail::Storage* storage) const = 0; + virtual bool is_disjoint_for(const Domain* launch_domain) const = 0; + virtual bool satisfies_restrictions(const Restrictions& restrictions) const = 0; public: virtual Legion::LogicalPartition construct(Legion::LogicalRegion region, - bool disjoint = false, - bool complete = false) const = 0; - virtual std::unique_ptr get_projection(detail::LogicalStore* store, - int32_t launch_ndim) const = 0; + bool complete = false) const = 0; public: virtual bool has_launch_domain() const = 0; @@ -67,9 +69,9 @@ struct Partition { virtual std::string to_string() const = 0; public: - virtual const Shape& tile_shape() const = 0; - virtual const Shape& color_shape() const = 0; - virtual const Shape& offsets() const = 0; + virtual const Shape& tile_shape() const = 0; + virtual const Shape& color_shape() const = 0; + virtual const tuple& offsets() const = 0; }; class NoPartition : public Partition { @@ -77,28 +79,25 @@ class NoPartition : public Partition { NoPartition(); public: - virtual Kind kind() const override { return Kind::NO_PARTITION; } + Kind kind() const override { return Kind::NO_PARTITION; } public: - virtual bool is_complete_for(const detail::LogicalStore* store) const override; - virtual bool is_disjoint_for(const detail::LogicalStore* store) const override; + bool is_complete_for(const detail::Storage* storage) const override; + bool is_disjoint_for(const Domain* launch_domain) const override; + bool satisfies_restrictions(const Restrictions& restrictions) const override; public: - virtual Legion::LogicalPartition construct(Legion::LogicalRegion region, - bool disjoint, - bool complete) const override; - virtual std::unique_ptr get_projection(detail::LogicalStore* store, - int32_t launch_ndim) const override; + Legion::LogicalPartition construct(Legion::LogicalRegion region, bool complete) const override; public: - virtual bool has_launch_domain() const override; - virtual Legion::Domain launch_domain() const override; + bool has_launch_domain() const override; + Legion::Domain launch_domain() const override; public: - virtual std::unique_ptr clone() const override; + std::unique_ptr clone() const override; public: - virtual std::string to_string() const override; + std::string to_string() const override; public: const Shape& tile_shape() const override @@ -109,7 +108,7 @@ class NoPartition : public Partition { { throw std::invalid_argument("Partition kind doesn't support color_shape"); } - const Shape& offsets() const override + const tuple& offsets() const override { throw std::invalid_argument("Partition kind doesn't support offsets"); } @@ -117,7 +116,7 @@ class NoPartition : public Partition { class Tiling : public Partition { public: - Tiling(Shape&& tile_shape, Shape&& color_shape, Shape&& offsets); + Tiling(Shape&& tile_shape, Shape&& color_shape, tuple&& offsets); public: Tiling(const Tiling&) = default; @@ -127,45 +126,46 @@ class Tiling : public Partition { bool operator<(const Tiling& other) const; public: - virtual Kind kind() const override { return Kind::TILING; } + Kind kind() const override { return Kind::TILING; } public: - virtual bool is_complete_for(const detail::LogicalStore* store) const override; - virtual bool is_disjoint_for(const detail::LogicalStore* store) const override; + bool is_complete_for(const detail::Storage* storage) const override; + bool is_disjoint_for(const Domain* launch_domain) const override; + bool satisfies_restrictions(const Restrictions& restrictions) const override; public: - virtual Legion::LogicalPartition construct(Legion::LogicalRegion region, - bool disjoint, - bool complete) const override; - virtual std::unique_ptr get_projection(detail::LogicalStore* store, - int32_t launch_ndim) const override; + Legion::LogicalPartition construct(Legion::LogicalRegion region, bool complete) const override; public: - virtual bool has_launch_domain() const override; - virtual Legion::Domain launch_domain() const override; + bool has_launch_domain() const override; + Legion::Domain launch_domain() const override; public: - virtual std::unique_ptr clone() const override; + std::unique_ptr clone() const override; public: - virtual std::string to_string() const override; + std::string to_string() const override; public: const Shape& tile_shape() const override { return tile_shape_; } const Shape& color_shape() const override { return color_shape_; } - const Shape& offsets() const override { return offsets_; } + const tuple& offsets() const override { return offsets_; } + + public: + Shape get_child_extents(const Shape& extents, const Shape& color); + Shape get_child_offsets(const Shape& color); private: Shape tile_shape_; Shape color_shape_; - Shape offsets_; + tuple offsets_; }; std::unique_ptr create_no_partition(); std::unique_ptr create_tiling(Shape&& tile_shape, Shape&& color_shape, - Shape&& offsets = {}); + tuple&& offsets = {}); std::ostream& operator<<(std::ostream& out, const Partition& partition); diff --git a/src/core/partitioning/partitioner.cc b/src/core/partitioning/partitioner.cc index c83f7367af..0b7a526bcc 100644 --- a/src/core/partitioning/partitioner.cc +++ b/src/core/partitioning/partitioner.cc @@ -19,7 +19,7 @@ #include "core/data/logical_store_detail.h" #include "core/data/scalar.h" #include "core/partitioning/constraint.h" -#include "core/partitioning/constraint_graph.h" +#include "core/partitioning/constraint_solver.h" #include "core/partitioning/partition.h" #include "core/runtime/launcher.h" #include "core/runtime/operation.h" @@ -160,6 +160,22 @@ const Legion::FieldSpace& Strategy::find_field_space(const Variable* partition_s return finder->second; } +void Strategy::dump() const +{ + log_legate.debug("===== Solution ====="); + for (const auto& [symbol, part] : assignments_) + log_legate.debug() << symbol.to_string() << ": " << part->to_string(); + for (const auto& [symbol, fspace] : field_spaces_) + log_legate.debug() << symbol.to_string() << ": " << fspace; + for (const auto& [op, domain] : launch_domains_) { + if (nullptr == domain) + log_legate.debug() << op->to_string() << ": (sequential)"; + else + log_legate.debug() << op->to_string() << ": " << *domain; + } + log_legate.debug("===================="); +} + void Strategy::compute_launch_domains() { std::map domain_resolvers; @@ -191,60 +207,76 @@ Partitioner::Partitioner(std::vector&& operations) : operations_(std { } -std::unique_ptr Partitioner::solve() +std::unique_ptr Partitioner::partition_stores() { - ConstraintGraph constraints; + ConstraintSolver solver; - for (auto op : operations_) op->add_to_constraint_graph(constraints); + for (auto op : operations_) op->add_to_solver(solver); - constraints.compute_equivalence_classes(); + solver.solve_constraints(); #ifdef DEBUG_LEGATE - constraints.dump(); + solver.dump(); #endif auto strategy = std::make_unique(); // Copy the list of partition symbols as we will sort them inplace - std::vector partition_symbols(constraints.partition_symbols()); + auto partition_symbols = solver.partition_symbols(); - solve_for_unbound_stores(partition_symbols, strategy.get(), constraints); + auto remaining_symbols = handle_unbound_stores(strategy.get(), partition_symbols, solver); - std::stable_sort(partition_symbols.begin(), - partition_symbols.end(), - [](const auto& part_symb_a, const auto& part_symb_b) { - auto get_storage_size = [](const auto& part_symb) { - auto* op = part_symb->operation(); - auto* store = op->find_store(part_symb); + auto comparison_key = [&solver](const auto& part_symb) { + auto* op = part_symb->operation(); + auto* store = op->find_store(part_symb); + auto has_key_part = + store->has_key_partition(op->machine(), solver.find_restrictions(part_symb)); #ifdef DEBUG_LEGATE - assert(!store->unbound()); + assert(!store->unbound()); #endif - return store->storage_size(); - }; - return get_storage_size(part_symb_a) > get_storage_size(part_symb_b); + return std::make_pair(store->storage_size(), has_key_part); + }; + + for (auto& part_symb : remaining_symbols) { + auto [size, has_key_part] = comparison_key(part_symb); + log_legate.debug() << part_symb->to_string() << " " << size << " " << has_key_part; + } + + std::stable_sort(remaining_symbols.begin(), + remaining_symbols.end(), + [&solver, &comparison_key](const auto& part_symb_a, const auto& part_symb_b) { + return comparison_key(part_symb_a) > comparison_key(part_symb_b); }); - for (auto& part_symb : partition_symbols) { + for (auto& part_symb : remaining_symbols) { if (strategy->has_assignment(part_symb)) continue; - auto equiv_class = constraints.find_equivalence_class(part_symb); - auto restrictions = constraints.find_restrictions(part_symb); + const auto& equiv_class = solver.find_equivalence_class(part_symb); + const auto& restrictions = solver.find_restrictions(part_symb); auto* op = part_symb->operation(); auto store = op->find_store(part_symb); auto partition = store->find_or_create_key_partition(op->machine(), restrictions); +#ifdef DEBUG_LEGATE + assert(partition != nullptr); +#endif for (auto symb : equiv_class) strategy->insert(symb, partition); } strategy->compute_launch_domains(); +#ifdef DEBUG_LEGATE + strategy->dump(); +#endif + return strategy; } -void Partitioner::solve_for_unbound_stores(std::vector& partition_symbols, - Strategy* strategy, - const ConstraintGraph& constraints) +std::vector Partitioner::handle_unbound_stores( + Strategy* strategy, + const std::vector& partition_symbols, + const ConstraintSolver& solver) { auto runtime = Runtime::get_runtime(); @@ -260,14 +292,14 @@ void Partitioner::solve_for_unbound_stores(std::vector& partiti continue; } - auto equiv_class = constraints.find_equivalence_class(part_symb); + auto equiv_class = solver.find_equivalence_class(part_symb); std::shared_ptr partition(create_no_partition()); auto field_space = runtime->create_field_space(); for (auto symb : equiv_class) strategy->insert(symb, partition, field_space); } - partition_symbols.swap(filtered); + return filtered; } } // namespace legate diff --git a/src/core/partitioning/partitioner.h b/src/core/partitioning/partitioner.h index 31f6313cfa..30042f60bf 100644 --- a/src/core/partitioning/partitioner.h +++ b/src/core/partitioning/partitioner.h @@ -21,11 +21,12 @@ #include "core/data/shape.h" #include "core/partitioning/constraint.h" +#include "core/partitioning/restriction.h" #include "legion.h" namespace legate { -class ConstraintGraph; +class ConstraintSolver; class LogicalStore; class Operation; class Partition; @@ -52,6 +53,9 @@ class Strategy { const std::shared_ptr& operator[](const Variable* partition_symbol) const; const Legion::FieldSpace& find_field_space(const Variable* partition_symbol) const; + public: + void dump() const; + private: void compute_launch_domains(); @@ -66,12 +70,15 @@ class Partitioner { Partitioner(std::vector&& operations); public: - std::unique_ptr solve(); + std::unique_ptr partition_stores(); private: - void solve_for_unbound_stores(std::vector& partition_symbols, - Strategy* strategy, - const ConstraintGraph& constraints); + // Populates solutions for unbound stores in the `strategy` and returns remaining partition + // symbols + std::vector handle_unbound_stores( + Strategy* strategy, + const std::vector& partition_symbols, + const ConstraintSolver& constraints); private: std::vector operations_; diff --git a/src/core/runtime/field_manager.cc b/src/core/runtime/field_manager.cc index 9df4d64b8d..a0de6b1f97 100644 --- a/src/core/runtime/field_manager.cc +++ b/src/core/runtime/field_manager.cc @@ -18,6 +18,7 @@ #include "core/data/logical_region_field.h" #include "core/runtime/region_manager.h" +#include "core/runtime/runtime.h" namespace legate { diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/launcher.cc index 1a4d4aa10c..bd97ff6ed8 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -82,12 +82,14 @@ void TaskLauncher::add_output(detail::LogicalStore* store, void TaskLauncher::add_reduction(detail::LogicalStore* store, std::unique_ptr proj, + bool read_write, Legion::MappingTagID tag, - Legion::RegionFlags flags, - bool read_write /*= false*/) + Legion::RegionFlags flags) { - assert(!read_write); - add_store(reductions_, store, std::move(proj), REDUCE, tag, flags); + if (read_write) + add_store(reductions_, store, std::move(proj), READ_WRITE, tag, flags); + else + add_store(reductions_, store, std::move(proj), REDUCE, tag, flags); } void TaskLauncher::add_unbound_output(detail::LogicalStore* store, diff --git a/src/core/runtime/launcher.h b/src/core/runtime/launcher.h index 16d47da941..2164b9cbd0 100644 --- a/src/core/runtime/launcher.h +++ b/src/core/runtime/launcher.h @@ -69,9 +69,9 @@ class TaskLauncher { Legion::RegionFlags flags = LEGION_NO_FLAG); void add_reduction(detail::LogicalStore* store, std::unique_ptr proj, + bool read_write = false, Legion::MappingTagID tag = 0, - Legion::RegionFlags flags = LEGION_NO_FLAG, - bool read_write = false); + Legion::RegionFlags flags = LEGION_NO_FLAG); void add_unbound_output(detail::LogicalStore* store, Legion::FieldSpace field_space, Legion::FieldID field_id); diff --git a/src/core/runtime/launcher_arg.cc b/src/core/runtime/launcher_arg.cc index c0e10fcc5d..66754dfa2a 100644 --- a/src/core/runtime/launcher_arg.cc +++ b/src/core/runtime/launcher_arg.cc @@ -83,7 +83,7 @@ void FutureStoreArg::pack(BufferBuilder& buffer) const buffer.pack(read_only_); buffer.pack(has_storage_); buffer.pack(store_->type().size()); - buffer.pack(store_->extents().data()); + buffer.pack(store_->get_storage()->extents().data()); } } // namespace legate diff --git a/src/core/runtime/operation.cc b/src/core/runtime/operation.cc index 99060b09c5..fc6552a821 100644 --- a/src/core/runtime/operation.cc +++ b/src/core/runtime/operation.cc @@ -22,7 +22,7 @@ #include "core/data/logical_store_detail.h" #include "core/data/scalar.h" #include "core/partitioning/constraint.h" -#include "core/partitioning/constraint_graph.h" +#include "core/partitioning/constraint_solver.h" #include "core/partitioning/partition.h" #include "core/partitioning/partitioner.h" #include "core/runtime/context.h" @@ -77,45 +77,50 @@ void Task::launch(Strategy* p_strategy) { auto& strategy = *p_strategy; TaskLauncher launcher(library_, machine_, provenance_, task_id_); - auto launch_domain = strategy.launch_domain(this); - auto launch_ndim = launch_domain != nullptr ? launch_domain->dim : 0; + const auto* launch_domain = strategy.launch_domain(this); - for (auto& pair : inputs_) { - auto& store = pair.first; - auto& var = pair.second; - auto proj = strategy[var]->get_projection(store, launch_ndim); + // Add input stores + for (auto& [store, var] : inputs_) { + const auto& part = strategy[var]; + auto store_partition = store->create_partition(part); + auto proj = store_partition->create_projection(launch_domain); launcher.add_input(store, std::move(proj)); } - for (auto& pair : outputs_) { - auto& store = pair.first; + + // Add normal output stores + for (auto& [store, var] : outputs_) { if (store->unbound()) continue; - auto& var = pair.second; - auto part = strategy[var]; - auto proj = part->get_projection(store, launch_ndim); + const auto& part = strategy[var]; + auto store_partition = store->create_partition(part); + auto proj = store_partition->create_projection(launch_domain); launcher.add_output(store, std::move(proj)); - store->set_key_partition(part.get()); + store->set_key_partition(machine(), part.get()); } + + // Add reduction stores uint32_t idx = 0; - for (auto& pair : reductions_) { - auto& store = pair.first; - auto& var = pair.second; - auto proj = strategy[var]->get_projection(store, launch_ndim); - auto redop = reduction_ops_[idx++]; + for (auto& [store, var] : reductions_) { + const auto& part = strategy[var]; + auto store_partition = store->create_partition(part); + auto proj = store_partition->create_projection(launch_domain); + bool read_write = store_partition->is_disjoint_for(launch_domain); + auto redop = reduction_ops_[idx++]; proj->set_reduction_op(redop); - launcher.add_reduction(store, std::move(proj)); + launcher.add_reduction(store, std::move(proj), read_write); } + // Add unbound output stores auto* runtime = Runtime::get_runtime(); - for (auto& pair : outputs_) { - auto& store = pair.first; + for (auto& [store, var] : outputs_) { if (!store->unbound()) continue; - auto& var = pair.second; auto field_space = strategy.find_field_space(var); // TODO: We should reuse field ids here auto field_size = store->type().size(); auto field_id = runtime->allocate_field(field_space, field_size); launcher.add_unbound_output(store, field_space, field_id); } + + // Add by-value scalars for (auto& scalar : scalars_) launcher.add_scalar(scalar); if (launch_domain != nullptr) { @@ -240,12 +245,12 @@ void AutoTask::add_constraint(std::unique_ptr constraint) constraints_.push_back(std::move(constraint)); } -void AutoTask::add_to_constraint_graph(ConstraintGraph& constraint_graph) const +void AutoTask::add_to_solver(ConstraintSolver& solver) const { - for (auto& constraint : constraints_) constraint_graph.add_constraint(constraint.get()); - for (auto& [_, symb] : inputs_) constraint_graph.add_partition_symbol(symb); - for (auto& [_, symb] : outputs_) constraint_graph.add_partition_symbol(symb); - for (auto& [_, symb] : reductions_) constraint_graph.add_partition_symbol(symb); + for (auto& constraint : constraints_) solver.add_constraint(constraint.get()); + for (auto& [_, symb] : outputs_) solver.add_partition_symbol(symb); + for (auto& [_, symb] : reductions_) solver.add_partition_symbol(symb); + for (auto& [_, symb] : inputs_) solver.add_partition_symbol(symb); } //////////////////////////////////////////////////// @@ -316,6 +321,6 @@ void ManualTask::add_store(std::vector& store_args, void ManualTask::launch(Strategy*) { Task::launch(strategy_.get()); } -void ManualTask::add_to_constraint_graph(ConstraintGraph& constraint_graph) const {} +void ManualTask::add_to_solver(ConstraintSolver& constraint_graph) const {} } // namespace legate diff --git a/src/core/runtime/operation.h b/src/core/runtime/operation.h index 4ac428b91b..28416e5202 100644 --- a/src/core/runtime/operation.h +++ b/src/core/runtime/operation.h @@ -35,7 +35,7 @@ namespace legate { class Constraint; -class ConstraintGraph; +class ConstraintSolver; class LibraryContext; class Runtime; class Scalar; @@ -60,9 +60,9 @@ class Operation { virtual ~Operation() {} public: - virtual void add_to_constraint_graph(ConstraintGraph& constraint_graph) const = 0; - virtual void launch(Strategy* strategy) = 0; - virtual std::string to_string() const = 0; + virtual void add_to_solver(ConstraintSolver& constraint_graph) const = 0; + virtual void launch(Strategy* strategy) = 0; + virtual std::string to_string() const = 0; public: /** @@ -209,7 +209,7 @@ class AutoTask : public Task { * @param constraint A partitioning constraint */ void add_constraint(std::unique_ptr constraint); - void add_to_constraint_graph(ConstraintGraph& constraint_graph) const override; + void add_to_solver(ConstraintSolver& constraint_graph) const override; private: std::vector> constraints_{}; @@ -288,7 +288,7 @@ class ManualTask : public Task { void launch(Strategy* strategy) override; public: - void add_to_constraint_graph(ConstraintGraph& constraint_graph) const override; + void add_to_solver(ConstraintSolver& constraint_graph) const override; private: std::unique_ptr strategy_; diff --git a/src/core/runtime/partition_manager.cc b/src/core/runtime/partition_manager.cc index 3b69a128ae..bcc02105c3 100644 --- a/src/core/runtime/partition_manager.cc +++ b/src/core/runtime/partition_manager.cc @@ -197,6 +197,16 @@ Shape PartitionManager::compute_tile_shape(const Shape& extents, const Shape& la return tile_shape; } +bool PartitionManager::use_complete_tiling(const Shape& extents, const Shape& tile_shape) const +{ + // If it would generate a very large number of elements then + // we'll apply a heuristic for now and not actually tile it + // TODO: A better heuristic for this in the future + auto num_tiles = (extents / tile_shape).volume(); + auto num_pieces = Runtime::get_runtime()->get_machine().count(); + return !(num_tiles > 256 && num_tiles > 16 * num_pieces); +} + Legion::IndexPartition PartitionManager::find_index_partition(const Legion::IndexSpace& index_space, const Tiling& tiling) const { diff --git a/src/core/runtime/partition_manager.h b/src/core/runtime/partition_manager.h index 61763ff6f1..d02ad94e6a 100644 --- a/src/core/runtime/partition_manager.h +++ b/src/core/runtime/partition_manager.h @@ -47,6 +47,7 @@ class PartitionManager { const Restrictions& restrictions, const Shape& shape); Shape compute_tile_shape(const Shape& extents, const Shape& launch_shape); + bool use_complete_tiling(const Shape& extents, const Shape& tile_shape) const; public: Legion::IndexPartition find_index_partition(const Legion::IndexSpace& index_space, diff --git a/src/core/runtime/req_analyzer.cc b/src/core/runtime/req_analyzer.cc index 2dfc85d158..224e66b014 100644 --- a/src/core/runtime/req_analyzer.cc +++ b/src/core/runtime/req_analyzer.cc @@ -16,6 +16,7 @@ #include "core/runtime/req_analyzer.h" #include "core/runtime/launcher.h" +#include "core/runtime/runtime.h" namespace legate { @@ -74,13 +75,15 @@ void ProjectionInfo::populate_launcher(Legion::TaskLauncher* task, { Legion::RegionRequirement legion_req; + auto parent = Runtime::get_runtime()->find_parent_region(region); + if (LEGION_REDUCE == privilege) { #ifdef DEBUG_LEGATE assert(redop != -1); #endif - new (&legion_req) Legion::RegionRequirement(region, redop, LEGION_EXCLUSIVE, region, tag); + new (&legion_req) Legion::RegionRequirement(region, redop, LEGION_EXCLUSIVE, parent, tag); } else { - new (&legion_req) Legion::RegionRequirement(region, privilege, LEGION_EXCLUSIVE, region, tag); + new (&legion_req) Legion::RegionRequirement(region, privilege, LEGION_EXCLUSIVE, parent, tag); } legion_req.add_fields(fields).add_flags(flags); @@ -94,15 +97,17 @@ void ProjectionInfo::populate_launcher(Legion::IndexTaskLauncher* task, { Legion::RegionRequirement legion_req; + auto parent = Runtime::get_runtime()->find_parent_region(region); + // Broadcast if (Legion::LogicalPartition::NO_PART == partition) { if (LEGION_REDUCE == privilege) { #ifdef DEBUG_LEGATE assert(redop != -1); #endif - new (&legion_req) Legion::RegionRequirement(region, redop, LEGION_EXCLUSIVE, region, tag); + new (&legion_req) Legion::RegionRequirement(region, redop, LEGION_EXCLUSIVE, parent, tag); } else { - new (&legion_req) Legion::RegionRequirement(region, privilege, LEGION_EXCLUSIVE, region, tag); + new (&legion_req) Legion::RegionRequirement(region, privilege, LEGION_EXCLUSIVE, parent, tag); } } else { if (LEGION_REDUCE == privilege) { @@ -110,10 +115,10 @@ void ProjectionInfo::populate_launcher(Legion::IndexTaskLauncher* task, assert(redop != -1); #endif new (&legion_req) - Legion::RegionRequirement(partition, proj_id, redop, LEGION_EXCLUSIVE, region, tag); + Legion::RegionRequirement(partition, proj_id, redop, LEGION_EXCLUSIVE, parent, tag); } else { new (&legion_req) - Legion::RegionRequirement(partition, proj_id, privilege, LEGION_EXCLUSIVE, region, tag); + Legion::RegionRequirement(partition, proj_id, privilege, LEGION_EXCLUSIVE, parent, tag); } } diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index adc5d0be46..51c04bb8d5 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -470,7 +470,7 @@ void Runtime::schedule(std::vector> operations) for (auto& op : operations) op_pointers.push_back(op.get()); Partitioner partitioner(std::move(op_pointers)); - auto strategy = partitioner.solve(); + auto strategy = partitioner.partition_stores(); for (auto& op : operations) op->launch(strategy.get()); } @@ -499,6 +499,8 @@ LogicalStore Runtime::create_store(const Scalar& scalar) uint64_t Runtime::get_unique_store_id() { return next_store_id_++; } +uint64_t Runtime::get_unique_storage_id() { return next_storage_id_++; } + std::shared_ptr Runtime::create_region_field(const Shape& extents, uint32_t field_size) { @@ -524,10 +526,10 @@ std::shared_ptr Runtime::import_region_field(Legion::Logical return fld_mgr->import_field(region, field_id); } -RegionField Runtime::map_region_field(LibraryContext* context, const LogicalRegionField* rf) +RegionField Runtime::map_region_field(LibraryContext* context, const LogicalRegionField& rf) { - auto region = rf->region(); - auto field_id = rf->field_id(); + auto region = rf.region(); + auto field_id = rf.field_id(); Legion::PhysicalRegion pr; @@ -544,7 +546,7 @@ RegionField Runtime::map_region_field(LibraryContext* context, const LogicalRegi inline_mapped_.insert({key, pr}); } else pr = finder->second; - return RegionField(rf->dim(), pr, field_id); + return RegionField(rf.dim(), pr, field_id); } void Runtime::unmap_physical_region(Legion::PhysicalRegion pr) @@ -625,6 +627,23 @@ Legion::LogicalPartition Runtime::create_logical_partition( return legion_runtime_->get_logical_partition(legion_context_, logical_region, index_partition); } +Legion::LogicalRegion Runtime::get_subregion(const Legion::LogicalPartition& partition, + const Legion::DomainPoint& color) +{ + assert(nullptr != legion_context_); + return legion_runtime_->get_logical_subregion_by_color(legion_context_, partition, color); +} + +Legion::LogicalRegion Runtime::find_parent_region(const Legion::LogicalRegion& region) +{ + auto result = region; + while (legion_runtime_->has_parent_logical_partition(legion_context_, result)) { + auto partition = legion_runtime_->get_parent_logical_partition(legion_context_, result); + result = legion_runtime_->get_parent_logical_region(legion_context_, partition); + } + return result; +} + Legion::Future Runtime::create_future(const void* data, size_t datalen) const { return Legion::Future::from_untyped_pointer(data, datalen); diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 321dcbafa0..cac333a174 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -235,6 +235,7 @@ class Runtime { */ LogicalStore create_store(const Scalar& scalar); uint64_t get_unique_store_id(); + uint64_t get_unique_storage_id(); public: std::shared_ptr create_region_field(const Shape& extents, @@ -242,7 +243,7 @@ class Runtime { std::shared_ptr import_region_field(Legion::LogicalRegion region, Legion::FieldID field_id, uint32_t field_size); - RegionField map_region_field(LibraryContext* context, const LogicalRegionField* region_field); + RegionField map_region_field(LibraryContext* context, const LogicalRegionField& region_field); void unmap_physical_region(Legion::PhysicalRegion pr); public: @@ -264,6 +265,9 @@ class Runtime { const Legion::FieldSpace& field_space); Legion::LogicalPartition create_logical_partition(const Legion::LogicalRegion& logical_region, const Legion::IndexPartition& index_partition); + Legion::LogicalRegion get_subregion(const Legion::LogicalPartition& partition, + const Legion::DomainPoint& color); + Legion::LogicalRegion find_parent_region(const Legion::LogicalRegion& region); Legion::Future create_future(const void* data, size_t datalen) const; Legion::FieldID allocate_field(const Legion::FieldSpace& field_space, size_t field_size); Legion::FieldID allocate_field(const Legion::FieldSpace& field_space, @@ -359,6 +363,7 @@ class Runtime { using RegionFieldID = std::pair; std::map inline_mapped_; uint64_t next_store_id_{1}; + uint64_t next_storage_id_{1}; private: bool in_callback_{false}; diff --git a/src/core/utilities/tuple.h b/src/core/utilities/tuple.h index c54dc39aee..3014b3e12b 100644 --- a/src/core/utilities/tuple.h +++ b/src/core/utilities/tuple.h @@ -37,18 +37,28 @@ class tuple { tuple(size_t size, T init); public: - tuple(const tuple&) = default; - tuple(tuple&&) = default; - tuple& operator=(const tuple&) = default; - tuple& operator=(tuple&&) = default; + tuple(const tuple&) = default; + tuple(tuple&&) = default; + tuple& operator=(const tuple&) = default; + tuple& operator=(tuple&&) = default; public: const T& operator[](uint32_t idx) const; T& operator[](uint32_t idx); public: - bool operator==(const tuple& other) const; - bool operator<(const tuple& other) const; + bool operator==(const tuple& other) const; + bool operator<(const tuple& other) const; + tuple operator+(const tuple& other) const; + tuple operator+(const T& other) const; + tuple operator-(const tuple& other) const; + tuple operator-(const T& other) const; + tuple operator*(const tuple& other) const; + tuple operator*(const T& other) const; + tuple operator%(const tuple& other) const; + tuple operator%(const T& other) const; + tuple operator/(const tuple& other) const; + tuple operator/(const T& other) const; public: bool empty() const; @@ -58,6 +68,7 @@ class tuple { tuple insert(int32_t pos, const T& value) const; tuple append(const T& value) const; tuple remove(int32_t pos) const; + tuple update(int32_t pos, const T& value) const; void insert_inplace(int32_t pos, const T& value); void append_inplace(const T& value); @@ -66,11 +77,15 @@ class tuple { public: template T reduce(FUNC func, const T& init) const; + T sum() const; T volume() const; + bool all() const; template bool all(PRED pred) const; + bool any() const; template bool any(PRED pred) const; + tuple map(const std::vector& mapping) const; public: std::string to_string() const; @@ -90,6 +105,15 @@ tuple from_range(T stop); template tuple from_range(T start, T stop); +template +auto apply(FUNC func, const tuple& rhs); + +template +auto apply(FUNC func, const tuple& rhs1, const tuple& rhs2); + +template +auto apply(FUNC func, const tuple& rhs1, const T2& rhs2); + } // namespace legate #include "core/utilities/tuple.inl" diff --git a/src/core/utilities/tuple.inl b/src/core/utilities/tuple.inl index 13b3272844..326cc5e99d 100644 --- a/src/core/utilities/tuple.inl +++ b/src/core/utilities/tuple.inl @@ -58,17 +58,77 @@ T& tuple::operator[](uint32_t idx) } template -bool tuple::operator==(const tuple& other) const +bool tuple::operator==(const tuple& other) const { return data_ == other.data_; } template -bool tuple::operator<(const tuple& other) const +bool tuple::operator<(const tuple& other) const { return data_ < other.data_; } +template +tuple tuple::operator+(const tuple& other) const +{ + return apply(std::plus{}, *this, other); +} + +template +tuple tuple::operator+(const T& other) const +{ + return apply(std::plus{}, *this, other); +} + +template +tuple tuple::operator-(const tuple& other) const +{ + return apply(std::minus{}, *this, other); +} + +template +tuple tuple::operator-(const T& other) const +{ + return apply(std::minus{}, *this, other); +} + +template +tuple tuple::operator*(const tuple& other) const +{ + return apply(std::multiplies{}, *this, other); +} + +template +tuple tuple::operator*(const T& other) const +{ + return apply(std::multiplies{}, *this, other); +} + +template +tuple tuple::operator%(const tuple& other) const +{ + return apply(std::modulus{}, *this, other); +} + +template +tuple tuple::operator%(const T& other) const +{ + return apply(std::modulus{}, *this, other); +} + +template +tuple tuple::operator/(const tuple& other) const +{ + return apply(std::divides{}, *this, other); +} + +template +tuple tuple::operator/(const T& other) const +{ + return apply(std::divides{}, *this, other); +} + template bool tuple::empty() const { @@ -112,6 +172,14 @@ tuple tuple::remove(int32_t pos) const return tuple(std::move(new_values)); } +template +tuple tuple::update(int32_t pos, const T& value) const +{ + std::vector new_values(data_); + new_values[pos] = value; + return tuple(std::move(new_values)); +} + template void tuple::insert_inplace(int32_t pos, const T& value) { @@ -143,12 +211,24 @@ T tuple::reduce(FUNC func, const T& init) const return agg; } +template +T tuple::sum() const +{ + return reduce(std::plus{}, T(0)); +} + template T tuple::volume() const { return reduce(std::multiplies{}, T{1}); } +template +bool tuple::all() const +{ + return all([](auto v) { return static_cast(v); }); +} + template template bool tuple::all(PRED pred) const @@ -156,6 +236,12 @@ bool tuple::all(PRED pred) const return std::all_of(data_.begin(), data_.end(), pred); } +template +bool tuple::any() const +{ + return any([](auto v) { return static_cast(v); }); +} + template template bool tuple::any(PRED pred) const @@ -163,6 +249,14 @@ bool tuple::any(PRED pred) const return std::any_of(data_.begin(), data_.end(), pred); } +template +tuple tuple::map(const std::vector& mapping) const +{ + std::vector new_values; + for (auto idx : mapping) new_values.push_back(data_[idx]); + return tuple(std::move(new_values)); +} + template std::string tuple::to_string() const { @@ -200,4 +294,32 @@ tuple from_range(T start, T stop) return tuple(std::move(values)); } +template +auto apply(FUNC func, const tuple& rhs) +{ + using VAL = typename std::invoke_result::type; + std::vector result; + for (auto& v : rhs.data()) result.push_back(func(v)); + return tuple(std::move(result)); +} + +template +auto apply(FUNC func, const tuple& rhs1, const tuple& rhs2) +{ + if (rhs1.size() != rhs2.size()) throw std::invalid_argument("Operands should have the same size"); + using VAL = typename std::invoke_result::type; + std::vector result; + for (uint32_t idx = 0; idx < rhs1.size(); ++idx) result.push_back(func(rhs1[idx], rhs2[idx])); + return tuple(std::move(result)); +} + +template +auto apply(FUNC func, const tuple& rhs1, const T2& rhs2) +{ + using VAL = typename std::invoke_result::type; + std::vector result; + for (uint32_t idx = 0; idx < rhs1.size(); ++idx) result.push_back(func(rhs1[idx], rhs2)); + return tuple(std::move(result)); +} + } // namespace legate diff --git a/tests/cpp/unit/store.cc b/tests/cpp/unit/store.cc index 1b60e45584..b57cfa09a0 100644 --- a/tests/cpp/unit/store.cc +++ b/tests/cpp/unit/store.cc @@ -18,7 +18,6 @@ #include #include #include -#include #include "legate.h" @@ -66,7 +65,7 @@ TEST(Store, Transform) })); EXPECT_TRUE(projected.transformed()); - auto sliced = store.slice(1, std::slice(1, 3, 1)); + auto sliced = store.slice(1, legate::Slice(1, 3)); EXPECT_EQ(sliced.extents(), (std::vector{4, 2})); EXPECT_TRUE(sliced.transformed()); @@ -93,9 +92,7 @@ TEST(Store, InvalidTransform) EXPECT_THROW(store.project(-3, 1), std::invalid_argument); EXPECT_THROW(store.project(0, 4), std::invalid_argument); - EXPECT_THROW(store.slice(2, std::slice(1, 3, 1)), std::invalid_argument); - EXPECT_THROW(store.slice(1, std::slice(1, 3, 2)), std::invalid_argument); - EXPECT_THROW(store.slice(1, std::slice(1, 4, 1)), std::invalid_argument); + EXPECT_THROW(store.slice(2, legate::Slice(1, 3)), std::invalid_argument); EXPECT_THROW(store.transpose({ 2, From 288ff56690cf940a157cd6f4d0a43bfc5b02926c Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Thu, 25 May 2023 22:19:27 -0700 Subject: [PATCH 0131/1425] Integration test for alignment constraints * Don't try to convert key partitions when the transformation is not convertible * Minor fix for the return type for MachineDesc * Fix broken integration tests * Integration test for alignment constraints * Increase parallelism for test build * Start testing with multiple CPUs and in test mode See merge request legate/legate.core.internal!40 --- src/core/data/logical_store_detail.cc | 8 +- src/core/data/transform.cc | 11 +- src/core/data/transform.h | 118 +++++----- src/core/mapping/machine.cc | 4 +- src/core/mapping/machine.h | 4 +- tests/cpp/build.sh | 2 +- .../cpp/integration/alignment_constraints.cc | 218 ++++++++++++++++++ .../cpp/integration/broadcast_constraints.cc | 42 +++- tests/cpp/integration/multi_scalar_out.cc | 4 +- tests/cpp/run.sh | 2 +- 10 files changed, 333 insertions(+), 80 deletions(-) create mode 100644 tests/cpp/integration/alignment_constraints.cc diff --git a/src/core/data/logical_store_detail.cc b/src/core/data/logical_store_detail.cc index 1bbd1eac84..193367a514 100644 --- a/src/core/data/logical_store_detail.cc +++ b/src/core/data/logical_store_detail.cc @@ -578,7 +578,10 @@ std::shared_ptr LogicalStore::find_or_create_key_partition( return key_partition_; } - Partition* storage_part = storage_->find_key_partition(machine, transform_->invert(restrictions)); + Partition* storage_part = nullptr; + if (transform_->is_convertible()) + storage_part = storage_->find_key_partition(machine, transform_->invert(restrictions)); + std::unique_ptr store_part = nullptr; if (nullptr == storage_part) { auto part_mgr = Runtime::get_runtime()->partition_manager(); @@ -607,7 +610,8 @@ bool LogicalStore::has_key_partition(const mapping::MachineDesc& machine, key_partition_->satisfies_restrictions(restrictions)) return true; else - return storage_->find_key_partition(machine, transform_->invert(restrictions)) != nullptr; + return transform_->is_convertible() & + storage_->find_key_partition(machine, transform_->invert(restrictions)) != nullptr; } void LogicalStore::set_key_partition(const mapping::MachineDesc& machine, diff --git a/src/core/data/transform.cc b/src/core/data/transform.cc index 747b464301..1c75149b52 100644 --- a/src/core/data/transform.cc +++ b/src/core/data/transform.cc @@ -34,13 +34,17 @@ Legion::DomainAffineTransform combine(const Legion::DomainAffineTransform& lhs, TransformStack::TransformStack(std::unique_ptr&& transform, const std::shared_ptr& parent) - : transform_(std::move(transform)), parent_(parent) + : transform_(std::move(transform)), + parent_(parent), + convertible_(transform_->is_convertible() && parent_->is_convertible()) { } TransformStack::TransformStack(std::unique_ptr&& transform, std::shared_ptr&& parent) - : transform_(std::move(transform)), parent_(std::move(parent)) + : transform_(std::move(transform)), + parent_(std::move(parent)), + convertible_(transform_->is_convertible() && parent_->is_convertible()) { } @@ -582,8 +586,9 @@ std::unique_ptr Transpose::invert(const Partition* partition) const proj::SymbolicPoint Transpose::invert(const proj::SymbolicPoint& point) const { std::vector exprs; + exprs.resize(point.size()); std::transform( - point.data().begin(), point.data().end(), exprs.end(), [&](const proj::SymbolicExpr& expr) { + point.data().begin(), point.data().end(), exprs.begin(), [&](const proj::SymbolicExpr& expr) { auto dim = inverse_[expr.dim()]; return proj::SymbolicExpr(dim, expr.weight(), expr.offset()); }); diff --git a/src/core/data/transform.h b/src/core/data/transform.h index 3e92cdf3db..1b0b074e58 100644 --- a/src/core/data/transform.h +++ b/src/core/data/transform.h @@ -48,6 +48,7 @@ struct Transform { virtual Restrictions invert(const Restrictions& restrictions) const = 0; virtual Shape invert_extents(const Shape& extents) const = 0; virtual Shape invert_point(const Shape& point) const = 0; + virtual bool is_convertible() const = 0; virtual void pack(BufferBuilder& buffer) const = 0; virtual void print(std::ostream& out) const = 0; }; @@ -66,17 +67,18 @@ struct TransformStack : public Transform, std::enable_shared_from_this&& parent); public: - virtual Domain transform(const Domain& input) const override; - virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; - virtual std::unique_ptr convert(const Partition* partition) const override; - virtual std::unique_ptr invert(const Partition* partition) const override; - virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; - virtual Restrictions convert(const Restrictions& restrictions) const override; - virtual Restrictions invert(const Restrictions& restrictions) const override; - virtual Shape invert_extents(const Shape& extents) const override; - virtual Shape invert_point(const Shape& point) const override; - virtual void pack(BufferBuilder& buffer) const override; - virtual void print(std::ostream& out) const override; + Domain transform(const Domain& input) const override; + Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; + std::unique_ptr convert(const Partition* partition) const override; + std::unique_ptr invert(const Partition* partition) const override; + proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; + Restrictions convert(const Restrictions& restrictions) const override; + Restrictions invert(const Restrictions& restrictions) const override; + Shape invert_extents(const Shape& extents) const override; + Shape invert_point(const Shape& point) const override; + bool is_convertible() const override { return convertible_; } + void pack(BufferBuilder& buffer) const override; + void print(std::ostream& out) const override; public: std::unique_ptr pop(); @@ -89,6 +91,7 @@ struct TransformStack : public Transform, std::enable_shared_from_this transform_{nullptr}; std::shared_ptr parent_{nullptr}; + bool convertible_{true}; }; class Shift : public StoreTransform { @@ -96,17 +99,18 @@ class Shift : public StoreTransform { Shift(int32_t dim, int64_t offset); public: - virtual Domain transform(const Domain& input) const override; - virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; - virtual std::unique_ptr convert(const Partition* partition) const override; - virtual std::unique_ptr invert(const Partition* partition) const override; - virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; - virtual Restrictions convert(const Restrictions& restrictions) const override; - virtual Restrictions invert(const Restrictions& restrictions) const override; - virtual Shape invert_extents(const Shape& extents) const override; - virtual Shape invert_point(const Shape& point) const override; - virtual void pack(BufferBuilder& buffer) const override; - virtual void print(std::ostream& out) const override; + Domain transform(const Domain& input) const override; + Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; + std::unique_ptr convert(const Partition* partition) const override; + std::unique_ptr invert(const Partition* partition) const override; + proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; + Restrictions convert(const Restrictions& restrictions) const override; + Restrictions invert(const Restrictions& restrictions) const override; + Shape invert_extents(const Shape& extents) const override; + Shape invert_point(const Shape& point) const override; + bool is_convertible() const override { return true; } + void pack(BufferBuilder& buffer) const override; + void print(std::ostream& out) const override; public: virtual int32_t target_ndim(int32_t source_ndim) const override; @@ -121,17 +125,18 @@ class Promote : public StoreTransform { Promote(int32_t extra_dim, int64_t dim_size); public: - virtual Domain transform(const Domain& input) const override; - virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; - virtual std::unique_ptr convert(const Partition* partition) const override; - virtual std::unique_ptr invert(const Partition* partition) const override; - virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; - virtual Restrictions convert(const Restrictions& restrictions) const override; - virtual Restrictions invert(const Restrictions& restrictions) const override; - virtual Shape invert_extents(const Shape& extents) const override; - virtual Shape invert_point(const Shape& point) const override; - virtual void pack(BufferBuilder& buffer) const override; - virtual void print(std::ostream& out) const override; + Domain transform(const Domain& input) const override; + Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; + std::unique_ptr convert(const Partition* partition) const override; + std::unique_ptr invert(const Partition* partition) const override; + proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; + Restrictions convert(const Restrictions& restrictions) const override; + Restrictions invert(const Restrictions& restrictions) const override; + Shape invert_extents(const Shape& extents) const override; + Shape invert_point(const Shape& point) const override; + bool is_convertible() const override { return true; } + void pack(BufferBuilder& buffer) const override; + void print(std::ostream& out) const override; public: virtual int32_t target_ndim(int32_t source_ndim) const override; @@ -147,17 +152,18 @@ class Project : public StoreTransform { virtual ~Project() {} public: - virtual Domain transform(const Domain& domain) const override; - virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; - virtual std::unique_ptr convert(const Partition* partition) const override; - virtual std::unique_ptr invert(const Partition* partition) const override; - virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; - virtual Restrictions convert(const Restrictions& restrictions) const override; - virtual Restrictions invert(const Restrictions& restrictions) const override; - virtual Shape invert_extents(const Shape& extents) const override; - virtual Shape invert_point(const Shape& point) const override; - virtual void pack(BufferBuilder& buffer) const override; - virtual void print(std::ostream& out) const override; + Domain transform(const Domain& domain) const override; + Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; + std::unique_ptr convert(const Partition* partition) const override; + std::unique_ptr invert(const Partition* partition) const override; + proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; + Restrictions convert(const Restrictions& restrictions) const override; + Restrictions invert(const Restrictions& restrictions) const override; + Shape invert_extents(const Shape& extents) const override; + Shape invert_point(const Shape& point) const override; + bool is_convertible() const override { return true; } + void pack(BufferBuilder& buffer) const override; + void print(std::ostream& out) const override; public: virtual int32_t target_ndim(int32_t source_ndim) const override; @@ -181,6 +187,7 @@ class Transpose : public StoreTransform { virtual Restrictions invert(const Restrictions& restrictions) const override; virtual Shape invert_extents(const Shape& extents) const override; virtual Shape invert_point(const Shape& point) const override; + virtual bool is_convertible() const override { return true; } virtual void pack(BufferBuilder& buffer) const override; virtual void print(std::ostream& out) const override; @@ -197,17 +204,18 @@ class Delinearize : public StoreTransform { Delinearize(int32_t dim, std::vector&& sizes); public: - virtual Domain transform(const Domain& domain) const override; - virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; - virtual std::unique_ptr convert(const Partition* partition) const override; - virtual std::unique_ptr invert(const Partition* partition) const override; - virtual proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; - virtual Restrictions convert(const Restrictions& restrictions) const override; - virtual Restrictions invert(const Restrictions& restrictions) const override; - virtual Shape invert_extents(const Shape& extents) const override; - virtual Shape invert_point(const Shape& point) const override; - virtual void pack(BufferBuilder& buffer) const override; - virtual void print(std::ostream& out) const override; + Domain transform(const Domain& domain) const override; + Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const override; + std::unique_ptr convert(const Partition* partition) const override; + std::unique_ptr invert(const Partition* partition) const override; + proj::SymbolicPoint invert(const proj::SymbolicPoint& point) const override; + Restrictions convert(const Restrictions& restrictions) const override; + Restrictions invert(const Restrictions& restrictions) const override; + Shape invert_extents(const Shape& extents) const override; + Shape invert_point(const Shape& point) const override; + bool is_convertible() const override { return false; } + void pack(BufferBuilder& buffer) const override; + void print(std::ostream& out) const override; public: virtual int32_t target_ndim(int32_t source_ndim) const override; diff --git a/src/core/mapping/machine.cc b/src/core/mapping/machine.cc index 2ce7c54b36..e596017726 100644 --- a/src/core/mapping/machine.cc +++ b/src/core/mapping/machine.cc @@ -187,9 +187,9 @@ std::vector MachineDesc::valid_targets_except( return result; } -size_t MachineDesc::count() const { return count(preferred_target); } +uint32_t MachineDesc::count() const { return count(preferred_target); } -size_t MachineDesc::count(TaskTarget target) const { return processor_range(target).count(); } +uint32_t MachineDesc::count(TaskTarget target) const { return processor_range(target).count(); } std::string MachineDesc::to_string() const { diff --git a/src/core/mapping/machine.h b/src/core/mapping/machine.h index 8ee67d1e46..ee04f3fd60 100644 --- a/src/core/mapping/machine.h +++ b/src/core/mapping/machine.h @@ -181,7 +181,7 @@ struct MachineDesc { * * @return Processor count */ - size_t count() const; + uint32_t count() const; /** * @brief Returns the number of processors of a given type * @@ -189,7 +189,7 @@ struct MachineDesc { * * @return Processor count */ - size_t count(TaskTarget target) const; + uint32_t count(TaskTarget target) const; /** * @brief Converts the machine descriptor to a human-readable string diff --git a/tests/cpp/build.sh b/tests/cpp/build.sh index dd0fb9706a..789f587fc1 100755 --- a/tests/cpp/build.sh +++ b/tests/cpp/build.sh @@ -2,4 +2,4 @@ rm -rf build cmake -B build -S . -D legate_core_ROOT=../../build -D CMAKE_BUILD_TYPE=Debug -cmake --build build -j 4 +cmake --build build -j 8 diff --git a/tests/cpp/integration/alignment_constraints.cc b/tests/cpp/integration/alignment_constraints.cc new file mode 100644 index 0000000000..014d357e64 --- /dev/null +++ b/tests/cpp/integration/alignment_constraints.cc @@ -0,0 +1,218 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "legate.h" + +namespace alignment_constraints { + +namespace { + +static const char* library_name = "test_alignment_constraints"; + +enum TaskIDs { + INIT = 0, + ALIGNMENT_TESTER = 0, + ALIGNMENT_BROADCAST_TESTER = 3, + TRANSFORMED_TESTER = 6, +}; + +// Dummy task to make the runtime think the store is initialized +struct Initializer : public legate::LegateTask { + static const int32_t TASK_ID = INIT; + static void cpu_variant(legate::TaskContext& context) {} +}; + +template +struct AlignmentTester : public legate::LegateTask> { + static const int32_t TASK_ID = ALIGNMENT_TESTER + DIM; + static void cpu_variant(legate::TaskContext& context) + { + auto& outputs = context.outputs(); + auto shape = outputs.at(0).shape(); + for (auto& output : outputs) EXPECT_EQ(shape, output.shape()); + } +}; + +template +struct AlignmentBroadcastTester : public legate::LegateTask> { + static const int32_t TASK_ID = ALIGNMENT_BROADCAST_TESTER + DIM; + static void cpu_variant(legate::TaskContext& context) + { + auto shape1 = context.outputs().at(0).shape(); + auto shape2 = context.outputs().at(1).shape(); + auto extent = context.scalars().at(0).value(); + EXPECT_EQ(shape1, shape2); + EXPECT_EQ(shape1.lo[0], 0); + EXPECT_EQ(shape1.hi[0] + 1, extent); + EXPECT_EQ(shape1.hi[0] - shape1.lo[0] + 1, extent); + } +}; + +template +struct TransformedTester : public legate::LegateTask> { + static const int32_t TASK_ID = TRANSFORMED_TESTER + DIM; + static void cpu_variant(legate::TaskContext& context) + { + auto shape1 = context.inputs().at(0).shape(); + auto shape2 = context.inputs().at(1).shape(); + EXPECT_EQ(shape1, shape2); + } +}; + +void prepare() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + Initializer::register_variants(context); + AlignmentTester<1>::register_variants(context); + AlignmentTester<2>::register_variants(context); + AlignmentTester<3>::register_variants(context); + AlignmentBroadcastTester<1>::register_variants(context); + AlignmentBroadcastTester<2>::register_variants(context); + AlignmentBroadcastTester<3>::register_variants(context); + TransformedTester<1>::register_variants(context); + TransformedTester<2>::register_variants(context); + TransformedTester<3>::register_variants(context); +} + +void test_alignment() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + auto launch_tester = [&](const std::vector& extents) { + auto store1 = runtime->create_store(extents, legate::int64()); + auto store2 = runtime->create_store(extents, legate::int64()); + auto store3 = runtime->create_store(extents, legate::int64()); + + auto task = runtime->create_task(context, ALIGNMENT_TESTER + extents.size()); + auto part1 = task->declare_partition(); + auto part2 = task->declare_partition(); + auto part3 = task->declare_partition(); + + task->add_output(store1, part1); + task->add_output(store2, part2); + task->add_output(store3, part3); + + task->add_constraint(legate::align(part1, part2)); + task->add_constraint(legate::align(part2, part3)); + + runtime->submit(std::move(task)); + }; + + launch_tester({10}); + launch_tester({10, 10}); + launch_tester({10, 10, 10}); +} + +void test_alignment_and_broadcast() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + auto launch_tester = [&](const std::vector& extents) { + auto store1 = runtime->create_store(extents, legate::int64()); + auto store2 = runtime->create_store(extents, legate::int64()); + + auto task = runtime->create_task(context, ALIGNMENT_BROADCAST_TESTER + extents.size()); + auto part1 = task->declare_partition(); + auto part2 = task->declare_partition(); + + task->add_output(store1, part1); + task->add_output(store2, part2); + + task->add_scalar_arg(legate::Scalar(extents[0])); + + task->add_constraint(legate::align(part1, part2)); + task->add_constraint(legate::broadcast(part1, {0})); + + runtime->submit(std::move(task)); + }; + + launch_tester({100}); + launch_tester({100, 10}); + launch_tester({100, 10, 20}); +} + +void initialize(legate::LogicalStore store) +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + auto task = runtime->create_task(context, INIT); + + auto part = task->declare_partition(); + task->add_output(store, part); + + runtime->submit(std::move(task)); +} + +void test_alignment_transformed() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + auto launch_tester = [&](auto store1, auto store2) { + auto task = runtime->create_task(context, TRANSFORMED_TESTER + store1.dim()); + auto part1 = task->declare_partition(); + auto part2 = task->declare_partition(); + + task->add_input(store1, part1); + task->add_input(store2, part2); + + task->add_constraint(legate::align(part1, part2)); + + runtime->submit(std::move(task)); + }; + + // TODO: Extents are chosen such that imaginary dimensions are not partitioned. + // Such scenarios should be tested once we pass down partition metadata to the tasks + // and have them compute extents on imaginary dimensions based on that + auto store1 = runtime->create_store({100}, legate::int64()); + auto store2 = runtime->create_store({100, 10}, legate::int64()); + auto store3 = runtime->create_store({100, 10, 2}, legate::int64()); + auto store4 = runtime->create_store({10, 10}, legate::int64()); + + initialize(store1); + initialize(store2); + initialize(store3); + initialize(store4); + + launch_tester(store1.promote(1, 10), store2); + launch_tester(store1, store2.project(1, 5)); + launch_tester(store1.promote(1, 10), store3.project(2, 0)); + launch_tester(store1.promote(1, 10).promote(2, 2), store3); + launch_tester(store1.promote(1, 5), store2.slice(1, legate::Slice(3, 8))); + launch_tester(store1.promote(0, 10).transpose({1, 0}), store2); + launch_tester(store1.delinearize(0, {10, 10}), store4); +} + +} // namespace + +TEST(Integration, AlignmentConstraints) +{ + legate::Core::perform_registration(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + test_alignment(); + test_alignment_and_broadcast(); + test_alignment_transformed(); +} + +} // namespace alignment_constraints diff --git a/tests/cpp/integration/broadcast_constraints.cc b/tests/cpp/integration/broadcast_constraints.cc index 81130dec21..0cf1ce3109 100644 --- a/tests/cpp/integration/broadcast_constraints.cc +++ b/tests/cpp/integration/broadcast_constraints.cc @@ -27,13 +27,16 @@ static const char* library_name = "test_broadcast_constraints"; constexpr size_t EXT_SMALL = 10; constexpr size_t EXT_LARGE = 100; +constexpr int32_t TESTER = 0; +constexpr int32_t INITIALIZER = 1; + struct TesterTask : public legate::LegateTask { - static const int32_t TASK_ID = 0; static void cpu_variant(legate::TaskContext& context) { - auto shape = context.outputs().at(0).shape<3>(); - auto extent = context.scalars().at(0).value(); - auto dims = context.scalars().at(1).values(); + auto extent = context.scalars().at(0).value(); + auto dims = context.scalars().at(1).values(); + auto is_read = context.scalars().at(2).value(); + auto shape = is_read ? context.inputs().at(0).shape<3>() : context.outputs().at(0).shape<3>(); for (auto dim : dims) { EXPECT_EQ(shape.lo[dim], 0); @@ -42,11 +45,16 @@ struct TesterTask : public legate::LegateTask { } }; +struct Initializer : public legate::LegateTask { + static void cpu_variant(legate::TaskContext& context) {} +}; + void prepare() { auto runtime = legate::Runtime::get_runtime(); auto context = runtime->create_library(library_name); - TesterTask::register_variants(context); + TesterTask::register_variants(context, TESTER); + Initializer::register_variants(context, INITIALIZER); } void test_normal_store() @@ -58,11 +66,12 @@ void test_normal_store() std::vector extents(3, EXT_SMALL); for (auto dim : dims) extents[dim] = EXT_LARGE; auto store = runtime->create_store(extents, legate::int64()); - auto task = runtime->create_task(context, 0); + auto task = runtime->create_task(context, TESTER); auto part = task->declare_partition(); task->add_output(store, part); task->add_scalar_arg(legate::Scalar(EXT_LARGE)); task->add_scalar_arg(legate::Scalar(dims)); + task->add_scalar_arg(legate::Scalar(false)); task->add_constraint(legate::broadcast(part, dims)); runtime->submit(std::move(task)); }; @@ -81,16 +90,25 @@ void test_promoted_store() auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(library_name); + auto initialize = [&](auto store) { + auto task = runtime->create_task(context, INITIALIZER); + auto part = task->declare_partition(); + task->add_output(store, part); + runtime->submit(std::move(task)); + }; + auto launch_tester = [&](const int32_t dim) { std::vector extents(2, EXT_SMALL); - extents[dim] = EXT_LARGE; - auto store = runtime->create_store(extents, legate::int64()); - auto promoted = store.promote(2, EXT_LARGE); - auto task = runtime->create_task(context, 0); - auto part = task->declare_partition(); - task->add_output(promoted, part); + extents[dim] = EXT_LARGE; + auto store = runtime->create_store(extents, legate::int64()); + initialize(store); + + auto task = runtime->create_task(context, TESTER); + auto part = task->declare_partition(); + task->add_input(store.promote(2, EXT_LARGE), part); task->add_scalar_arg(legate::Scalar(EXT_LARGE)); task->add_scalar_arg(legate::Scalar(std::vector{dim})); + task->add_scalar_arg(legate::Scalar(true)); task->add_constraint(legate::broadcast(part, {dim})); runtime->submit(std::move(task)); }; diff --git a/tests/cpp/integration/multi_scalar_out.cc b/tests/cpp/integration/multi_scalar_out.cc index ef4f3f2ec4..f12aa8c7a1 100644 --- a/tests/cpp/integration/multi_scalar_out.cc +++ b/tests/cpp/integration/multi_scalar_out.cc @@ -46,7 +46,7 @@ void test_reducer_auto(legate::LibraryContext* context, auto part2 = task->declare_partition(); auto part3 = task->declare_partition(); auto redop1 = scalar1.type().find_reduction_operator(legate::ReductionOpKind::ADD); - auto redop2 = scalar1.type().find_reduction_operator(legate::ReductionOpKind::MUL); + auto redop2 = scalar2.type().find_reduction_operator(legate::ReductionOpKind::MUL); task->add_reduction(scalar1, redop1, part1); task->add_reduction(scalar2, redop2, part2); task->add_output(store, part3); @@ -80,7 +80,7 @@ void print_stores(legate::LibraryContext* context, task::simple::logger.print() << ss.str(); } -TEST(Integration, ManualScalarOut) +TEST(Integration, MultiScalarOut) { legate::Core::perform_registration(); diff --git a/tests/cpp/run.sh b/tests/cpp/run.sh index 1717b0a8ed..ffc95617e7 100755 --- a/tests/cpp/run.sh +++ b/tests/cpp/run.sh @@ -1,4 +1,4 @@ #!/bin/bash cd build -ctest --output-on-failure +LEGATE_TEST=1 LEGION_DEFAULT_ARGS="-ll:cpu 4" ctest --output-on-failure From 26ba938ba4a35e956cd7faf51302e163e21280bf Mon Sep 17 00:00:00 2001 From: Malte Foerster Date: Fri, 26 May 2023 11:01:52 -0700 Subject: [PATCH 0132/1425] added missing header to install target * added missing header to install target See merge request legate/legate.core.internal!41 --- legate_core_cpp.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index 6de0a7b548..b15e3ba91b 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -451,6 +451,7 @@ install( install( FILES src/core/partitioning/constraint.h + src/core/partitioning/partition.h src/core/partitioning/restriction.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/legate/core/partitioning) From 5ea56fe018715e43f1340e08153d498501aed0e4 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Thu, 1 Jun 2023 17:30:02 -0700 Subject: [PATCH 0133/1425] Complete task launcher implementation * Minor improvement for the build script * Install buffer builder for now to allow testing in non-editable mode * Exception handling * Logic for barrier insertion * Communicator support * Partition cache for weighted partitions and logic to import output partitions * Add support for weighted partitions See merge request legate/legate.core.internal!42 --- legate_core_cpp.cmake | 5 +- src/core/comm/comm.cc | 11 ++ src/core/comm/comm.h | 6 +- src/core/comm/comm_cpu.cc | 72 +++++++++ src/core/comm/comm_cpu.h | 8 +- src/core/comm/comm_nccl.cu | 67 ++++++++ src/core/comm/comm_nccl.h | 8 +- src/core/data/logical_store_detail.cc | 2 + src/core/data/logical_store_detail.h | 2 +- src/core/data/shape.cc | 17 +- src/core/data/shape.h | 9 +- src/core/mapping/machine.cc | 13 ++ src/core/mapping/machine.h | 1 + src/core/partitioning/partition.cc | 88 ++++++++++- src/core/partitioning/partition.h | 83 ++++++---- src/core/partitioning/partitioner.cc | 25 ++- src/core/partitioning/partitioner.h | 4 +- src/core/runtime/communicator_manager.cc | 82 ++++++++++ src/core/runtime/communicator_manager.h | 95 +++++++++++ src/core/runtime/launcher.cc | 114 +++++++++++--- src/core/runtime/launcher.h | 37 ++++- src/core/runtime/operation.cc | 99 +++++++++--- src/core/runtime/operation.h | 38 ++++- src/core/runtime/partition_manager.cc | 33 +++- src/core/runtime/partition_manager.h | 8 + src/core/runtime/req_analyzer.h | 2 + src/core/runtime/runtime.cc | 142 ++++++++++++++++- src/core/runtime/runtime.h | 69 +++++++- src/core/task/return.cc | 8 + src/core/task/return.h | 5 + tests/cpp/build.sh | 4 +- tests/cpp/integration/cpu_communicator.cc | 106 +++++++++++++ tests/cpp/integration/exception.cc | 183 ++++++++++++++++++++++ tests/cpp/integration/weighted.cc | 137 ++++++++++++++++ 34 files changed, 1457 insertions(+), 126 deletions(-) create mode 100644 src/core/runtime/communicator_manager.cc create mode 100644 src/core/runtime/communicator_manager.h create mode 100644 tests/cpp/integration/cpu_communicator.cc create mode 100644 tests/cpp/integration/exception.cc create mode 100644 tests/cpp/integration/weighted.cc diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index b15e3ba91b..bb4355ef51 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -213,6 +213,7 @@ list(APPEND legate_core_SOURCES src/core/partitioning/partition.cc src/core/partitioning/partitioner.cc src/core/partitioning/restriction.cc + src/core/runtime/communicator_manager.cc src/core/runtime/context.cc src/core/runtime/field_manager.cc src/core/runtime/launcher_arg.cc @@ -484,7 +485,9 @@ install( src/core/type/type_traits.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/legate/core/type) install( - FILES src/core/utilities/debug.h + FILES src/core/utilities/buffer_builder.h + src/core/utilities/buffer_builder.inl + src/core/utilities/debug.h src/core/utilities/deserializer.h src/core/utilities/deserializer.inl src/core/utilities/dispatch.h diff --git a/src/core/comm/comm.cc b/src/core/comm/comm.cc index 003af32886..d6e0598516 100644 --- a/src/core/comm/comm.cc +++ b/src/core/comm/comm.cc @@ -21,6 +21,9 @@ #include "core/comm/comm_cpu.h" #include "env_defaults.h" +#include "core/runtime/communicator_manager.h" +#include "core/runtime/runtime.h" + namespace legate { namespace comm { @@ -36,5 +39,13 @@ void register_tasks(Legion::Machine machine, if (!disable_mpi) { cpu::register_tasks(machine, runtime, context); } } +void register_builtin_communicator_factories(const LibraryContext* context) +{ +#ifdef LEGATE_USE_CUDA + nccl::register_factory(context); +#endif + cpu::register_factory(context); +} + } // namespace comm } // namespace legate diff --git a/src/core/comm/comm.h b/src/core/comm/comm.h index ef1b63f5cf..4e6e912da6 100644 --- a/src/core/comm/comm.h +++ b/src/core/comm/comm.h @@ -16,15 +16,19 @@ #pragma once -#include "core/runtime/context.h" #include "legate.h" namespace legate { + +class LibraryContext; + namespace comm { void register_tasks(Legion::Machine machine, Legion::Runtime* runtime, const LibraryContext* context); +void register_builtin_communicator_factories(const LibraryContext* context); + } // namespace comm } // namespace legate diff --git a/src/core/comm/comm_cpu.cc b/src/core/comm/comm_cpu.cc index 7ba2d2d00f..4034071da9 100644 --- a/src/core/comm/comm_cpu.cc +++ b/src/core/comm/comm_cpu.cc @@ -15,6 +15,9 @@ */ #include "core/comm/comm_cpu.h" +#include "core/runtime/communicator_manager.h" +#include "core/runtime/launcher.h" +#include "core/runtime/runtime.h" #include "legate.h" #include "core/comm/coll.h" @@ -23,6 +26,69 @@ namespace legate { namespace comm { namespace cpu { +class Factory : public CommunicatorFactory { + public: + Factory(const LibraryContext* core_context); + + public: + bool needs_barrier() const override { return false; } + bool is_supported_target(mapping::TaskTarget target) const override; + + protected: + Legion::FutureMap initialize(const mapping::MachineDesc& machine, uint32_t num_tasks) override; + void finalize(const mapping::MachineDesc& machine, + uint32_t num_tasks, + const Legion::FutureMap& communicator) override; + + private: + const LibraryContext* core_context_; +}; + +Factory::Factory(const LibraryContext* core_context) : core_context_(core_context) {} + +bool Factory::is_supported_target(mapping::TaskTarget target) const +{ + return target == mapping::TaskTarget::OMP || target == mapping::TaskTarget::CPU; +} + +Legion::FutureMap Factory::initialize(const mapping::MachineDesc& machine, uint32_t num_tasks) +{ + Domain launch_domain(Rect<1>(Point<1>(0), Point<1>(static_cast(num_tasks) - 1))); + auto tag = + machine.preferred_target == mapping::TaskTarget::OMP ? LEGATE_OMP_VARIANT : LEGATE_CPU_VARIANT; + + // Generate a unique ID + auto comm_id = Legion::Future::from_value(coll::collInitComm()); + + // Find a mapping of all participants + TaskLauncher init_cpucoll_mapping_launcher( + core_context_, machine, LEGATE_CORE_INIT_CPUCOLL_MAPPING_TASK_ID, tag); + init_cpucoll_mapping_launcher.add_future(comm_id); + auto mapping = init_cpucoll_mapping_launcher.execute(launch_domain); + + // Then create communicators on participating processors + TaskLauncher init_cpucoll_launcher(core_context_, machine, LEGATE_CORE_INIT_CPUCOLL_TASK_ID, tag); + init_cpucoll_launcher.add_future(comm_id); + init_cpucoll_launcher.set_concurrent(true); + + auto domain = mapping.get_future_map_domain(); + for (Domain::DomainPointIterator it(domain); it; ++it) + init_cpucoll_launcher.add_future(mapping.get_future(*it)); + return init_cpucoll_launcher.execute(launch_domain); +} + +void Factory::finalize(const mapping::MachineDesc& machine, + uint32_t num_tasks, + const Legion::FutureMap& communicator) +{ + Domain launch_domain(Rect<1>(Point<1>(0), Point<1>(static_cast(num_tasks) - 1))); + TaskLauncher launcher( + core_context_, machine, LEGATE_CORE_FINALIZE_NCCL_TASK_ID, LEGATE_GPU_VARIANT); + launcher.set_concurrent(true); + launcher.add_future_map(communicator); + launcher.execute(launch_domain); +} + static int init_cpucoll_mapping(const Legion::Task* task, const std::vector& regions, Legion::Context context, @@ -155,6 +221,12 @@ void register_tasks(Legion::Machine machine, } } +void register_factory(const LibraryContext* context) +{ + auto* comm_mgr = Runtime::get_runtime()->communicator_manager(); + comm_mgr->register_factory("cpu", std::make_unique(context)); +} + } // namespace cpu } // namespace comm } // namespace legate diff --git a/src/core/comm/comm_cpu.h b/src/core/comm/comm_cpu.h index 54402e73df..3286f3db4a 100644 --- a/src/core/comm/comm_cpu.h +++ b/src/core/comm/comm_cpu.h @@ -16,10 +16,12 @@ #pragma once -#include "core/runtime/context.h" -#include "legate.h" +#include "legion.h" namespace legate { + +class LibraryContext; + namespace comm { namespace cpu { @@ -27,6 +29,8 @@ void register_tasks(Legion::Machine machine, Legion::Runtime* runtime, const LibraryContext* context); +void register_factory(const LibraryContext* context); + } // namespace cpu } // namespace comm } // namespace legate diff --git a/src/core/comm/comm_nccl.cu b/src/core/comm/comm_nccl.cu index 8d1f9564ad..ecc2e65c53 100644 --- a/src/core/comm/comm_nccl.cu +++ b/src/core/comm/comm_nccl.cu @@ -18,6 +18,9 @@ #include "core/cuda/cuda_help.h" #include "core/cuda/stream_pool.h" #include "core/data/buffer.h" +#include "core/runtime/communicator_manager.h" +#include "core/runtime/launcher.h" +#include "core/runtime/runtime.h" #include "core/utilities/nvtx_help.h" #include "core/utilities/typedefs.h" #include "legate.h" @@ -52,6 +55,64 @@ inline void check_nccl(ncclResult_t error, const char* file, int line) } } +class Factory : public CommunicatorFactory { + public: + Factory(const LibraryContext* core_context); + + public: + bool needs_barrier() const override; + bool is_supported_target(mapping::TaskTarget target) const override; + + protected: + Legion::FutureMap initialize(const mapping::MachineDesc& machine, uint32_t num_tasks) override; + void finalize(const mapping::MachineDesc& machine, + uint32_t num_tasks, + const Legion::FutureMap& communicator) override; + + private: + const LibraryContext* core_context_; +}; + +Factory::Factory(const LibraryContext* core_context) : core_context_(core_context) {} + +bool Factory::needs_barrier() const { return legate::comm::nccl::needs_barrier(); } + +bool Factory::is_supported_target(mapping::TaskTarget target) const +{ + return target == mapping::TaskTarget::GPU; +} + +Legion::FutureMap Factory::initialize(const mapping::MachineDesc& machine, uint32_t num_tasks) +{ + Domain launch_domain(Rect<1>(Point<1>(0), Point<1>(static_cast(num_tasks) - 1))); + + // Create a communicator ID + TaskLauncher init_nccl_id_launcher( + core_context_, machine, LEGATE_CORE_INIT_NCCL_ID_TASK_ID, LEGATE_GPU_VARIANT); + init_nccl_id_launcher.set_side_effect(true); + auto nccl_id = init_nccl_id_launcher.execute_single(); + + // Then create the communicators on participating GPUs + TaskLauncher init_nccl_launcher( + core_context_, machine, LEGATE_CORE_INIT_NCCL_TASK_ID, LEGATE_GPU_VARIANT); + init_nccl_launcher.add_future(nccl_id); + init_nccl_launcher.set_concurrent(true); + return init_nccl_launcher.execute(launch_domain); +} + +void Factory::finalize(const mapping::MachineDesc& machine, + uint32_t num_tasks, + const Legion::FutureMap& communicator) +{ + Domain launch_domain(Rect<1>(Point<1>(0), Point<1>(static_cast(num_tasks) - 1))); + + TaskLauncher launcher( + core_context_, machine, LEGATE_CORE_FINALIZE_NCCL_TASK_ID, LEGATE_GPU_VARIANT); + launcher.set_concurrent(true); + launcher.add_future_map(communicator); + launcher.execute(launch_domain); +} + static ncclUniqueId init_nccl_id(const Legion::Task* task, const std::vector& regions, Legion::Context context, @@ -174,6 +235,12 @@ bool needs_barrier() return true; } +void register_factory(const LibraryContext* context) +{ + auto* comm_mgr = Runtime::get_runtime()->communicator_manager(); + comm_mgr->register_factory("nccl", std::make_unique(context)); +} + } // namespace nccl } // namespace comm } // namespace legate diff --git a/src/core/comm/comm_nccl.h b/src/core/comm/comm_nccl.h index 34470d291b..64d40048e9 100644 --- a/src/core/comm/comm_nccl.h +++ b/src/core/comm/comm_nccl.h @@ -16,10 +16,12 @@ #pragma once -#include "core/runtime/context.h" -#include "legate.h" +#include "legion.h" namespace legate { + +class LibraryContext; + namespace comm { namespace nccl { @@ -29,6 +31,8 @@ void register_tasks(Legion::Machine machine, bool needs_barrier(); +void register_factory(const LibraryContext* context); + } // namespace nccl } // namespace comm } // namespace legate diff --git a/src/core/data/logical_store_detail.cc b/src/core/data/logical_store_detail.cc index 193367a514..828ea56ebc 100644 --- a/src/core/data/logical_store_detail.cc +++ b/src/core/data/logical_store_detail.cc @@ -112,6 +112,8 @@ const Shape& Storage::offsets() const return offsets_; } +size_t Storage::volume() const { return extents().volume(); } + std::shared_ptr Storage::slice(Shape tile_shape, Shape offsets) { if (Kind::FUTURE == kind_) return shared_from_this(); diff --git a/src/core/data/logical_store_detail.h b/src/core/data/logical_store_detail.h index 5e28f50101..6f3355403b 100644 --- a/src/core/data/logical_store_detail.h +++ b/src/core/data/logical_store_detail.h @@ -60,7 +60,7 @@ class Storage : public std::enable_shared_from_this { bool unbound() const { return unbound_; } const Shape& extents() const; const Shape& offsets() const; - size_t volume() const { return volume_; } + size_t volume() const; int32_t dim() { return dim_; } const Type& type() const { return *type_; } Kind kind() const { return kind_; } diff --git a/src/core/data/shape.cc b/src/core/data/shape.cc index 44733859a6..a415a88d78 100644 --- a/src/core/data/shape.cc +++ b/src/core/data/shape.cc @@ -18,9 +18,9 @@ namespace legate { -Legion::Domain to_domain(const tuple& shape) +Domain to_domain(const tuple& shape) { - Legion::Domain domain; + Domain domain; auto ndim = static_cast(shape.size()); domain.dim = ndim; for (int32_t idx = 0; idx < ndim; ++idx) { @@ -30,13 +30,22 @@ Legion::Domain to_domain(const tuple& shape) return domain; } -Legion::DomainPoint to_domain_point(const Shape& shape) +DomainPoint to_domain_point(const Shape& shape) { - Legion::DomainPoint point; + DomainPoint point; auto ndim = static_cast(shape.size()); point.dim = ndim; for (int32_t idx = 0; idx < ndim; ++idx) point[idx] = static_cast(shape[idx]); return point; } +Shape from_domain(const Domain& domain) +{ + std::vector result; + auto lo = domain.lo(); + auto hi = domain.hi(); + for (int32_t idx = 0; idx < domain.dim; ++idx) result.push_back(hi[idx] - lo[idx] + 1); + return Shape(std::move(result)); +} + } // namespace legate diff --git a/src/core/data/shape.h b/src/core/data/shape.h index 06e0dcd084..594415ea0c 100644 --- a/src/core/data/shape.h +++ b/src/core/data/shape.h @@ -17,15 +17,16 @@ #pragma once #include "core/utilities/tuple.h" - -#include "legion.h" +#include "core/utilities/typedefs.h" namespace legate { using Shape = tuple; -Legion::Domain to_domain(const Shape& shape); +Domain to_domain(const Shape& shape); + +DomainPoint to_domain_point(const Shape& shape); -Legion::DomainPoint to_domain_point(const Shape& shape); +Shape from_domain(const Domain& domain); } // namespace legate diff --git a/src/core/mapping/machine.cc b/src/core/mapping/machine.cc index e596017726..26ce331a97 100644 --- a/src/core/mapping/machine.cc +++ b/src/core/mapping/machine.cc @@ -103,6 +103,19 @@ bool ProcessorRange::operator==(const ProcessorRange& other) const bool ProcessorRange::operator!=(const ProcessorRange& other) const { return !operator==(other); } +bool ProcessorRange::operator<(const ProcessorRange& other) const +{ + if (low < other.low) + return true; + else if (low > other.low) + return false; + if (high < other.high) + return true; + else if (high > other.high) + return false; + return per_node_count < other.per_node_count; +} + uint32_t ProcessorRange::count() const { return high - low; } bool ProcessorRange::empty() const { return high <= low; } diff --git a/src/core/mapping/machine.h b/src/core/mapping/machine.h index ee04f3fd60..9f361addcb 100644 --- a/src/core/mapping/machine.h +++ b/src/core/mapping/machine.h @@ -59,6 +59,7 @@ struct ProcessorRange { ProcessorRange operator&(const ProcessorRange&) const; bool operator==(const ProcessorRange&) const; bool operator!=(const ProcessorRange&) const; + bool operator<(const ProcessorRange&) const; /** * @brief Returns the number of processors in the range * diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index 3b18ef1d9f..0df708bec8 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -110,6 +110,8 @@ bool Tiling::is_complete_for(const detail::Storage* storage) const bool Tiling::is_disjoint_for(const Domain* launch_domain) const { + // TODO: The check really should be that every two points from the launch domain are mapped + // to two different colors return nullptr == launch_domain || launch_domain->get_volume() <= color_shape_.volume(); } @@ -185,16 +187,94 @@ Shape Tiling::get_child_offsets(const Shape& color) offsets_); } -std::unique_ptr create_no_partition() { return std::make_unique(); } +Weighted::Weighted(const Legion::FutureMap& weights, const Domain& color_domain) + : weights_(weights), color_domain_(color_domain), color_shape_(from_domain(color_domain)) +{ +} + +bool Weighted::operator==(const Weighted& other) const +{ + // Since both color_domain_ and color_shape_ are derived from weights_, they don't need to + // be compared + return weights_ == other.weights_; +} + +bool Weighted::operator<(const Weighted& other) const { return weights_ < other.weights_; } + +bool Weighted::is_complete_for(const detail::Storage*) const +{ + // Partition-by-weight partitions are complete by definition + return true; +} + +bool Weighted::is_disjoint_for(const Domain* launch_domain) const +{ + // TODO: The check really should be that every two points from the launch domain are mapped + // to two different colors + return nullptr == launch_domain || launch_domain->get_volume() <= color_domain_.get_volume(); +} + +bool Weighted::satisfies_restrictions(const Restrictions& restrictions) const +{ + static auto satisfies_restriction = [](Restriction r, size_t ext) { + return r != Restriction::FORBID || ext == 1; + }; + return apply(satisfies_restriction, restrictions, color_shape_).all(); +} + +Legion::LogicalPartition Weighted::construct(Legion::LogicalRegion region, bool) const +{ + auto runtime = Runtime::get_runtime(); + auto part_mgr = runtime->partition_manager(); -std::unique_ptr create_tiling(Shape&& tile_shape, - Shape&& color_shape, - tuple&& offsets /*= {}*/) + const auto& index_space = region.get_index_space(); + auto index_partition = part_mgr->find_index_partition(index_space, *this); + if (index_partition != Legion::IndexPartition::NO_PART) + return runtime->create_logical_partition(region, index_partition); + + auto color_space = runtime->find_or_create_index_space(color_domain_); + index_partition = runtime->create_weighted_partition(index_space, color_space, weights_); + part_mgr->record_index_partition(index_space, *this, index_partition); + return runtime->create_logical_partition(region, index_partition); +} + +bool Weighted::has_launch_domain() const { return true; } + +Domain Weighted::launch_domain() const { return color_domain_; } + +std::unique_ptr Weighted::clone() const +{ + return create_weighted(weights_, color_domain_); +} + +std::string Weighted::to_string() const +{ + std::stringstream ss; + ss << "Weighted({"; + for (Domain::DomainPointIterator it(color_domain_); it; ++it) { + auto& p = *it; + ss << p << ":" << weights_.get_result(p) << ","; + } + ss << "})"; + return std::move(ss).str(); +} + +std::unique_ptr create_no_partition() { return std::make_unique(); } + +std::unique_ptr create_tiling(Shape&& tile_shape, + Shape&& color_shape, + tuple&& offsets /*= {}*/) { return std::make_unique( std::move(tile_shape), std::move(color_shape), std::move(offsets)); } +std::unique_ptr create_weighted(const Legion::FutureMap& weights, + const Domain& color_domain) +{ + return std::make_unique(weights, color_domain); +} + std::ostream& operator<<(std::ostream& out, const Partition& partition) { out << partition.to_string(); diff --git a/src/core/partitioning/partition.h b/src/core/partitioning/partition.h index 0a0640b433..85daae159d 100644 --- a/src/core/partitioning/partition.h +++ b/src/core/partitioning/partition.h @@ -40,6 +40,7 @@ struct Partition { enum class Kind : int32_t { NO_PARTITION = 0, TILING = 1, + WEIGHTED = 2, }; public: @@ -59,19 +60,14 @@ struct Partition { bool complete = false) const = 0; public: - virtual bool has_launch_domain() const = 0; - virtual Legion::Domain launch_domain() const = 0; + virtual bool has_launch_domain() const = 0; + virtual Domain launch_domain() const = 0; public: virtual std::unique_ptr clone() const = 0; public: virtual std::string to_string() const = 0; - - public: - virtual const Shape& tile_shape() const = 0; - virtual const Shape& color_shape() const = 0; - virtual const tuple& offsets() const = 0; }; class NoPartition : public Partition { @@ -91,27 +87,13 @@ class NoPartition : public Partition { public: bool has_launch_domain() const override; - Legion::Domain launch_domain() const override; + Domain launch_domain() const override; public: std::unique_ptr clone() const override; public: std::string to_string() const override; - - public: - const Shape& tile_shape() const override - { - throw std::invalid_argument("Partition kind doesn't support tile_shape"); - } - const Shape& color_shape() const override - { - throw std::invalid_argument("Partition kind doesn't support color_shape"); - } - const tuple& offsets() const override - { - throw std::invalid_argument("Partition kind doesn't support offsets"); - } }; class Tiling : public Partition { @@ -138,7 +120,7 @@ class Tiling : public Partition { public: bool has_launch_domain() const override; - Legion::Domain launch_domain() const override; + Domain launch_domain() const override; public: std::unique_ptr clone() const override; @@ -147,9 +129,9 @@ class Tiling : public Partition { std::string to_string() const override; public: - const Shape& tile_shape() const override { return tile_shape_; } - const Shape& color_shape() const override { return color_shape_; } - const tuple& offsets() const override { return offsets_; } + const Shape& tile_shape() const { return tile_shape_; } + const Shape& color_shape() const { return color_shape_; } + const tuple& offsets() const { return offsets_; } public: Shape get_child_extents(const Shape& extents, const Shape& color); @@ -161,11 +143,52 @@ class Tiling : public Partition { tuple offsets_; }; -std::unique_ptr create_no_partition(); +class Weighted : public Partition { + public: + Weighted(const Legion::FutureMap& weights, const Domain& color_domain); + + public: + Weighted(const Weighted&) = default; + + public: + bool operator==(const Weighted& other) const; + bool operator<(const Weighted& other) const; + + public: + Kind kind() const override { return Kind::WEIGHTED; } + + public: + bool is_complete_for(const detail::Storage* storage) const override; + bool is_disjoint_for(const Domain* launch_domain) const override; + bool satisfies_restrictions(const Restrictions& restrictions) const override; + + public: + Legion::LogicalPartition construct(Legion::LogicalRegion region, bool complete) const override; + + public: + bool has_launch_domain() const override; + Domain launch_domain() const override; + + public: + std::unique_ptr clone() const override; + + public: + std::string to_string() const override; + + private: + Legion::FutureMap weights_; + Domain color_domain_; + Shape color_shape_; +}; + +std::unique_ptr create_no_partition(); + +std::unique_ptr create_tiling(Shape&& tile_shape, + Shape&& color_shape, + tuple&& offsets = {}); -std::unique_ptr create_tiling(Shape&& tile_shape, - Shape&& color_shape, - tuple&& offsets = {}); +std::unique_ptr create_weighted(const Legion::FutureMap& weights, + const Domain& color_domain); std::ostream& operator<<(std::ostream& out, const Partition& partition); diff --git a/src/core/partitioning/partitioner.cc b/src/core/partitioning/partitioner.cc index 0b7a526bcc..e38a6ceba2 100644 --- a/src/core/partitioning/partitioner.cc +++ b/src/core/partitioning/partitioner.cc @@ -36,21 +36,21 @@ class LaunchDomainResolver { static constexpr int32_t UNSET = -1; public: - void record_launch_domain(const Legion::Domain& launch_domain); + void record_launch_domain(const Domain& launch_domain); void record_unbound_store(int32_t unbound_dim); public: - std::unique_ptr resolve_launch_domain() const; + std::unique_ptr resolve_launch_domain() const; private: bool must_be_sequential_{false}; bool must_be_1d_{false}; int32_t unbound_dim_{UNSET}; - std::set launch_domains_; + std::set launch_domains_; std::set launch_volumes_; }; -void LaunchDomainResolver::record_launch_domain(const Legion::Domain& launch_domain) +void LaunchDomainResolver::record_launch_domain(const Domain& launch_domain) { launch_domains_.insert(launch_domain); launch_volumes_.insert(launch_domain.get_volume()); @@ -66,7 +66,7 @@ void LaunchDomainResolver::record_unbound_store(int32_t unbound_dim) unbound_dim_ = unbound_dim; } -std::unique_ptr LaunchDomainResolver::resolve_launch_domain() const +std::unique_ptr LaunchDomainResolver::resolve_launch_domain() const { if (must_be_sequential_ || launch_domains_.empty()) return nullptr; if (must_be_1d_) { @@ -77,7 +77,7 @@ std::unique_ptr LaunchDomainResolver::resolve_launch_domain() co assert(launch_volumes_.size() == 1); #endif int64_t volume = *launch_volumes_.begin(); - return std::make_unique(0, volume - 1); + return std::make_unique(0, volume - 1); } } else { #ifdef DEBUG_LEGATE @@ -86,9 +86,9 @@ std::unique_ptr LaunchDomainResolver::resolve_launch_domain() co auto& launch_domain = *launch_domains_.begin(); if (unbound_dim_ != UNSET && launch_domain.dim != unbound_dim_) { int64_t volume = *launch_volumes_.begin(); - return std::make_unique(0, volume - 1); + return std::make_unique(0, volume - 1); } - return std::make_unique(launch_domain); + return std::make_unique(launch_domain); } } @@ -104,7 +104,7 @@ bool Strategy::parallel(const Operation* op) const return finder != launch_domains_.end() && finder->second != nullptr; } -const Legion::Domain* Strategy::launch_domain(const Operation* op) const +const Domain* Strategy::launch_domain(const Operation* op) const { auto finder = launch_domains_.find(op); return finder != launch_domains_.end() ? finder->second.get() : nullptr; @@ -115,7 +115,7 @@ void Strategy::set_launch_shape(const Operation* op, const Shape& shape) #ifdef DEBUG_LEGATE assert(launch_domains_.find(op) == launch_domains_.end()); #endif - launch_domains_.insert({op, std::make_unique(to_domain(shape))}); + launch_domains_.insert({op, std::make_unique(to_domain(shape))}); } void Strategy::insert(const Variable* partition_symbol, std::shared_ptr partition) @@ -237,11 +237,6 @@ std::unique_ptr Partitioner::partition_stores() return std::make_pair(store->storage_size(), has_key_part); }; - for (auto& part_symb : remaining_symbols) { - auto [size, has_key_part] = comparison_key(part_symb); - log_legate.debug() << part_symb->to_string() << " " << size << " " << has_key_part; - } - std::stable_sort(remaining_symbols.begin(), remaining_symbols.end(), [&solver, &comparison_key](const auto& part_symb_a, const auto& part_symb_b) { diff --git a/src/core/partitioning/partitioner.h b/src/core/partitioning/partitioner.h index 30042f60bf..f200b515c7 100644 --- a/src/core/partitioning/partitioner.h +++ b/src/core/partitioning/partitioner.h @@ -41,7 +41,7 @@ class Strategy { public: bool parallel(const Operation* op) const; - const Legion::Domain* launch_domain(const Operation* op) const; + const Domain* launch_domain(const Operation* op) const; void set_launch_shape(const Operation* op, const Shape& shape); public: @@ -62,7 +62,7 @@ class Strategy { private: std::map> assignments_{}; std::map field_spaces_{}; - std::map> launch_domains_{}; + std::map> launch_domains_{}; }; class Partitioner { diff --git a/src/core/runtime/communicator_manager.cc b/src/core/runtime/communicator_manager.cc new file mode 100644 index 0000000000..282118ae56 --- /dev/null +++ b/src/core/runtime/communicator_manager.cc @@ -0,0 +1,82 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/runtime/communicator_manager.h" + +#include "core/runtime/machine_manager.h" +#include "core/runtime/runtime.h" + +namespace legate { + +CommunicatorFactory::CommunicatorFactory() {} + +CommunicatorFactory::~CommunicatorFactory() {} + +Legion::FutureMap CommunicatorFactory::find_or_create(const mapping::TaskTarget& target, + const mapping::ProcessorRange& range, + const Domain& launch_domain) +{ + if (launch_domain.dim == 1) return find_or_create(target, range, launch_domain.get_volume()); + + AliasKey key{launch_domain, target, range}; + auto finder = nd_aliases_.find(key); + if (finder != nd_aliases_.end()) return finder->second; + + auto communicator = find_or_create(target, range, launch_domain.get_volume()); + communicator = transform(communicator, launch_domain); + nd_aliases_.insert({key, communicator}); + return communicator; +} + +void CommunicatorFactory::destroy() +{ + for (auto& [key, communicator] : communicators_) + finalize(key.get_machine(), key.desc, communicator); +} + +Legion::FutureMap CommunicatorFactory::find_or_create(const mapping::TaskTarget& target, + const mapping::ProcessorRange& range, + uint32_t num_tasks) +{ + CommKey key{num_tasks, target, range}; + auto finder = communicators_.find(key); + if (finder != communicators_.end()) return finder->second; + + auto communicator = initialize(key.get_machine(), num_tasks); + communicators_.insert({key, communicator}); + return communicator; +} + +Legion::FutureMap CommunicatorFactory::transform(const Legion::FutureMap& communicator, + const Domain& launch_domain) +{ + auto* runtime = Runtime::get_runtime(); + auto new_domain = runtime->find_or_create_index_space(launch_domain); + return runtime->delinearize_future_map(communicator, new_domain); +} + +CommunicatorFactory* CommunicatorManager::find_factory(const std::string& name) +{ + return factories_.at(name).get(); +} + +void CommunicatorManager::register_factory(const std::string& name, + std::unique_ptr factory) +{ + factories_.insert({name, std::move(factory)}); +} + +} // namespace legate diff --git a/src/core/runtime/communicator_manager.h b/src/core/runtime/communicator_manager.h new file mode 100644 index 0000000000..18344d520c --- /dev/null +++ b/src/core/runtime/communicator_manager.h @@ -0,0 +1,95 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include +#include + +#include "core/mapping/machine.h" +#include "core/utilities/typedefs.h" + +namespace legate { + +class CommunicatorFactory { + protected: + CommunicatorFactory(); + + public: + virtual ~CommunicatorFactory(); + + public: + Legion::FutureMap find_or_create(const mapping::TaskTarget& target, + const mapping::ProcessorRange& range, + const Domain& launch_domain); + void destroy(); + + protected: + Legion::FutureMap find_or_create(const mapping::TaskTarget& target, + const mapping::ProcessorRange& range, + uint32_t num_tasks); + Legion::FutureMap transform(const Legion::FutureMap& communicator, const Domain& launch_domain); + + public: + virtual bool needs_barrier() const = 0; + virtual bool is_supported_target(mapping::TaskTarget target) const = 0; + + protected: + virtual Legion::FutureMap initialize(const mapping::MachineDesc& machine, uint32_t num_tasks) = 0; + virtual void finalize(const mapping::MachineDesc& machine, + uint32_t num_tasks, + const Legion::FutureMap& communicator) = 0; + + private: + template + struct CacheKey { + Desc desc; + mapping::TaskTarget target; + mapping::ProcessorRange range; + mapping::MachineDesc get_machine() const { return mapping::MachineDesc({{target, range}}); } + bool operator==(const CacheKey& other) const + { + return desc == other.desc && target == other.target && range == other.range; + } + bool operator<(const CacheKey& other) const + { + if (desc < other.desc) + return true; + else if (other.desc < desc) + return false; + if (target < other.target) + return true; + else if (target > other.target) + return false; + return range < other.range; + } + }; + using CommKey = CacheKey; + using AliasKey = CacheKey; + std::map communicators_; + std::map nd_aliases_; +}; + +class CommunicatorManager { + public: + CommunicatorFactory* find_factory(const std::string& name); + void register_factory(const std::string& name, std::unique_ptr factory); + + private: + std::unordered_map> factories_; +}; + +} // namespace legate diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/launcher.cc index bd97ff6ed8..05e76e886f 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -22,6 +22,7 @@ #include "core/mapping/machine.h" #include "core/runtime/context.h" #include "core/runtime/launcher_arg.h" +#include "core/runtime/partition_manager.h" #include "core/runtime/req_analyzer.h" #include "core/runtime/runtime.h" #include "core/utilities/buffer_builder.h" @@ -29,18 +30,23 @@ namespace legate { -TaskLauncher::TaskLauncher(LibraryContext* library, +TaskLauncher::TaskLauncher(const LibraryContext* library, + const mapping::MachineDesc& machine, + int64_t task_id, + int64_t tag /*= 0*/) + : library_(library), task_id_(task_id), tag_(tag), machine_(machine), provenance_("") +{ + initialize(); +} + +TaskLauncher::TaskLauncher(const LibraryContext* library, const mapping::MachineDesc& machine, const std::string& provenance, int64_t task_id, int64_t tag /*= 0*/) - : library_(library), task_id_(task_id), tag_(tag), provenance_(provenance) + : library_(library), task_id_(task_id), tag_(tag), machine_(machine), provenance_(provenance) { - req_analyzer_ = new RequirementAnalyzer(); - out_analyzer_ = new OutputRequirementAnalyzer(); - buffer_ = new BufferBuilder(); - mapper_arg_ = new BufferBuilder(); - machine.pack(*mapper_arg_); + initialize(); } TaskLauncher::~TaskLauncher() @@ -55,6 +61,15 @@ TaskLauncher::~TaskLauncher() delete mapper_arg_; } +void TaskLauncher::initialize() +{ + req_analyzer_ = new RequirementAnalyzer(); + out_analyzer_ = new OutputRequirementAnalyzer(); + buffer_ = new BufferBuilder(); + mapper_arg_ = new BufferBuilder(); + machine_.pack(*mapper_arg_); +} + int64_t TaskLauncher::legion_task_id() const { return library_->get_task_id(task_id_); } int64_t TaskLauncher::legion_mapper_id() const { return library_->get_mapper_id(); } @@ -114,6 +129,11 @@ void TaskLauncher::add_future_map(const Legion::FutureMap& future_map) future_maps_.push_back(future_map); } +void TaskLauncher::add_communicator(const Legion::FutureMap& communicator) +{ + communicators_.push_back(communicator); +} + Legion::FutureMap TaskLauncher::execute(const Legion::Domain& launch_domain) { auto legion_launcher = build_index_task(launch_domain); @@ -121,7 +141,7 @@ Legion::FutureMap TaskLauncher::execute(const Legion::Domain& launch_domain) if (output_requirements_.empty()) return Runtime::get_runtime()->dispatch(legion_launcher.get()); auto result = Runtime::get_runtime()->dispatch(legion_launcher.get(), &output_requirements_); - bind_region_fields_to_unbound_stores(); + post_process_unbound_stores(result, launch_domain); return result; } @@ -131,7 +151,7 @@ Legion::Future TaskLauncher::execute_single() if (output_requirements_.empty()) return Runtime::get_runtime()->dispatch(legion_launcher.get()); auto result = Runtime::get_runtime()->dispatch(legion_launcher.get(), &output_requirements_); - bind_region_fields_to_unbound_stores(); + post_process_unbound_stores(); return result; } @@ -184,8 +204,7 @@ std::unique_ptr TaskLauncher::build_single_task() pack_args(outputs_); pack_args(reductions_); pack_args(scalars_); - // can_raise_exception - buffer_->pack(false); + buffer_->pack(can_throw_exception_); // insert_barrier buffer_->pack(false); // # communicators @@ -206,6 +225,9 @@ std::unique_ptr TaskLauncher::build_single_task() req_analyzer_->populate_launcher(single_task.get()); out_analyzer_->populate_output_requirements(output_requirements_); + single_task->local_function_task = + !has_side_effect_ && req_analyzer_->empty() && out_analyzer_->empty(); + return single_task; } @@ -221,12 +243,9 @@ std::unique_ptr TaskLauncher::build_index_task( pack_args(outputs_); pack_args(reductions_); pack_args(scalars_); - // can_raise_exception - buffer_->pack(false); - // insert_barrier - buffer_->pack(false); - // # communicators - buffer_->pack(0); + buffer_->pack(can_throw_exception_); + buffer_->pack(insert_barrier_); + buffer_->pack(communicators_.size()); pack_sharding_functor_id(); auto* runtime = Runtime::get_runtime(); @@ -242,17 +261,31 @@ std::unique_ptr TaskLauncher::build_index_task( mapper_arg_->to_legion_buffer(), provenance_.c_str()); for (auto& future : futures_) index_task->add_future(future); + if (insert_barrier_) { + size_t num_tasks = launch_domain.get_volume(); + auto [arrival_barrier, wait_barrier] = runtime->create_barriers(num_tasks); + index_task->add_future(Legion::Future::from_value(arrival_barrier)); + index_task->add_future(Legion::Future::from_value(wait_barrier)); + } + for (auto& communicator : communicators_) index_task->point_futures.push_back(communicator); for (auto& future_map : future_maps_) index_task->point_futures.push_back(future_map); req_analyzer_->populate_launcher(index_task.get()); out_analyzer_->populate_output_requirements(output_requirements_); + index_task->concurrent = concurrent_ || !communicators_.empty(); + return index_task; } -void TaskLauncher::bind_region_fields_to_unbound_stores() +void TaskLauncher::bind_region_fields_to_unbound_stores() {} + +void TaskLauncher::post_process_unbound_stores() { + if (unbound_stores_.empty()) return; + auto* runtime = Runtime::get_runtime(); + auto no_part = create_no_partition(); for (auto& arg : unbound_stores_) { #ifdef DEBUG_LEGATE @@ -263,6 +296,51 @@ void TaskLauncher::bind_region_fields_to_unbound_stores() auto region_field = runtime->import_region_field(req.parent, arg->field_id(), store->type().size()); store->set_region_field(std::move(region_field)); + store->set_key_partition(machine_, no_part.get()); + } +} + +void TaskLauncher::post_process_unbound_stores(const Legion::FutureMap& result, + const Legion::Domain& launch_domain) +{ + if (unbound_stores_.empty()) return; + + auto* runtime = Runtime::get_runtime(); + auto* part_mgr = runtime->partition_manager(); + + auto post_process_unbound_store = + [&runtime, &part_mgr, &launch_domain]( + auto*& arg, const auto& req, const auto& weights, const auto& machine) { + auto* store = arg->store(); + auto region_field = + runtime->import_region_field(req.parent, arg->field_id(), store->type().size()); + store->set_region_field(std::move(region_field)); + + // TODO: Need to handle key partitions for multi-dimensional unbound stores + if (store->dim() > 1) return; + + auto partition = create_weighted(weights, launch_domain); + store->set_key_partition(machine, partition.get()); + + const auto& index_partition = req.partition.get_index_partition(); + part_mgr->record_index_partition(req.parent.get_index_space(), *partition, index_partition); + }; + + if (unbound_stores_.size() == 1) { + auto* arg = unbound_stores_.front(); + const auto& req = output_requirements_[arg->requirement_index()]; + post_process_unbound_store(arg, req, result, machine_); + } else { + uint32_t idx = 0; + for (auto& arg : unbound_stores_) { + const auto& req = output_requirements_[arg->requirement_index()]; + if (arg->store()->dim() == 1) + post_process_unbound_store( + arg, req, runtime->extract_scalar(result, idx, launch_domain), machine_); + else + post_process_unbound_store(arg, req, result, machine_); + ++idx; + } } } diff --git a/src/core/runtime/launcher.h b/src/core/runtime/launcher.h index 2164b9cbd0..f6a98135e2 100644 --- a/src/core/runtime/launcher.h +++ b/src/core/runtime/launcher.h @@ -20,6 +20,8 @@ #include "legion.h" +#include "core/mapping/machine.h" + namespace legate { class ArgWrapper; @@ -32,12 +34,6 @@ class OutputRequirementAnalyzer; class RequirementAnalyzer; class Scalar; -namespace mapping { - -class MachineDesc; - -} // namespace mapping - namespace detail { class LogicalStore; @@ -46,13 +42,20 @@ class LogicalStore; class TaskLauncher { public: - TaskLauncher(LibraryContext* library, + TaskLauncher(const LibraryContext* library, + const mapping::MachineDesc& machine, + int64_t task_id, + int64_t tag = 0); + TaskLauncher(const LibraryContext* library, const mapping::MachineDesc& machine, const std::string& provenance, int64_t task_id, int64_t tag = 0); ~TaskLauncher(); + private: + void initialize(); + public: int64_t legion_task_id() const; int64_t legion_mapper_id() const; @@ -79,6 +82,13 @@ class TaskLauncher { public: void add_future(const Legion::Future& future); void add_future_map(const Legion::FutureMap& future_map); + void add_communicator(const Legion::FutureMap& communicator); + + public: + void set_side_effect(bool has_side_effect) { has_side_effect_ = has_side_effect; } + void set_concurrent(bool is_concurrent) { concurrent_ = is_concurrent; } + void set_insert_barrier(bool insert_barrier) { insert_barrier_ = insert_barrier; } + void throws_exception(bool can_throw_exception) { can_throw_exception_ = can_throw_exception; } private: void add_store(std::vector& args, @@ -98,13 +108,23 @@ class TaskLauncher { std::unique_ptr build_index_task(const Legion::Domain& launch_domain); std::unique_ptr build_single_task(); void bind_region_fields_to_unbound_stores(); + void post_process_unbound_stores(); + void post_process_unbound_stores(const Legion::FutureMap& result, + const Legion::Domain& launch_domain); private: - LibraryContext* library_; + const LibraryContext* library_; int64_t task_id_; int64_t tag_; + mapping::MachineDesc machine_; std::string provenance_; + private: + bool has_side_effect_{true}; + bool concurrent_{false}; + bool insert_barrier_{false}; + bool can_throw_exception_{false}; + private: std::vector inputs_; std::vector outputs_; @@ -113,6 +133,7 @@ class TaskLauncher { std::vector futures_; std::vector unbound_stores_; std::vector future_maps_; + std::vector communicators_; private: RequirementAnalyzer* req_analyzer_; diff --git a/src/core/runtime/operation.cc b/src/core/runtime/operation.cc index fc6552a821..14736e7e0b 100644 --- a/src/core/runtime/operation.cc +++ b/src/core/runtime/operation.cc @@ -25,6 +25,7 @@ #include "core/partitioning/constraint_solver.h" #include "core/partitioning/partition.h" #include "core/partitioning/partitioner.h" +#include "core/runtime/communicator_manager.h" #include "core/runtime/context.h" #include "core/runtime/launcher.h" #include "core/runtime/req_analyzer.h" @@ -73,6 +74,21 @@ Task::Task(LibraryContext* library, void Task::add_scalar_arg(const Scalar& scalar) { scalars_.push_back(scalar); } +void Task::set_concurrent(bool concurrent) { concurrent_ = concurrent; } + +void Task::set_side_effect(bool has_side_effect) { has_side_effect_ = has_side_effect; } + +void Task::throws_exception(bool can_throw_exception) +{ + can_throw_exception_ = can_throw_exception; +} + +void Task::add_communicator(const std::string& name) +{ + auto* comm_mgr = Runtime::get_runtime()->communicator_manager(); + communicator_factories_.push_back(comm_mgr->find_factory(name)); +} + void Task::launch(Strategy* p_strategy) { auto& strategy = *p_strategy; @@ -123,6 +139,21 @@ void Task::launch(Strategy* p_strategy) // Add by-value scalars for (auto& scalar : scalars_) launcher.add_scalar(scalar); + // Add communicators + if (launch_domain != nullptr) + for (auto* factory : communicator_factories_) { + auto target = machine_.preferred_target; + if (!factory->is_supported_target(target)) continue; + auto& processor_range = machine_.processor_range(); + auto communicator = factory->find_or_create(target, processor_range, *launch_domain); + launcher.add_communicator(communicator); + if (factory->needs_barrier()) launcher.set_insert_barrier(true); + } + + launcher.set_side_effect(has_side_effect_); + launcher.set_concurrent(concurrent_); + launcher.throws_exception(can_throw_exception_); + if (launch_domain != nullptr) { auto result = launcher.execute(*launch_domain); demux_scalar_stores(result, *launch_domain); @@ -134,25 +165,33 @@ void Task::launch(Strategy* p_strategy) void Task::demux_scalar_stores(const Legion::Future& result) { - // TODO: Handle unbound stores - auto num_scalar_outs = scalar_outputs_.size(); - auto num_scalar_reds = scalar_reductions_.size(); + auto num_scalar_outs = scalar_outputs_.size(); + auto num_scalar_reds = scalar_reductions_.size(); + auto num_unbound_outs = unbound_outputs_.size(); - auto total = num_scalar_outs + num_scalar_reds; + auto total = num_scalar_outs + num_scalar_reds + num_unbound_outs + + static_cast(can_throw_exception_); if (0 == total) return; else if (1 == total) { if (1 == num_scalar_outs) { auto [store, _] = outputs_[scalar_outputs_.front()]; store->set_future(result); - } else { - assert(1 == num_scalar_reds); + } else if (1 == num_scalar_reds) { auto [store, _] = reductions_[scalar_reductions_.front()]; store->set_future(result); + } else if (can_throw_exception_) { + auto* runtime = Runtime::get_runtime(); + runtime->record_pending_exception(result); + } +#ifdef DEBUG_LEGATE + else { + assert(1 == num_unbound_outs); } +#endif } else { auto* runtime = Runtime::get_runtime(); - uint32_t idx = 0; + uint32_t idx = num_unbound_outs; for (const auto& out_idx : scalar_outputs_) { auto [store, _] = outputs_[out_idx]; store->set_future(runtime->extract_scalar(result, idx++)); @@ -161,33 +200,48 @@ void Task::demux_scalar_stores(const Legion::Future& result) auto [store, _] = reductions_[red_idx]; store->set_future(runtime->extract_scalar(result, idx++)); } + if (can_throw_exception_) + runtime->record_pending_exception(runtime->extract_scalar(result, idx)); } } -void Task::demux_scalar_stores(const Legion::FutureMap& result, const Legion::Domain& launch_domain) +void Task::demux_scalar_stores(const Legion::FutureMap& result, const Domain& launch_domain) { // Tasks with scalar outputs shouldn't have been parallelized assert(scalar_outputs_.empty()); - // TODO: Handle unbound stores - auto num_scalar_reds = scalar_reductions_.size(); + auto num_scalar_reds = scalar_reductions_.size(); + auto num_unbound_outs = unbound_outputs_.size(); - auto total = num_scalar_reds; + auto total = num_scalar_reds + num_unbound_outs + static_cast(can_throw_exception_); if (0 == total) return; auto* runtime = Runtime::get_runtime(); if (1 == total) { - assert(1 == num_scalar_reds); - auto red_idx = scalar_reductions_.front(); - auto [store, _] = reductions_[red_idx]; - store->set_future(runtime->reduce_future_map(result, reduction_ops_[red_idx])); + if (1 == num_scalar_reds) { + auto red_idx = scalar_reductions_.front(); + auto [store, _] = reductions_[red_idx]; + store->set_future(runtime->reduce_future_map(result, reduction_ops_[red_idx])); + } else if (can_throw_exception_) { + auto* runtime = Runtime::get_runtime(); + runtime->record_pending_exception(runtime->reduce_exception_future_map(result)); + } +#ifdef DEBUG_LEGATE + else { + assert(1 == num_unbound_outs); + } +#endif } else { - uint32_t idx = 0; + uint32_t idx = num_unbound_outs; for (const auto& red_idx : scalar_reductions_) { auto [store, _] = reductions_[red_idx]; auto values = runtime->extract_scalar(result, idx++, launch_domain); store->set_future(runtime->reduce_future_map(values, reduction_ops_[red_idx])); } + if (can_throw_exception_) { + auto exn_fm = runtime->extract_scalar(result, idx, launch_domain); + runtime->record_pending_exception(runtime->reduce_exception_future_map(exn_fm)); + } } } @@ -217,7 +271,10 @@ void AutoTask::add_input(LogicalStore store, const Variable* partition_symbol) void AutoTask::add_output(LogicalStore store, const Variable* partition_symbol) { - if (store.impl()->has_scalar_storage()) scalar_outputs_.push_back(outputs_.size()); + if (store.impl()->has_scalar_storage()) + scalar_outputs_.push_back(outputs_.size()); + else if (store.impl()->unbound()) + unbound_outputs_.push_back(outputs_.size()); add_store(outputs_, store, partition_symbol); } @@ -273,7 +330,10 @@ void ManualTask::add_input(LogicalStore store) { add_store(inputs_, store, creat void ManualTask::add_output(LogicalStore store) { - if (store.impl()->has_scalar_storage()) scalar_outputs_.push_back(outputs_.size()); + if (store.impl()->has_scalar_storage()) + scalar_outputs_.push_back(outputs_.size()); + else if (store.impl()->unbound()) + unbound_outputs_.push_back(outputs_.size()); add_store(outputs_, store, create_no_partition()); } @@ -291,6 +351,9 @@ void ManualTask::add_input(LogicalStorePartition store_partition) void ManualTask::add_output(LogicalStorePartition store_partition) { +#ifdef DEBUG_LEGATE + assert(!store_partition.store().unbound()); +#endif if (store_partition.store().impl()->has_scalar_storage()) scalar_outputs_.push_back(outputs_.size()); add_store(outputs_, store_partition.store(), store_partition.partition()); diff --git a/src/core/runtime/operation.h b/src/core/runtime/operation.h index 28416e5202..98fe867eb5 100644 --- a/src/core/runtime/operation.h +++ b/src/core/runtime/operation.h @@ -34,6 +34,7 @@ namespace legate { +class CommunicatorFactory; class Constraint; class ConstraintSolver; class LibraryContext; @@ -129,22 +130,57 @@ class Task : public Operation { * @param scalar A scalar to add to the task */ void add_scalar_arg(const Scalar& scalar); + /** + * @brief Sets whether the task needs a concurrent task launch. + * + * Any task with at least one communicator will implicitly use concurrent task launch, so this + * method is to be used when the task needs a concurrent task launch for a reason unknown to + * Legate. + * + * @param concurrent A boolean value indicating whether the task needs a concurrent task launch + */ + void set_concurrent(bool concurrent); + /** + * @brief Sets whether the task has side effects or not. + * + * A task is assumed to be free of side effects by default if the task only has scalar arguments. + * + * @param has_side_effect A boolean value indicating whether the task has side effects + */ + void set_side_effect(bool has_side_effect); + /** + * @brief Sets whether the task can throw an exception or not. + * + * @param can_throw_exception A boolean value indicating whether the task can throw an exception + */ + void throws_exception(bool can_throw_exception); + /** + * @brief Requests a communicator for this task. + * + * @param name The name of the communicator to use for this task + */ + void add_communicator(const std::string& name); public: virtual void launch(Strategy* strategy) override; private: void demux_scalar_stores(const Legion::Future& result); - void demux_scalar_stores(const Legion::FutureMap& result, const Legion::Domain& launch_domain); + void demux_scalar_stores(const Legion::FutureMap& result, const Domain& launch_domain); public: std::string to_string() const override; protected: int64_t task_id_; + bool concurrent_{false}; + bool has_side_effect_{false}; + bool can_throw_exception_{false}; std::vector scalars_{}; + std::vector unbound_outputs_{}; std::vector scalar_outputs_{}; std::vector scalar_reductions_{}; + std::vector communicator_factories_{}; }; /** diff --git a/src/core/runtime/partition_manager.cc b/src/core/runtime/partition_manager.cc index bcc02105c3..be650a8e02 100644 --- a/src/core/runtime/partition_manager.cc +++ b/src/core/runtime/partition_manager.cc @@ -207,16 +207,34 @@ bool PartitionManager::use_complete_tiling(const Shape& extents, const Shape& ti return !(num_tiles > 256 && num_tiles > 16 * num_pieces); } -Legion::IndexPartition PartitionManager::find_index_partition(const Legion::IndexSpace& index_space, - const Tiling& tiling) const +namespace { + +template +Legion::IndexPartition _find_index_partition(const Cache& cache, + const Legion::IndexSpace& index_space, + const Partition& partition) { - auto finder = tiling_cache_.find(std::make_pair(index_space, tiling)); - if (finder != tiling_cache_.end()) + auto finder = cache.find(std::make_pair(index_space, partition)); + if (finder != cache.end()) return finder->second; else return Legion::IndexPartition::NO_PART; } +} // namespace + +Legion::IndexPartition PartitionManager::find_index_partition(const Legion::IndexSpace& index_space, + const Tiling& tiling) const +{ + return _find_index_partition(tiling_cache_, index_space, tiling); +} + +Legion::IndexPartition PartitionManager::find_index_partition(const Legion::IndexSpace& index_space, + const Weighted& weighted) const +{ + return _find_index_partition(weighted_cache_, index_space, weighted); +} + void PartitionManager::record_index_partition(const Legion::IndexSpace& index_space, const Tiling& tiling, const Legion::IndexPartition& index_partition) @@ -224,4 +242,11 @@ void PartitionManager::record_index_partition(const Legion::IndexSpace& index_sp tiling_cache_[std::make_pair(index_space, tiling)] = index_partition; } +void PartitionManager::record_index_partition(const Legion::IndexSpace& index_space, + const Weighted& weighted, + const Legion::IndexPartition& index_partition) +{ + weighted_cache_[std::make_pair(index_space, weighted)] = index_partition; +} + } // namespace legate diff --git a/src/core/runtime/partition_manager.h b/src/core/runtime/partition_manager.h index d02ad94e6a..8ba7620b85 100644 --- a/src/core/runtime/partition_manager.h +++ b/src/core/runtime/partition_manager.h @@ -28,6 +28,7 @@ namespace legate { class LibraryContext; class Runtime; class Tiling; +class Weighted; namespace mapping { @@ -52,9 +53,14 @@ class PartitionManager { public: Legion::IndexPartition find_index_partition(const Legion::IndexSpace& index_space, const Tiling& tiling) const; + Legion::IndexPartition find_index_partition(const Legion::IndexSpace& index_space, + const Weighted& weighted) const; void record_index_partition(const Legion::IndexSpace& index_space, const Tiling& tiling, const Legion::IndexPartition& index_partition); + void record_index_partition(const Legion::IndexSpace& index_space, + const Weighted& weighted, + const Legion::IndexPartition& index_partition); private: int64_t min_shard_volume_; @@ -63,6 +69,8 @@ class PartitionManager { private: using TilingCacheKey = std::pair; std::map tiling_cache_; + using WeightedCacheKey = std::pair; + std::map weighted_cache_; }; } // namespace legate diff --git a/src/core/runtime/req_analyzer.h b/src/core/runtime/req_analyzer.h index d2de3cc139..8acfb6a7c7 100644 --- a/src/core/runtime/req_analyzer.h +++ b/src/core/runtime/req_analyzer.h @@ -113,6 +113,7 @@ class RequirementAnalyzer { uint32_t get_requirement_index(const Legion::LogicalRegion& region, Legion::PrivilegeMode privilege, const ProjectionInfo* proj_info) const; + bool empty() const { return field_sets_.empty(); } public: void analyze_requirements(); @@ -131,6 +132,7 @@ class OutputRequirementAnalyzer { void insert(int32_t dim, const Legion::FieldSpace& field_space, Legion::FieldID field_id); uint32_t get_requirement_index(const Legion::FieldSpace& field_space, Legion::FieldID field_id) const; + bool empty() const { return field_groups_.empty(); } public: void analyze_requirements(); diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 51c04bb8d5..297e80e24d 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -26,6 +26,7 @@ #include "core/mapping/machine.h" #include "core/partitioning/partition.h" #include "core/partitioning/partitioner.h" +#include "core/runtime/communicator_manager.h" #include "core/runtime/context.h" #include "core/runtime/field_manager.h" #include "core/runtime/launcher.h" @@ -35,12 +36,13 @@ #include "core/runtime/provenance_manager.h" #include "core/runtime/region_manager.h" #include "core/runtime/shard.h" -#include "core/task/exception.h" +#include "core/task/return.h" #include "core/task/task.h" #include "core/type/type_info.h" #include "core/utilities/deserializer.h" #include "core/utilities/machine.h" #include "core/utilities/nvtx_help.h" +#include "env_defaults.h" #include "legate.h" namespace legate { @@ -315,13 +317,20 @@ constexpr uint32_t CUSTOM_TYPE_UID_BASE = 1000; } // namespace Runtime::Runtime(Legion::Runtime* legion_runtime) - : legion_runtime_(legion_runtime), next_type_uid_(CUSTOM_TYPE_UID_BASE) + : legion_runtime_(legion_runtime), + next_type_uid_(CUSTOM_TYPE_UID_BASE), + max_pending_exceptions_(extract_env( + "LEGATE_MAX_PENDING_EXCEPTIONS", MAX_PENDING_EXCEPTIONS_DEFAULT, MAX_PENDING_EXCEPTIONS_TEST)) { } Runtime::~Runtime() { for (auto& [_, context] : libraries_) delete context; + delete communicator_manager_; + delete machine_manager_; + delete partition_manager_; + delete provenance_manager_; } LibraryContext* Runtime::find_library(const std::string& library_name, @@ -396,13 +405,15 @@ bool Runtime::is_in_callback() const { return in_callback_; } void Runtime::post_startup_initialization(Legion::Context legion_context) { - legion_context_ = legion_context; - core_context_ = find_library(core_library_name); - partition_manager_ = new PartitionManager(this, core_context_); - machine_manager_ = new MachineManager(); - provenance_manager_ = new ProvenanceManager(); + legion_context_ = legion_context; + core_context_ = find_library(core_library_name); + communicator_manager_ = new CommunicatorManager(); + partition_manager_ = new PartitionManager(this, core_context_); + machine_manager_ = new MachineManager(); + provenance_manager_ = new ProvenanceManager(); Core::retrieve_tunable(legion_context_, legion_runtime_, core_context_); initialize_toplevel_machine(); + comm::register_builtin_communicator_factories(core_context_); } mapping::MachineDesc Runtime::slice_machine_for_task(LibraryContext* library, int64_t task_id) @@ -481,7 +492,12 @@ LogicalStore Runtime::create_store(std::unique_ptr type, int32_t dim) return LogicalStore(std::make_shared(std::move(storage))); } -LogicalStore Runtime::create_store(std::vector extents, +LogicalStore Runtime::create_store(const Type& type, int32_t dim) +{ + return create_store(type.clone(), dim); +} + +LogicalStore Runtime::create_store(const Shape& extents, std::unique_ptr type, bool optimize_scalar /*=false*/) { @@ -489,6 +505,13 @@ LogicalStore Runtime::create_store(std::vector extents, return LogicalStore(std::make_shared(std::move(storage))); } +LogicalStore Runtime::create_store(const Shape& extents, + const Type& type, + bool optimize_scalar /*=false*/) +{ + return create_store(extents, type.clone(), optimize_scalar); +} + LogicalStore Runtime::create_store(const Scalar& scalar) { Shape extents{1}; @@ -497,6 +520,47 @@ LogicalStore Runtime::create_store(const Scalar& scalar) return LogicalStore(std::make_shared(std::move(storage))); } +uint32_t Runtime::max_pending_exceptions() const { return max_pending_exceptions_; } + +void Runtime::set_max_pending_exceptions(uint32_t max_pending_exceptions) +{ + uint32_t old_value = max_pending_exceptions_; + max_pending_exceptions_ = max_pending_exceptions; + if (old_value > max_pending_exceptions_) raise_pending_task_exception(); +} + +void Runtime::raise_pending_task_exception() +{ + auto exn = check_pending_task_exception(); + if (exn.has_value()) throw exn.value(); +} + +std::optional Runtime::check_pending_task_exception() +{ + // If there's already an outstanding exception from the previous scan, we just return that. + if (!outstanding_exceptions_.empty()) { + std::optional result = outstanding_exceptions_.front(); + outstanding_exceptions_.pop_front(); + return result; + } + + // Othrewise, we unpack all pending exceptions and push them to the outstanding exception queue + for (auto& pending_exception : pending_exceptions_) { + auto returned_exception = pending_exception.get_result(); + auto result = returned_exception.to_task_exception(); + if (result.has_value()) outstanding_exceptions_.push_back(result.value()); + } + pending_exceptions_.clear(); + return outstanding_exceptions_.empty() ? std::nullopt : check_pending_task_exception(); +} + +void Runtime::record_pending_exception(const Legion::Future& pending_exception) +{ + pending_exceptions_.push_back(pending_exception); + if (outstanding_exceptions_.size() + pending_exceptions_.size() >= max_pending_exceptions_) + raise_pending_task_exception(); +} + uint64_t Runtime::get_unique_store_id() { return next_store_id_++; } uint64_t Runtime::get_unique_storage_id() { return next_storage_id_++; } @@ -607,6 +671,14 @@ Legion::IndexPartition Runtime::create_restricted_partition( legion_context_, index_space, color_space, transform, extent, kind); } +Legion::IndexPartition Runtime::create_weighted_partition(const Legion::IndexSpace& index_space, + const Legion::IndexSpace& color_space, + const Legion::FutureMap& weights) +{ + return legion_runtime_->create_partition_by_weights( + legion_context_, index_space, weights, color_space); +} + Legion::FieldSpace Runtime::create_field_space() { assert(nullptr != legion_context_); @@ -671,6 +743,47 @@ Domain Runtime::get_index_space_domain(const Legion::IndexSpace& index_space) co return legion_runtime_->get_index_space_domain(legion_context_, index_space); } +namespace { + +Legion::DomainPoint _delinearize_future_map(const DomainPoint& point, + const Domain& domain, + const Domain& range) +{ + assert(range.dim == 1); + DomainPoint result; + result.dim = 1; + + int32_t ndim = domain.dim; + int64_t idx = point[0]; + for (int32_t dim = 1; dim < ndim; ++dim) { + int64_t extent = domain.rect_data[dim + ndim] - domain.rect_data[dim] + 1; + idx = idx * extent + point[dim]; + } + result[0] = idx; + return result; +} + +} // namespace + +Legion::FutureMap Runtime::delinearize_future_map(const Legion::FutureMap& future_map, + const Legion::IndexSpace& new_domain) const +{ + return legion_runtime_->transform_future_map( + legion_context_, future_map, new_domain, _delinearize_future_map); +} + +std::pair Runtime::create_barriers(size_t num_tasks) +{ + auto arrival_barrier = legion_runtime_->create_phase_barrier(legion_context_, num_tasks); + auto wait_barrier = legion_runtime_->advance_phase_barrier(legion_context_, arrival_barrier); + return std::make_pair(arrival_barrier, wait_barrier); +} + +void Runtime::destroy_barrier(Legion::PhaseBarrier barrier) +{ + legion_runtime_->destroy_phase_barrier(legion_context_, barrier); +} + Legion::Future Runtime::dispatch(Legion::TaskLauncher* launcher, std::vector* output_requirements) { @@ -721,6 +834,17 @@ Legion::Future Runtime::reduce_future_map(const Legion::FutureMap& future_map, core_context_->get_mapper_id()); } +Legion::Future Runtime::reduce_exception_future_map(const Legion::FutureMap& future_map) const +{ + auto reduction_op = core_context_->get_reduction_op_id(LEGATE_CORE_JOIN_EXCEPTION_OP); + return legion_runtime_->reduce_future_map(legion_context_, + future_map, + reduction_op, + false /*deterministic*/, + core_context_->get_mapper_id(), + LEGATE_CORE_JOIN_EXCEPTION_TAG); +} + void Runtime::issue_execution_fence(bool block /*=false*/) { flush_scheduling_window(); @@ -807,6 +931,8 @@ Legion::ProjectionID Runtime::get_delinearizing_projection() return core_context_->get_projection_id(LEGATE_CORE_DELINEARIZE_PROJ_ID); } +CommunicatorManager* Runtime::communicator_manager() const { return communicator_manager_; } + MachineManager* Runtime::machine_manager() const { return machine_manager_; } /*static*/ void Runtime::initialize(int32_t argc, char** argv) diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index cac333a174..ff2a2b2d93 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -17,6 +17,7 @@ #pragma once #include +#include #include "legion.h" @@ -27,6 +28,7 @@ #include "core/legate_c.h" #include "core/mapping/machine.h" #include "core/runtime/resource.h" +#include "core/task/exception.h" #include "core/type/type_info.h" #include "core/utilities/typedefs.h" @@ -37,7 +39,6 @@ namespace legate { class LibraryContext; class Scalar; -class TaskException; namespace mapping { @@ -90,6 +91,7 @@ struct Core { }; class AutoTask; +class CommunicatorManager; class FieldManager; class LogicalRegionField; class LogicalStore; @@ -213,6 +215,15 @@ class Runtime { * @return Logical store */ LogicalStore create_store(std::unique_ptr type, int32_t dim = 1); + /** + * @brief Creates an unbound store + * + * @param type Element type + * @param dim Number of dimensions of the store + * + * @return Logical store + */ + LogicalStore create_store(const Type& type, int32_t dim = 1); /** * @brief Creates a normal store * @@ -223,9 +234,20 @@ class Runtime { * * @return Logical store */ - LogicalStore create_store(std::vector extents, + LogicalStore create_store(const Shape& extents, std::unique_ptr type, bool optimize_scalar = false); + /** + * @brief Creates a normal store + * + * @param extents Shape of the store + * @param type Element type + * @param optimize_scalar When true, the runtime internally uses futures optimized for storing + * scalars + * + * @return Logical store + */ + LogicalStore create_store(const Shape& extents, const Type& type, bool optimize_scalar = false); /** * @brief Creates a normal store out of a `Scalar` object * @@ -234,6 +256,34 @@ class Runtime { * @return Logical store */ LogicalStore create_store(const Scalar& scalar); + + public: + /** + * @brief Returns the maximum number of pending exceptions + * + * @return Maximum number of pending exceptions + */ + uint32_t max_pending_exceptions() const; + /** + * @brief Updates the maximum number of pending exceptions + * + * If the new maximum number of pending exceptions is smaller than the previous value, + * `raise_pending_task_exception` will be invoked. + * + * @param max_pending_exceptions A new maximum number of pending exceptions + */ + void set_max_pending_exceptions(uint32_t max_pending_exceptions); + /** + * @brief Inspects all pending exceptions and immediately raises the first one if there exists any + */ + void raise_pending_task_exception(); + /** + * @brief Returns the first pending exception. + */ + std::optional check_pending_task_exception(); + void record_pending_exception(const Legion::Future& pending_exception); + + public: uint64_t get_unique_store_id(); uint64_t get_unique_storage_id(); @@ -249,6 +299,7 @@ class Runtime { public: RegionManager* find_or_create_region_manager(const Legion::Domain& shape); FieldManager* find_or_create_field_manager(const Legion::Domain& shape, uint32_t field_size); + CommunicatorManager* communicator_manager() const; MachineManager* machine_manager() const; PartitionManager* partition_manager() const; ProvenanceManager* provenance_manager() const; @@ -260,6 +311,9 @@ class Runtime { Legion::PartitionKind kind, const Legion::DomainTransform& transform, const Legion::Domain& extent); + Legion::IndexPartition create_weighted_partition(const Legion::IndexSpace& index_space, + const Legion::IndexSpace& color_space, + const Legion::FutureMap& weights); Legion::FieldSpace create_field_space(); Legion::LogicalRegion create_region(const Legion::IndexSpace& index_space, const Legion::FieldSpace& field_space); @@ -274,6 +328,10 @@ class Runtime { Legion::FieldID field_id, size_t field_size); Legion::Domain get_index_space_domain(const Legion::IndexSpace& index_space) const; + Legion::FutureMap delinearize_future_map(const Legion::FutureMap& future_map, + const Legion::IndexSpace& new_domain) const; + std::pair create_barriers(size_t num_tasks); + void destroy_barrier(Legion::PhaseBarrier barrier); public: Legion::Future dispatch(Legion::TaskLauncher* launcher, @@ -287,6 +345,7 @@ class Runtime { uint32_t idx, const Legion::Domain& launch_domain) const; Legion::Future reduce_future_map(const Legion::FutureMap& future_map, int32_t reduction_op) const; + Legion::Future reduce_exception_future_map(const Legion::FutureMap& future_map) const; public: /** @@ -342,6 +401,7 @@ class Runtime { using FieldManagerKey = std::pair; std::map field_managers_; std::map region_managers_; + CommunicatorManager* communicator_manager_{nullptr}; MachineManager* machine_manager_{nullptr}; PartitionManager* partition_manager_{nullptr}; ProvenanceManager* provenance_manager_{nullptr}; @@ -374,6 +434,11 @@ class Runtime { private: uint32_t next_type_uid_; std::map, int32_t> reduction_ops_{}; + + private: + uint32_t max_pending_exceptions_; + std::vector pending_exceptions_{}; + std::deque outstanding_exceptions_{}; }; /** diff --git a/src/core/task/return.cc b/src/core/task/return.cc index f6e464c110..4b68c3b66d 100644 --- a/src/core/task/return.cc +++ b/src/core/task/return.cc @@ -131,6 +131,14 @@ ReturnedException::ReturnedException(int32_t index, const std::string& error_mes { } +std::optional ReturnedException::to_task_exception() const +{ + if (!raised_) + return std::nullopt; + else + return TaskException(index_, error_message_); +} + size_t ReturnedException::legion_buffer_size() const { size_t size = sizeof(bool); diff --git a/src/core/task/return.h b/src/core/task/return.h index 6b7b9935fa..ae8bb1c7a8 100644 --- a/src/core/task/return.h +++ b/src/core/task/return.h @@ -16,7 +16,9 @@ #pragma once +#include #include +#include "core/task/exception.h" #include "core/utilities/typedefs.h" namespace legate { @@ -56,6 +58,9 @@ struct ReturnedException { public: bool raised() const { return raised_; } + public: + std::optional to_task_exception() const; + public: size_t legion_buffer_size() const; void legion_serialize(void* buffer) const; diff --git a/tests/cpp/build.sh b/tests/cpp/build.sh index 789f587fc1..cb953ded04 100755 --- a/tests/cpp/build.sh +++ b/tests/cpp/build.sh @@ -1,5 +1,7 @@ #!/bin/bash +legate_root=`python -c 'import legate.install_info as i; from pathlib import Path; print(Path(i.libpath).parent.resolve())'` +echo "Using Legate at $legate_root" rm -rf build -cmake -B build -S . -D legate_core_ROOT=../../build -D CMAKE_BUILD_TYPE=Debug +cmake -B build -S . -D legate_core_ROOT="$legate_root" -D CMAKE_BUILD_TYPE=Debug cmake --build build -j 8 diff --git a/tests/cpp/integration/cpu_communicator.cc b/tests/cpp/integration/cpu_communicator.cc new file mode 100644 index 0000000000..445a93497e --- /dev/null +++ b/tests/cpp/integration/cpu_communicator.cc @@ -0,0 +1,106 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "core/comm/coll.h" +#include "legate.h" + +namespace cpu_communicator { + +namespace { + +const char* library_name = "test_cpu_communicator"; + +enum TaskIDs { + CPU_COMM_TESTER = 0, +}; + +constexpr size_t SIZE = 10; + +struct CPUCommunicatorTester : public legate::LegateTask { + static void cpu_variant(legate::TaskContext& context) + { + EXPECT_TRUE((context.is_single_task() && context.communicators().empty()) || + context.communicators().size() == 1); + if (context.is_single_task()) return; + auto comm = context.communicators().at(0).get(); + + int64_t value = 12345; + size_t num_tasks = context.get_launch_domain().get_volume(); + std::vector recv_buffer(num_tasks, 0); + auto result = collAllgather( + &value, recv_buffer.data(), 1, legate::comm::coll::CollDataType::CollInt64, comm); + EXPECT_EQ(result, legate::comm::coll::CollSuccess); + for (auto v : recv_buffer) EXPECT_EQ(v, 12345); + } +}; + +void prepare() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + CPUCommunicatorTester::register_variants(context, CPU_COMM_TESTER); +} + +void test_cpu_communicator_auto(int32_t ndim) +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + auto store = runtime->create_store(std::vector(ndim, SIZE), legate::int32()); + + auto task = runtime->create_task(context, CPU_COMM_TESTER); + auto part = task->declare_partition(); + task->add_output(store, part); + task->add_communicator("cpu"); + runtime->submit(std::move(task)); +} + +void test_cpu_communicator_manual(int32_t ndim) +{ + auto runtime = legate::Runtime::get_runtime(); + size_t num_procs = runtime->get_machine().count(); + if (num_procs <= 1) return; + + auto context = runtime->find_library(library_name); + auto store = runtime->create_store(std::vector(ndim, SIZE), legate::int32()); + std::vector launch_shape(ndim, 1); + std::vector tile_shape(ndim, 1); + launch_shape[0] = num_procs; + tile_shape[0] = (SIZE + num_procs - 1) / num_procs; + + auto part = store.partition_by_tiling(std::move(tile_shape)); + + auto task = runtime->create_task(context, CPU_COMM_TESTER, std::move(launch_shape)); + task->add_output(part); + task->add_communicator("cpu"); + runtime->submit(std::move(task)); +} + +} // namespace + +// Test case with single unbound store +TEST(Integration, CPUCommunicator) +{ + legate::Core::perform_registration(); + + for (int32_t ndim : {1, 3}) { + test_cpu_communicator_auto(ndim); + test_cpu_communicator_manual(ndim); + } +} + +} // namespace cpu_communicator diff --git a/tests/cpp/integration/exception.cc b/tests/cpp/integration/exception.cc new file mode 100644 index 0000000000..46c9bd447f --- /dev/null +++ b/tests/cpp/integration/exception.cc @@ -0,0 +1,183 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "legate.h" + +namespace exception { + +namespace { + +const char* library_name = "test_exception"; + +static legate::Logger logger(library_name); + +enum TaskIDs { + EXCEPTION_TASK = 0, +}; + +const char* EXN_MSG = "Exception Test"; +constexpr size_t SIZE = 10; + +struct ExceptionTask : public legate::LegateTask { + static void cpu_variant(legate::TaskContext& context) + { + auto raise = context.scalars().at(0).value(); + auto index = context.scalars().at(1).value(); + // Make sure only some of the point tasks raise an exception + if (raise && (context.is_single_task() || context.get_task_index()[0] == 0)) { + if (context.is_single_task()) + logger.debug() << "Raise an exception for index " << index; + else + logger.debug() << "Raise an exception for index " << index << " (task " + << context.get_task_index() << ")"; + throw legate::TaskException(index, EXN_MSG); + } else { + if (context.is_single_task()) + logger.debug() << "Don't raise an exception for index " << index; + else + logger.debug() << "Don't raise an exception for index " << index << " (task " + << context.get_task_index() << ")"; + ; + } + } +}; + +void prepare() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + ExceptionTask::register_variants(context, EXCEPTION_TASK); +} + +void test_single() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + auto issue_task = [&](int32_t index, bool raise) { + auto task = runtime->create_task(context, EXCEPTION_TASK); + task->add_scalar_arg(legate::Scalar(raise)); + task->add_scalar_arg(legate::Scalar(index)); + task->throws_exception(true); + runtime->submit(std::move(task)); + }; + + // Test the immediate exception handling + runtime->set_max_pending_exceptions(1); + EXPECT_THROW(issue_task(0, true), legate::TaskException); + + // Increase the window size to test more interesting scenarios + runtime->set_max_pending_exceptions(4); + + // These three tasks shouldn't fill upthe window + issue_task(1, true); + issue_task(2, false); + issue_task(3, true); + + auto check_exn = [&](int32_t index) { + auto exn = runtime->check_pending_task_exception(); + EXPECT_TRUE(exn.has_value()); + EXPECT_EQ(exn.value().index(), index); + EXPECT_EQ(exn.value().error_message(), EXN_MSG); + }; + // At this point, there are three pending exceptions and no outstanding exceptions + // Querying the first one filters out empty ones and turns the remaining ones into outstanding + // exceptions + check_exn(1); + // With this task, there will be one pending exception and one outstanding exceptions + issue_task(4, true); + // Querying the next puts the state back to one outstanding exception and no pending exception + check_exn(3); + issue_task(5, false); + issue_task(6, false); + // This task should hit the max pending exception limit and raise the exception + EXPECT_THROW(issue_task(7, false), legate::TaskException); + + // At this point there's no outstanding or pending exception + // So the query should return a null value + EXPECT_FALSE(runtime->check_pending_task_exception().has_value()); + // And it should be idempotent + EXPECT_FALSE(runtime->check_pending_task_exception().has_value()); + + issue_task(7, true); + issue_task(8, false); + // Immediate check should raise the exception + EXPECT_THROW(runtime->raise_pending_task_exception(), legate::TaskException); + // Then nothing should happen after that and the operation should be idempotent + runtime->raise_pending_task_exception(); + runtime->raise_pending_task_exception(); + + issue_task(6, true); + // Finally crank the maximum number of pending exceptions down to 1 will flush the window + EXPECT_THROW(runtime->set_max_pending_exceptions(1), legate::TaskException); +} + +void test_multi(bool use_auto_task) +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + // Turn off immediate exception raise + runtime->set_max_pending_exceptions(2); + + auto store = runtime->create_store({SIZE, SIZE}, legate::int64()); + if (use_auto_task) { + auto task = runtime->create_task(context, EXCEPTION_TASK); + auto part = task->declare_partition(); + // Dummy store argument to trigger parallelization + task->add_output(store, part); + task->throws_exception(true); + task->add_scalar_arg(legate::Scalar(true)); + task->add_scalar_arg(legate::Scalar(12345)); + + runtime->submit(std::move(task)); + } else { + auto task = runtime->create_task(context, EXCEPTION_TASK, {2, 2}); + auto part = store.partition_by_tiling({SIZE / 2, SIZE / 2}); + // Dummy store argument to trigger parallelization + task->add_output(part); + task->throws_exception(true); + task->add_scalar_arg(legate::Scalar(true)); + task->add_scalar_arg(legate::Scalar(12345)); + runtime->submit(std::move(task)); + } + + auto exn = runtime->check_pending_task_exception(); + EXPECT_TRUE(exn.has_value()); + EXPECT_EQ(exn.value().index(), 12345); + EXPECT_EQ(exn.value().error_message(), EXN_MSG); +} + +} // namespace + +TEST(Integration, ExceptionSingle) +{ + legate::Core::perform_registration(); + + test_single(); +} + +TEST(Integration, ExceptionMulti) +{ + legate::Core::perform_registration(); + + test_multi(true /* use_auto_task */); + test_multi(false /* use_auto_task */); +} + +} // namespace exception diff --git a/tests/cpp/integration/weighted.cc b/tests/cpp/integration/weighted.cc new file mode 100644 index 0000000000..2b7889e2e4 --- /dev/null +++ b/tests/cpp/integration/weighted.cc @@ -0,0 +1,137 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "legate.h" + +namespace weighted { + +namespace { + +const char* library_name = "test_weighted"; + +enum TaskIDs { + INIT = 0, + CHECK = 3, +}; + +constexpr uint32_t NUM_TASKS = 4; + +struct Initializer : public legate::LegateTask { + static void cpu_variant(legate::TaskContext& context) + { + auto task_idx = context.get_task_index()[0]; + auto& outputs = context.outputs(); + for (uint32_t idx = 0; idx < outputs.size(); ++idx) { + auto& output = outputs.at(idx); + output.create_output_buffer(legate::Point<1>(task_idx + 10 * (idx + 1)), true); + } + } +}; + +struct Tester : public legate::LegateTask { + static void cpu_variant(legate::TaskContext& context) + { + EXPECT_FALSE(context.is_single_task()); + EXPECT_EQ(context.get_launch_domain().get_volume(), NUM_TASKS); + auto task_idx = context.get_task_index()[0]; + auto& outputs = context.outputs(); + for (uint32_t idx = 0; idx < outputs.size(); ++idx) { + auto volume = outputs.at(idx).shape<1>().volume(); + EXPECT_EQ(volume, task_idx + 10 * (idx + 1)); + } + } +}; + +void prepare() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + Initializer::register_variants(context, INIT); + Tester::register_variants(context, CHECK); +} + +void initialize(legate::Runtime* runtime, + legate::LibraryContext* context, + const std::vector& outputs) +{ + auto task = runtime->create_task(context, INIT, {NUM_TASKS}); + + std::vector parts; + for (auto& output : outputs) { + auto part = task->declare_partition(); + task->add_output(output); + } + + runtime->submit(std::move(task)); +} + +void check(legate::Runtime* runtime, + legate::LibraryContext* context, + const std::vector& inputs) +{ + auto task = runtime->create_task(context, CHECK); + + for (auto& input : inputs) { + auto part_in = task->declare_partition(); + auto output = runtime->create_store(input.extents(), input.type()); + auto part_out = task->declare_partition(); + task->add_input(input, part_in); + task->add_output(output, part_out); + task->add_constraint(legate::align(part_in, part_out)); + } + + runtime->submit(std::move(task)); +} + +void test_weighted(uint32_t num_stores) +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + std::vector stores; + for (uint32_t idx = 0; idx < num_stores; ++idx) + stores.push_back(runtime->create_store(legate::int32())); + initialize(runtime, context, stores); + check(runtime, context, stores); +} + +} // namespace + +// Test case with single unbound store +TEST(Integration, WeightedSingle) +{ + legate::Core::perform_registration(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + test_weighted(1); +} + +// Test case with multiple unbound stores +TEST(Integration, WeightedMultiple) +{ + legate::Core::perform_registration(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + test_weighted(3); +} + +} // namespace weighted From 566f19d1b636a4a16acbb0f7d47c1ab920b42a43 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Thu, 1 Jun 2023 22:25:09 -0700 Subject: [PATCH 0134/1425] Communicator clean-up code + integration test for NCCL * Integration test for NCCL * Add logic to destroy communicator factories during shutdown See merge request legate/legate.core.internal!44 --- src/core/comm/comm_cpu.cc | 5 +- src/core/runtime/communicator_manager.cc | 5 + src/core/runtime/communicator_manager.h | 3 + src/core/runtime/launcher.cc | 2 + src/core/runtime/runtime.cc | 8 ++ src/core/runtime/runtime.h | 1 + tests/cpp/CMakeLists.txt | 21 ++++ tests/cpp/integration/nccl.cu | 125 +++++++++++++++++++++++ 8 files changed, 168 insertions(+), 2 deletions(-) create mode 100644 tests/cpp/integration/nccl.cu diff --git a/src/core/comm/comm_cpu.cc b/src/core/comm/comm_cpu.cc index 4034071da9..044d42f139 100644 --- a/src/core/comm/comm_cpu.cc +++ b/src/core/comm/comm_cpu.cc @@ -81,9 +81,10 @@ void Factory::finalize(const mapping::MachineDesc& machine, uint32_t num_tasks, const Legion::FutureMap& communicator) { + auto tag = + machine.preferred_target == mapping::TaskTarget::OMP ? LEGATE_OMP_VARIANT : LEGATE_CPU_VARIANT; Domain launch_domain(Rect<1>(Point<1>(0), Point<1>(static_cast(num_tasks) - 1))); - TaskLauncher launcher( - core_context_, machine, LEGATE_CORE_FINALIZE_NCCL_TASK_ID, LEGATE_GPU_VARIANT); + TaskLauncher launcher(core_context_, machine, LEGATE_CORE_FINALIZE_CPUCOLL_TASK_ID, tag); launcher.set_concurrent(true); launcher.add_future_map(communicator); launcher.execute(launch_domain); diff --git a/src/core/runtime/communicator_manager.cc b/src/core/runtime/communicator_manager.cc index 282118ae56..1ca86b8f6f 100644 --- a/src/core/runtime/communicator_manager.cc +++ b/src/core/runtime/communicator_manager.cc @@ -79,4 +79,9 @@ void CommunicatorManager::register_factory(const std::string& name, factories_.insert({name, std::move(factory)}); } +void CommunicatorManager::destroy() +{ + for (auto& [_, factory] : factories_) factory->destroy(); +} + } // namespace legate diff --git a/src/core/runtime/communicator_manager.h b/src/core/runtime/communicator_manager.h index 18344d520c..76c2c75ace 100644 --- a/src/core/runtime/communicator_manager.h +++ b/src/core/runtime/communicator_manager.h @@ -88,6 +88,9 @@ class CommunicatorManager { CommunicatorFactory* find_factory(const std::string& name); void register_factory(const std::string& name, std::unique_ptr factory); + public: + void destroy(); + private: std::unordered_map> factories_; }; diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/launcher.cc index 05e76e886f..9a07a7a363 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/launcher.cc @@ -266,6 +266,8 @@ std::unique_ptr TaskLauncher::build_index_task( auto [arrival_barrier, wait_barrier] = runtime->create_barriers(num_tasks); index_task->add_future(Legion::Future::from_value(arrival_barrier)); index_task->add_future(Legion::Future::from_value(wait_barrier)); + runtime->destroy_barrier(arrival_barrier); + runtime->destroy_barrier(wait_barrier); } for (auto& communicator : communicators_) index_task->point_futures.push_back(communicator); for (auto& future_map : future_maps_) index_task->point_futures.push_back(future_map); diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 297e80e24d..05e94502cf 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -966,6 +966,8 @@ MachineManager* Runtime::machine_manager() const { return machine_manager_; } int32_t Runtime::wait_for_shutdown() { + destroy(); + // Mark that we are done excecuting the top-level task // After this call the context is no longer valid Legion::Runtime::get_runtime()->finish_implicit_task(legion_context_); @@ -982,6 +984,12 @@ int32_t Runtime::wait_for_shutdown() runtime_ = new Runtime(legion_runtime); } +void Runtime::destroy() +{ + issue_execution_fence(); + communicator_manager_->destroy(); +} + void initialize(int32_t argc, char** argv) { Runtime::initialize(argc, argv); } int32_t start(int32_t argc, char** argv) { return Runtime::start(argc, argv); } diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index ff2a2b2d93..e0b3d89ccf 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -387,6 +387,7 @@ class Runtime { */ static Runtime* get_runtime(); static void create_runtime(Legion::Runtime* legion_runtime); + void destroy(); int32_t wait_for_shutdown(); private: diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt index ef89bf6960..4da5a893db 100644 --- a/tests/cpp/CMakeLists.txt +++ b/tests/cpp/CMakeLists.txt @@ -40,9 +40,30 @@ file(GLOB integration_SRC ${PROJECT_SOURCE_DIR}/integration/*.cc) file(GLOB tasks_SRC ${PROJECT_SOURCE_DIR}/integration/tasks/*.cc) file(GLOB unit_SRC ${PROJECT_SOURCE_DIR}/unit/*.cc) +if(Legion_USE_CUDA) + +find_package(CUDAToolkit REQUIRED) +enable_language(CUDA) + +file(GLOB integration_GPU_SRC ${PROJECT_SOURCE_DIR}/integration/*.cu) +list(APPEND integration_SRC ${integration_GPU_SRC}) + +if(NOT EXISTS ${CMAKE_BINARY_DIR}/RAPIDS.cmake) + file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-23.04/RAPIDS.cmake + ${CMAKE_BINARY_DIR}/RAPIDS.cmake) +endif() +include(${CMAKE_BINARY_DIR}/RAPIDS.cmake) +include(rapids-find) +include(cmake/thirdparty/get_nccl.cmake) + +endif() + add_executable(cpp_tests ${main_SRC} ${tasks_SRC} ${integration_SRC} ${unit_SRC}) target_link_libraries(cpp_tests PRIVATE legate::core PRIVATE GTest::gtest) +if(Legion_USE_CUDA) +target_link_libraries(cpp_tests PRIVATE NCCL::NCCL) +endif() include(GoogleTest) gtest_discover_tests(cpp_tests) diff --git a/tests/cpp/integration/nccl.cu b/tests/cpp/integration/nccl.cu new file mode 100644 index 0000000000..c15b6d3fae --- /dev/null +++ b/tests/cpp/integration/nccl.cu @@ -0,0 +1,125 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "core/cuda/cuda_help.h" +#include "core/cuda/stream_pool.h" +#include "legate.h" + +#include + +namespace nccl { + +namespace { + +const char* library_name = "test_nccl"; + +enum TaskIDs { + NCCL_TESTER = 0, +}; + +constexpr size_t SIZE = 10; + +struct NCCLTester : public legate::LegateTask { + static void gpu_variant(legate::TaskContext& context) + { + EXPECT_TRUE((context.is_single_task() && context.communicators().empty()) || + context.communicators().size() == 1); + if (context.is_single_task()) return; + auto comm = context.communicators().at(0).get(); + + size_t num_tasks = context.get_launch_domain().get_volume(); + + auto recv_buffer = legate::create_buffer(num_tasks, legate::Memory::Kind::Z_COPY_MEM); + auto send_buffer = legate::create_buffer(1, legate::Memory::Kind::Z_COPY_MEM); + + auto* p_recv = recv_buffer.ptr(0); + auto* p_send = send_buffer.ptr(0); + + for (uint32_t idx = 0; idx < num_tasks; ++idx) p_recv[idx] = 0; + *p_send = 12345; + + auto stream = legate::cuda::StreamPool::get_stream_pool().get_stream(); + auto result = ncclAllGather(p_send, p_recv, 1, ncclUint64, *comm, stream); + EXPECT_EQ(result, ncclSuccess); + CHECK_CUDA(cudaStreamSynchronize(stream)); + for (uint32_t idx = 0; idx < num_tasks; ++idx) EXPECT_EQ(p_recv[idx], 12345); + } +}; + +void prepare() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + NCCLTester::register_variants(context, NCCL_TESTER); +} + +void test_nccl_auto(int32_t ndim) +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + auto store = runtime->create_store(std::vector(ndim, SIZE), legate::int32()); + + auto task = runtime->create_task(context, NCCL_TESTER); + auto part = task->declare_partition(); + task->add_output(store, part); + task->add_communicator("cpu"); // This requested will be ignored + task->add_communicator("nccl"); + runtime->submit(std::move(task)); +} + +void test_nccl_manual(int32_t ndim) +{ + auto runtime = legate::Runtime::get_runtime(); + size_t num_procs = runtime->get_machine().count(); + if (num_procs <= 1) return; + + auto context = runtime->find_library(library_name); + auto store = runtime->create_store(std::vector(ndim, SIZE), legate::int32()); + std::vector launch_shape(ndim, 1); + std::vector tile_shape(ndim, 1); + launch_shape[0] = num_procs; + tile_shape[0] = (SIZE + num_procs - 1) / num_procs; + + auto part = store.partition_by_tiling(std::move(tile_shape)); + + auto task = runtime->create_task(context, NCCL_TESTER, std::move(launch_shape)); + task->add_output(part); + task->add_communicator("cpu"); // This requested will be ignored + task->add_communicator("nccl"); + runtime->submit(std::move(task)); +} + +} // namespace + +// Test case with single unbound store +TEST(Integration, NCCL) +{ + legate::Core::perform_registration(); + + auto runtime = legate::Runtime::get_runtime(); + auto& machine = runtime->get_machine(); + if (machine.count(legate::mapping::TaskTarget::GPU) == 0) return; + legate::MachineTracker tracker(machine.only(legate::mapping::TaskTarget::GPU)); + + for (int32_t ndim : {1, 3}) { + test_nccl_auto(ndim); + test_nccl_manual(ndim); + } +} + +} // namespace nccl From 8daada71db280a85c12858e7ec4a21042cec957c Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Thu, 1 Jun 2023 23:23:01 -0700 Subject: [PATCH 0135/1425] Minor clean-up for integration tests * Minor clean-up for integration tests See merge request legate/legate.core.internal!45 --- tests/cpp/integration/alignment_constraints.cc | 3 --- tests/cpp/integration/broadcast_constraints.cc | 3 --- tests/cpp/integration/weighted.cc | 6 ------ 3 files changed, 12 deletions(-) diff --git a/tests/cpp/integration/alignment_constraints.cc b/tests/cpp/integration/alignment_constraints.cc index 014d357e64..e5ad231ed6 100644 --- a/tests/cpp/integration/alignment_constraints.cc +++ b/tests/cpp/integration/alignment_constraints.cc @@ -207,9 +207,6 @@ TEST(Integration, AlignmentConstraints) { legate::Core::perform_registration(); - auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library(library_name); - test_alignment(); test_alignment_and_broadcast(); test_alignment_transformed(); diff --git a/tests/cpp/integration/broadcast_constraints.cc b/tests/cpp/integration/broadcast_constraints.cc index 0cf1ce3109..3dcb929c3e 100644 --- a/tests/cpp/integration/broadcast_constraints.cc +++ b/tests/cpp/integration/broadcast_constraints.cc @@ -123,9 +123,6 @@ TEST(Integration, BroadcastConstraints) { legate::Core::perform_registration(); - auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library(library_name); - test_normal_store(); test_promoted_store(); } diff --git a/tests/cpp/integration/weighted.cc b/tests/cpp/integration/weighted.cc index 2b7889e2e4..1916914f51 100644 --- a/tests/cpp/integration/weighted.cc +++ b/tests/cpp/integration/weighted.cc @@ -117,9 +117,6 @@ TEST(Integration, WeightedSingle) { legate::Core::perform_registration(); - auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library(library_name); - test_weighted(1); } @@ -128,9 +125,6 @@ TEST(Integration, WeightedMultiple) { legate::Core::perform_registration(); - auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library(library_name); - test_weighted(3); } From eb68f6a7f96421aeb80dafaa0a2b300fb6ecfa50 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 2 Jun 2023 11:23:48 -0700 Subject: [PATCH 0136/1425] Start using nested namespaces * Start using nested namespaces See merge request legate/legate.core.internal!46 --- src/core/comm/coll.cc | 8 ++------ src/core/comm/coll.h | 8 ++------ src/core/comm/comm.cc | 6 ++---- src/core/comm/comm.h | 7 +++---- src/core/comm/comm_cpu.cc | 8 ++------ src/core/comm/comm_cpu.h | 9 +++------ src/core/comm/comm_nccl.cu | 8 ++------ src/core/comm/comm_nccl.h | 9 +++------ src/core/comm/communicator.h | 6 ++---- src/core/comm/local_comm.cc | 8 ++------ src/core/comm/mpi_comm.cc | 8 ++------ src/core/cuda/cuda_help.h | 6 ++---- src/core/cuda/stream_pool.cu | 6 ++---- src/core/cuda/stream_pool.h | 6 ++---- src/core/data/logical_store.h | 22 +++++++++------------- src/core/data/logical_store_detail.cc | 7 +++---- src/core/data/logical_store_detail.h | 11 ++++------- src/core/mapping/base_mapper.cc | 6 ++---- src/core/mapping/base_mapper.h | 6 ++---- src/core/mapping/core_mapper.cc | 8 ++++++-- src/core/mapping/default_mapper.cc | 6 ++---- src/core/mapping/default_mapper.h | 6 ++---- src/core/mapping/instance_manager.cc | 6 ++---- src/core/mapping/instance_manager.h | 6 ++---- src/core/mapping/machine.cc | 6 ++---- src/core/mapping/machine.h | 6 ++---- src/core/mapping/mapping.cc | 6 ++---- src/core/mapping/mapping.h | 6 ++---- src/core/mapping/operation.cc | 6 ++---- src/core/mapping/operation.h | 7 +++---- src/core/mapping/operation.inl | 6 ++---- src/core/mapping/store.cc | 6 ++---- src/core/mapping/store.h | 6 ++---- src/core/partitioning/partition.h | 12 +++++------- src/core/runtime/context.h | 8 +++----- src/core/runtime/launcher.h | 10 ++++------ src/core/runtime/machine_manager.h | 8 +++----- src/core/runtime/operation.h | 10 ++++------ src/core/runtime/partition_manager.h | 10 ++++------ src/core/runtime/projection.cc | 8 ++++---- src/core/runtime/projection.h | 12 ++++++------ src/core/runtime/runtime.h | 10 ++++------ src/core/task/task.cc | 6 ++---- src/core/task/task.inl | 8 ++++---- src/core/task/variant_helper.h | 8 ++------ src/core/utilities/deserializer.cc | 8 ++++---- src/core/utilities/deserializer.h | 8 ++++---- src/core/utilities/nvtx_help.h | 6 ++---- 48 files changed, 139 insertions(+), 235 deletions(-) diff --git a/src/core/comm/coll.cc b/src/core/comm/coll.cc index 4bd76a7586..9021426822 100644 --- a/src/core/comm/coll.cc +++ b/src/core/comm/coll.cc @@ -31,9 +31,7 @@ #include "legate.h" #include "legion.h" -namespace legate { -namespace comm { -namespace coll { +namespace legate::comm::coll { Logger log_coll("coll"); @@ -162,9 +160,7 @@ void* BackendNetwork::allocateInplaceBuffer(const void* recvbuf, size_t size) return sendbuf_tmp; } -} // namespace coll -} // namespace comm -} // namespace legate +} // namespace legate::comm::coll extern "C" { diff --git a/src/core/comm/coll.h b/src/core/comm/coll.h index 08397f53e6..6b3ae1bfb7 100644 --- a/src/core/comm/coll.h +++ b/src/core/comm/coll.h @@ -34,9 +34,7 @@ #include "core/comm/pthread_barrier.h" #endif -namespace legate { -namespace comm { -namespace coll { +namespace legate::comm::coll { #ifdef LEGATE_USE_NETWORK struct RankMappingTable { @@ -266,6 +264,4 @@ int collFinalize(); int collInitComm(); -} // namespace coll -} // namespace comm -} // namespace legate +} // namespace legate::comm::coll diff --git a/src/core/comm/comm.cc b/src/core/comm/comm.cc index d6e0598516..99de4e0d3f 100644 --- a/src/core/comm/comm.cc +++ b/src/core/comm/comm.cc @@ -24,8 +24,7 @@ #include "core/runtime/communicator_manager.h" #include "core/runtime/runtime.h" -namespace legate { -namespace comm { +namespace legate::comm { void register_tasks(Legion::Machine machine, Legion::Runtime* runtime, @@ -47,5 +46,4 @@ void register_builtin_communicator_factories(const LibraryContext* context) cpu::register_factory(context); } -} // namespace comm -} // namespace legate +} // namespace legate::comm diff --git a/src/core/comm/comm.h b/src/core/comm/comm.h index 4e6e912da6..fe5c9e6dce 100644 --- a/src/core/comm/comm.h +++ b/src/core/comm/comm.h @@ -19,10 +19,10 @@ #include "legate.h" namespace legate { - class LibraryContext; +} // namespace legate -namespace comm { +namespace legate::comm { void register_tasks(Legion::Machine machine, Legion::Runtime* runtime, @@ -30,5 +30,4 @@ void register_tasks(Legion::Machine machine, void register_builtin_communicator_factories(const LibraryContext* context); -} // namespace comm -} // namespace legate +} // namespace legate::comm diff --git a/src/core/comm/comm_cpu.cc b/src/core/comm/comm_cpu.cc index 044d42f139..b6beb521a8 100644 --- a/src/core/comm/comm_cpu.cc +++ b/src/core/comm/comm_cpu.cc @@ -22,9 +22,7 @@ #include "core/comm/coll.h" -namespace legate { -namespace comm { -namespace cpu { +namespace legate::comm::cpu { class Factory : public CommunicatorFactory { public: @@ -228,6 +226,4 @@ void register_factory(const LibraryContext* context) comm_mgr->register_factory("cpu", std::make_unique(context)); } -} // namespace cpu -} // namespace comm -} // namespace legate +} // namespace legate::comm::cpu diff --git a/src/core/comm/comm_cpu.h b/src/core/comm/comm_cpu.h index 3286f3db4a..ee2e733d02 100644 --- a/src/core/comm/comm_cpu.h +++ b/src/core/comm/comm_cpu.h @@ -19,11 +19,10 @@ #include "legion.h" namespace legate { - class LibraryContext; +} // namespace legate -namespace comm { -namespace cpu { +namespace legate::comm::cpu { void register_tasks(Legion::Machine machine, Legion::Runtime* runtime, @@ -31,6 +30,4 @@ void register_tasks(Legion::Machine machine, void register_factory(const LibraryContext* context); -} // namespace cpu -} // namespace comm -} // namespace legate +} // namespace legate::comm::cpu diff --git a/src/core/comm/comm_nccl.cu b/src/core/comm/comm_nccl.cu index ecc2e65c53..6dad06f9a3 100644 --- a/src/core/comm/comm_nccl.cu +++ b/src/core/comm/comm_nccl.cu @@ -28,9 +28,7 @@ #include #include -namespace legate { -namespace comm { -namespace nccl { +namespace legate::comm::nccl { struct _Payload { uint64_t field0; @@ -241,6 +239,4 @@ void register_factory(const LibraryContext* context) comm_mgr->register_factory("nccl", std::make_unique(context)); } -} // namespace nccl -} // namespace comm -} // namespace legate +} // namespace legate::comm::nccl diff --git a/src/core/comm/comm_nccl.h b/src/core/comm/comm_nccl.h index 64d40048e9..a879e21b84 100644 --- a/src/core/comm/comm_nccl.h +++ b/src/core/comm/comm_nccl.h @@ -19,11 +19,10 @@ #include "legion.h" namespace legate { - class LibraryContext; +} // namespace legate -namespace comm { -namespace nccl { +namespace legate::comm::nccl { void register_tasks(Legion::Machine machine, Legion::Runtime* runtime, @@ -33,6 +32,4 @@ bool needs_barrier(); void register_factory(const LibraryContext* context); -} // namespace nccl -} // namespace comm -} // namespace legate +} // namespace legate::comm::nccl diff --git a/src/core/comm/communicator.h b/src/core/comm/communicator.h index 4da3cf7b47..67dd280c02 100644 --- a/src/core/comm/communicator.h +++ b/src/core/comm/communicator.h @@ -23,8 +23,7 @@ * @brief Class definition for legate::comm::Communicator */ -namespace legate { -namespace comm { +namespace legate::comm { /** * @ingroup task @@ -64,5 +63,4 @@ class Communicator { Legion::Future future_{}; }; -} // namespace comm -} // namespace legate +} // namespace legate::comm diff --git a/src/core/comm/local_comm.cc b/src/core/comm/local_comm.cc index 29d317c38c..3d301dc357 100644 --- a/src/core/comm/local_comm.cc +++ b/src/core/comm/local_comm.cc @@ -23,9 +23,7 @@ #include "legate.h" #include "legion.h" -namespace legate { -namespace comm { -namespace coll { +namespace legate::comm::coll { extern Logger log_coll; @@ -345,6 +343,4 @@ void LocalNetwork::barrierLocal(CollComm global_comm) pthread_barrier_wait(const_cast(&(global_comm->local_comm->barrier))); } -} // namespace coll -} // namespace comm -} // namespace legate +} // namespace legate::comm::coll diff --git a/src/core/comm/mpi_comm.cc b/src/core/comm/mpi_comm.cc index 4a75effdfe..c9356e6dff 100644 --- a/src/core/comm/mpi_comm.cc +++ b/src/core/comm/mpi_comm.cc @@ -23,9 +23,7 @@ #include "legate.h" #include "legion.h" -namespace legate { -namespace comm { -namespace coll { +namespace legate::comm::coll { extern Logger log_coll; @@ -568,6 +566,4 @@ int MPINetwork::generateGatherTag(int rank, CollComm global_comm) return tag; } -} // namespace coll -} // namespace comm -} // namespace legate +} // namespace legate::comm::coll diff --git a/src/core/cuda/cuda_help.h b/src/core/cuda/cuda_help.h index 1a59a9c167..eee0856c1f 100644 --- a/src/core/cuda/cuda_help.h +++ b/src/core/cuda/cuda_help.h @@ -40,8 +40,7 @@ #endif -namespace legate { -namespace cuda { +namespace legate::cuda { __host__ inline void check_cuda(cudaError_t error, const char* file, int line) { @@ -60,5 +59,4 @@ __host__ inline void check_cuda(cudaError_t error, const char* file, int line) } } -} // namespace cuda -} // namespace legate +} // namespace legate::cuda diff --git a/src/core/cuda/stream_pool.cu b/src/core/cuda/stream_pool.cu index 88457c4f91..49214ea0bc 100644 --- a/src/core/cuda/stream_pool.cu +++ b/src/core/cuda/stream_pool.cu @@ -19,8 +19,7 @@ #include "core/mapping/machine.h" #include "core/runtime/runtime.h" -namespace legate { -namespace cuda { +namespace legate::cuda { StreamView::~StreamView() { @@ -69,5 +68,4 @@ StreamView StreamPool::get_stream() return pools[proc_id]; } -} // namespace cuda -} // namespace legate +} // namespace legate::cuda diff --git a/src/core/cuda/stream_pool.h b/src/core/cuda/stream_pool.h index 27c55fa906..aecd2e6af9 100644 --- a/src/core/cuda/stream_pool.h +++ b/src/core/cuda/stream_pool.h @@ -26,8 +26,7 @@ * @brief Class definition for legate::cuda::StreamPool */ -namespace legate { -namespace cuda { +namespace legate::cuda { /** * @ingroup task @@ -101,5 +100,4 @@ struct StreamPool { std::unique_ptr cached_stream_{nullptr}; }; -} // namespace cuda -} // namespace legate +} // namespace legate::cuda diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index 41daaba160..d44e5ad8ad 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -33,6 +33,15 @@ * legate::LogicalStorePartition */ +namespace legate::mapping { +class MachineDesc; +} // namespace legate::mapping + +namespace legate::detail { +class LogicalStore; +class LogicalStorePartition; +} // namespace legate::detail + namespace legate { class BufferBuilder; @@ -43,19 +52,6 @@ class Projection; class Runtime; class Store; -namespace mapping { - -class MachineDesc; - -} - -namespace detail { - -class LogicalStore; -class LogicalStorePartition; - -} // namespace detail - /** * @ingroup data * diff --git a/src/core/data/logical_store_detail.cc b/src/core/data/logical_store_detail.cc index 828ea56ebc..414a5f7264 100644 --- a/src/core/data/logical_store_detail.cc +++ b/src/core/data/logical_store_detail.cc @@ -25,10 +25,10 @@ #include "legate_defines.h" namespace legate { - extern Legion::Logger log_legate; +} // namespace legate -namespace detail { +namespace legate::detail { //////////////////////////////////////////////////// // legate::detail::Storage @@ -685,5 +685,4 @@ bool LogicalStorePartition::is_disjoint_for(const Domain* launch_domain) const return storage_partition_->is_disjoint_for(launch_domain); } -} // namespace detail -} // namespace legate +} // namespace legate::detail diff --git a/src/core/data/logical_store_detail.h b/src/core/data/logical_store_detail.h index 6f3355403b..5120a2b622 100644 --- a/src/core/data/logical_store_detail.h +++ b/src/core/data/logical_store_detail.h @@ -24,13 +24,11 @@ #include "core/partitioning/restriction.h" #include "core/runtime/runtime.h" -namespace legate { - -namespace mapping { +namespace legate::mapping { class MachineDesc; -} // namespace mapping +} // namespace legate::mapping -namespace detail { +namespace legate::detail { class StoragePartition; class LogicalStorePartition; @@ -253,5 +251,4 @@ class LogicalStorePartition : public std::enable_shared_from_this store_; }; -} // namespace detail -} // namespace legate +} // namespace legate::detail diff --git a/src/core/mapping/base_mapper.cc b/src/core/mapping/base_mapper.cc index ebe7618f3c..b6f0a1c71e 100644 --- a/src/core/mapping/base_mapper.cc +++ b/src/core/mapping/base_mapper.cc @@ -30,8 +30,7 @@ #include "core/utilities/linearize.h" #include "legate_defines.h" -namespace legate { -namespace mapping { +namespace legate::mapping { namespace { @@ -1335,5 +1334,4 @@ void BaseMapper::handle_task_result(const Legion::Mapping::MapperContext ctx, LEGATE_ABORT; } -} // namespace mapping -} // namespace legate +} // namespace legate::mapping diff --git a/src/core/mapping/base_mapper.h b/src/core/mapping/base_mapper.h index 5489cbe2ee..dc723948e5 100644 --- a/src/core/mapping/base_mapper.h +++ b/src/core/mapping/base_mapper.h @@ -27,8 +27,7 @@ #include "core/runtime/context.h" #include "core/utilities/typedefs.h" -namespace legate { -namespace mapping { +namespace legate::mapping { class InstanceManager; class Machine; @@ -311,5 +310,4 @@ class BaseMapper : public Legion::Mapping::Mapper, public MachineQueryInterface Machine machine; }; -} // namespace mapping -} // namespace legate +} // namespace legate::mapping diff --git a/src/core/mapping/core_mapper.cc b/src/core/mapping/core_mapper.cc index bce21425bb..4d95a08945 100644 --- a/src/core/mapping/core_mapper.cc +++ b/src/core/mapping/core_mapper.cc @@ -44,7 +44,9 @@ uint32_t extract_env(const char* env_name, const uint32_t default_value, const u return atoi(env_value); } -namespace mapping { +} // namespace legate + +namespace legate::mapping { // This is a custom mapper implementation that only has to map // start-up tasks associated with the Legate core, no one else @@ -402,7 +404,9 @@ void CoreMapper::select_tunable_value(const Legion::Mapping::MapperContext ctx, LEGATE_ABORT; } -} // namespace mapping +} // namespace legate::mapping + +namespace legate { void register_legate_core_mapper(Legion::Machine machine, Legion::Runtime* runtime, diff --git a/src/core/mapping/default_mapper.cc b/src/core/mapping/default_mapper.cc index 55417357af..fac10b8051 100644 --- a/src/core/mapping/default_mapper.cc +++ b/src/core/mapping/default_mapper.cc @@ -16,8 +16,7 @@ #include "core/mapping/default_mapper.h" -namespace legate { -namespace mapping { +namespace legate::mapping { // Default mapper doesn't use the machine query interface void DefaultMapper::set_machine(const MachineQueryInterface* machine) {} @@ -39,5 +38,4 @@ Scalar DefaultMapper::tunable_value(TunableID tunable_id) return Scalar(0); } -} // namespace mapping -} // namespace legate +} // namespace legate::mapping diff --git a/src/core/mapping/default_mapper.h b/src/core/mapping/default_mapper.h index 921971d9ce..3fca215c22 100644 --- a/src/core/mapping/default_mapper.h +++ b/src/core/mapping/default_mapper.h @@ -18,8 +18,7 @@ #pragma once -namespace legate { -namespace mapping { +namespace legate::mapping { class DefaultMapper : public Mapper { public: @@ -33,5 +32,4 @@ class DefaultMapper : public Mapper { Scalar tunable_value(TunableID tunable_id) override; }; -} // namespace mapping -} // namespace legate +} // namespace legate::mapping diff --git a/src/core/mapping/instance_manager.cc b/src/core/mapping/instance_manager.cc index 8b9296200d..5c7d9f232c 100644 --- a/src/core/mapping/instance_manager.cc +++ b/src/core/mapping/instance_manager.cc @@ -17,8 +17,7 @@ #include "core/mapping/instance_manager.h" #include "core/utilities/dispatch.h" -namespace legate { -namespace mapping { +namespace legate::mapping { using RegionGroupP = std::shared_ptr; @@ -480,5 +479,4 @@ void ReductionInstanceManager::erase(Instance inst) return manager; } -} // namespace mapping -} // namespace legate +} // namespace legate::mapping diff --git a/src/core/mapping/instance_manager.h b/src/core/mapping/instance_manager.h index 35a49befb5..eb7ac6877b 100644 --- a/src/core/mapping/instance_manager.h +++ b/src/core/mapping/instance_manager.h @@ -23,8 +23,7 @@ #include "core/mapping/mapping.h" -namespace legate { -namespace mapping { +namespace legate::mapping { // This class represents a set of regions that colocate in an instance struct RegionGroup { @@ -235,5 +234,4 @@ class ReductionInstanceManager : public BaseInstanceManager { std::map instance_sets_{}; }; -} // namespace mapping -} // namespace legate +} // namespace legate::mapping diff --git a/src/core/mapping/machine.cc b/src/core/mapping/machine.cc index 26ce331a97..23bc4b154f 100644 --- a/src/core/mapping/machine.cc +++ b/src/core/mapping/machine.cc @@ -17,8 +17,7 @@ #include "core/mapping/machine.h" #include "core/utilities/buffer_builder.h" -namespace legate { -namespace mapping { +namespace legate::mapping { TaskTarget to_target(Processor::Kind kind) { @@ -451,5 +450,4 @@ std::ostream& operator<<(std::ostream& stream, const MachineDesc& info) return stream; } -} // namespace mapping -} // namespace legate +} // namespace legate::mapping diff --git a/src/core/mapping/machine.h b/src/core/mapping/machine.h index 9f361addcb..d3465e2ae9 100644 --- a/src/core/mapping/machine.h +++ b/src/core/mapping/machine.h @@ -28,8 +28,7 @@ * @brief Legate machine interface */ -namespace legate { -namespace mapping { +namespace legate::mapping { TaskTarget to_target(Processor::Kind kind); @@ -383,5 +382,4 @@ class Machine { std::map socket_memories_; }; -} // namespace mapping -} // namespace legate +} // namespace legate::mapping diff --git a/src/core/mapping/mapping.cc b/src/core/mapping/mapping.cc index ed0ef8f3d1..6c1e17c292 100644 --- a/src/core/mapping/mapping.cc +++ b/src/core/mapping/mapping.cc @@ -21,8 +21,7 @@ #include "core/mapping/mapping.h" #include "core/mapping/store.h" -namespace legate { -namespace mapping { +namespace legate::mapping { DimOrdering::DimOrdering(Kind _kind, std::vector&& _dims) : kind(_kind), dims(std::move(_dims)) @@ -226,5 +225,4 @@ void StoreMapping::populate_layout_constraints( return mapping; } -} // namespace mapping -} // namespace legate +} // namespace legate::mapping diff --git a/src/core/mapping/mapping.h b/src/core/mapping/mapping.h index dadc747cf0..3889f0bbf8 100644 --- a/src/core/mapping/mapping.h +++ b/src/core/mapping/mapping.h @@ -27,8 +27,7 @@ * @brief Legate Mapping API */ -namespace legate { -namespace mapping { +namespace legate::mapping { class Store; class Task; @@ -432,5 +431,4 @@ class Mapper { virtual Scalar tunable_value(TunableID tunable_id) = 0; }; -} // namespace mapping -} // namespace legate +} // namespace legate::mapping diff --git a/src/core/mapping/operation.cc b/src/core/mapping/operation.cc index dbb516e32d..279e28481c 100644 --- a/src/core/mapping/operation.cc +++ b/src/core/mapping/operation.cc @@ -18,8 +18,7 @@ #include "core/runtime/context.h" #include "core/utilities/deserializer.h" -namespace legate { -namespace mapping { +namespace legate::mapping { Mappable::Mappable() {} @@ -88,5 +87,4 @@ Copy::Copy(const Legion::Copy* copy, #endif } -} // namespace mapping -} // namespace legate +} // namespace legate::mapping diff --git a/src/core/mapping/operation.h b/src/core/mapping/operation.h index b8fb3d4a55..9f39e0d377 100644 --- a/src/core/mapping/operation.h +++ b/src/core/mapping/operation.h @@ -29,11 +29,11 @@ */ namespace legate { - class LibraryContext; class TransformStack; +} // namespace legate -namespace mapping { +namespace legate::mapping { class Mappable { protected: @@ -143,7 +143,6 @@ class Copy : public Mappable { std::vector output_indirections_; }; -} // namespace mapping -} // namespace legate +} // namespace legate::mapping #include "core/mapping/operation.inl" diff --git a/src/core/mapping/operation.inl b/src/core/mapping/operation.inl index 062007ff73..789f6f3fb2 100644 --- a/src/core/mapping/operation.inl +++ b/src/core/mapping/operation.inl @@ -14,8 +14,7 @@ * */ -namespace legate { -namespace mapping { +namespace legate::mapping { template Legion::Rect RegionField::shape(Legion::Mapping::MapperRuntime* runtime, @@ -36,5 +35,4 @@ Legion::Rect Store::shape() const return Legion::Rect(domain()); } -} // namespace mapping -} // namespace legate +} // namespace legate::mapping diff --git a/src/core/mapping/store.cc b/src/core/mapping/store.cc index 23429cc0fb..dc8c9fad23 100644 --- a/src/core/mapping/store.cc +++ b/src/core/mapping/store.cc @@ -16,8 +16,7 @@ #include "core/mapping/store.h" -namespace legate { -namespace mapping { +namespace legate::mapping { RegionField::RegionField(const Legion::RegionRequirement* req, int32_t dim, @@ -136,5 +135,4 @@ Domain Store::domain() const return result; } -} // namespace mapping -} // namespace legate +} // namespace legate::mapping diff --git a/src/core/mapping/store.h b/src/core/mapping/store.h index 331a598103..69ddc2e512 100644 --- a/src/core/mapping/store.h +++ b/src/core/mapping/store.h @@ -19,8 +19,7 @@ #include "core/data/transform.h" #include "core/type/type_info.h" -namespace legate { -namespace mapping { +namespace legate::mapping { class RegionField { public: @@ -214,5 +213,4 @@ class Store { Legion::Mapping::MapperContext context_{nullptr}; }; -} // namespace mapping -} // namespace legate +} // namespace legate::mapping diff --git a/src/core/partitioning/partition.h b/src/core/partitioning/partition.h index 85daae159d..4299803aa8 100644 --- a/src/core/partitioning/partition.h +++ b/src/core/partitioning/partition.h @@ -24,16 +24,14 @@ #include "core/partitioning/restriction.h" #include "core/utilities/typedefs.h" -namespace legate { - -class Projection; - -namespace detail { - +namespace legate::detail { class LogicalStore; class Storage; +} // namespace legate::detail + +namespace legate { -} // namespace detail +class Projection; struct Partition { public: diff --git a/src/core/runtime/context.h b/src/core/runtime/context.h index 7f9e5b36fd..2948c62b7a 100644 --- a/src/core/runtime/context.h +++ b/src/core/runtime/context.h @@ -36,13 +36,11 @@ * @brief Class definitions for legate::LibraryContext and legate::TaskContext */ -namespace legate { - -namespace mapping { - +namespace legate::mapping { class Mapper; +} // namespace legate::mapping -} +namespace legate { class Store; class Scalar; diff --git a/src/core/runtime/launcher.h b/src/core/runtime/launcher.h index f6a98135e2..fdfa23b042 100644 --- a/src/core/runtime/launcher.h +++ b/src/core/runtime/launcher.h @@ -22,6 +22,10 @@ #include "core/mapping/machine.h" +namespace legate::detail { +class LogicalStore; +} // namespace legate::detail + namespace legate { class ArgWrapper; @@ -34,12 +38,6 @@ class OutputRequirementAnalyzer; class RequirementAnalyzer; class Scalar; -namespace detail { - -class LogicalStore; - -} // namespace detail - class TaskLauncher { public: TaskLauncher(const LibraryContext* library, diff --git a/src/core/runtime/machine_manager.h b/src/core/runtime/machine_manager.h index e4017aa861..d086521eaa 100644 --- a/src/core/runtime/machine_manager.h +++ b/src/core/runtime/machine_manager.h @@ -18,13 +18,11 @@ #include "core/mapping/machine.h" -namespace legate { - -namespace mapping { - +namespace legate::mapping { class MachineDesc; +} // namespace legate::mapping -} // namespace mapping +namespace legate { class MachineManager { public: diff --git a/src/core/runtime/operation.h b/src/core/runtime/operation.h index 98fe867eb5..2cc001791c 100644 --- a/src/core/runtime/operation.h +++ b/src/core/runtime/operation.h @@ -32,6 +32,10 @@ * @brief Class definitions for various operation kinds */ +namespace legate::detail { +class LogicalStore; +} // namespace legate::detail + namespace legate { class CommunicatorFactory; @@ -42,12 +46,6 @@ class Runtime; class Scalar; class Strategy; -namespace detail { - -class LogicalStore; - -} // namespace detail - /** * @ingroup op * @brief A base class for all operation kinds diff --git a/src/core/runtime/partition_manager.h b/src/core/runtime/partition_manager.h index 8ba7620b85..4ad8b93354 100644 --- a/src/core/runtime/partition_manager.h +++ b/src/core/runtime/partition_manager.h @@ -23,6 +23,10 @@ #include "core/data/shape.h" #include "core/partitioning/restriction.h" +namespace legate::mapping { +class MachineDesc; +} // namespace legate::mapping + namespace legate { class LibraryContext; @@ -30,12 +34,6 @@ class Runtime; class Tiling; class Weighted; -namespace mapping { - -class MachineDesc; - -} // namespace mapping - class PartitionManager { public: PartitionManager(Runtime* runtime, const LibraryContext* context); diff --git a/src/core/runtime/projection.cc b/src/core/runtime/projection.cc index 7e2d2734ba..6a748e0701 100644 --- a/src/core/runtime/projection.cc +++ b/src/core/runtime/projection.cc @@ -29,9 +29,7 @@ extern Legion::Logger log_legate; -namespace legate { - -namespace proj { +namespace legate::proj { SymbolicExpr::SymbolicExpr(int32_t dim, int32_t weight, int32_t offset) : dim_(dim), weight_(weight), offset_(offset) @@ -107,7 +105,9 @@ bool is_identity(int32_t src_ndim, const SymbolicPoint& point) return true; } -} // namespace proj +} // namespace legate::proj + +namespace legate { // This special functor overrides the default projection implementation because it needs // to know the the target color space for delinearization. Also note that this functor's diff --git a/src/core/runtime/projection.h b/src/core/runtime/projection.h index ef6de1bae3..7997a951dc 100644 --- a/src/core/runtime/projection.h +++ b/src/core/runtime/projection.h @@ -21,11 +21,7 @@ #include "core/utilities/tuple.h" #include "core/utilities/typedefs.h" -namespace legate { - -class LibraryContext; - -namespace proj { +namespace legate::proj { class SymbolicExpr { public: @@ -62,7 +58,11 @@ SymbolicPoint create_symbolic_point(int32_t ndim); bool is_identity(int32_t ndim, const SymbolicPoint& point); -} // namespace proj +} // namespace legate::proj + +namespace legate { + +class LibraryContext; // Interface for Legate projection functors class LegateProjectionFunctor : public Legion::ProjectionFunctor { diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index e0b3d89ccf..0f48e45d18 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -35,17 +35,15 @@ /** @defgroup runtime Runtime and library contexts */ +namespace legate::mapping { +class Mapper; +} // namespace legate::mapping + namespace legate { class LibraryContext; class Scalar; -namespace mapping { - -class Mapper; - -} // namespace mapping - extern uint32_t extract_env(const char* env_name, const uint32_t default_value, const uint32_t test_value); diff --git a/src/core/task/task.cc b/src/core/task/task.cc index 83901b210c..70b7cc352c 100644 --- a/src/core/task/task.cc +++ b/src/core/task/task.cc @@ -29,8 +29,7 @@ #include "core/utilities/nvtx_help.h" #include "core/utilities/typedefs.h" -namespace legate { -namespace detail { +namespace legate::detail { std::string generate_task_name(const std::type_info& ti) { @@ -88,5 +87,4 @@ void task_wrapper(VariantImpl variant_impl, return_values.finalize(legion_context); } -} // namespace detail -} // namespace legate +} // namespace legate::detail diff --git a/src/core/task/task.inl b/src/core/task/task.inl index 67f8895da5..797e2659c7 100644 --- a/src/core/task/task.inl +++ b/src/core/task/task.inl @@ -18,16 +18,16 @@ #include "core/task/task.h" -namespace legate { - -namespace detail { +namespace legate::detail { std::string generate_task_name(const std::type_info&); void task_wrapper( VariantImpl, const std::string&, const void*, size_t, const void*, size_t, Legion::Processor); -}; // namespace detail +}; // namespace legate::detail + +namespace legate { template template diff --git a/src/core/task/variant_helper.h b/src/core/task/variant_helper.h index 0bd2cf8798..9b16649aca 100644 --- a/src/core/task/variant_helper.h +++ b/src/core/task/variant_helper.h @@ -21,9 +21,7 @@ #include "core/task/task_info.h" #include "core/task/variant_options.h" -namespace legate { - -namespace detail { +namespace legate::detail { template using void_t = void; @@ -81,6 +79,4 @@ struct VariantHelper { } }; -} // namespace detail - -} // namespace legate +} // namespace legate::detail diff --git a/src/core/utilities/deserializer.cc b/src/core/utilities/deserializer.cc index c70fef9b28..d9a53e8b37 100644 --- a/src/core/utilities/deserializer.cc +++ b/src/core/utilities/deserializer.cc @@ -121,7 +121,9 @@ void TaskDeserializer::_unpack(Legion::PhaseBarrier& barrier) barrier = Legion::CObjectWrapper::unwrap(barrier_); } -namespace mapping { +} // namespace legate + +namespace legate::mapping { MapperDataDeserializer::MapperDataDeserializer(const Legion::Mappable* mappable) : BaseDeserializer(mappable->mapper_data, mappable->mapper_data_size) @@ -248,6 +250,4 @@ void CopyDeserializer::_unpack(RegionField& value) value = RegionField(req, dim, idx + req_index_offset_, fid); } -} // namespace mapping - -} // namespace legate +} // namespace legate::mapping diff --git a/src/core/utilities/deserializer.h b/src/core/utilities/deserializer.h index ad53f9c11e..4b1b577c66 100644 --- a/src/core/utilities/deserializer.h +++ b/src/core/utilities/deserializer.h @@ -108,7 +108,9 @@ class TaskDeserializer : public BaseDeserializer { std::vector outputs_; }; -namespace mapping { +} // namespace legate + +namespace legate::mapping { class MapperDataDeserializer : public BaseDeserializer { public: @@ -168,8 +170,6 @@ class CopyDeserializer : public BaseDeserializer { uint32_t req_index_offset_; }; -} // namespace mapping - -} // namespace legate +} // namespace legate::mapping #include "core/utilities/deserializer.inl" diff --git a/src/core/utilities/nvtx_help.h b/src/core/utilities/nvtx_help.h index 9b5c6270c0..113c294202 100644 --- a/src/core/utilities/nvtx_help.h +++ b/src/core/utilities/nvtx_help.h @@ -22,8 +22,7 @@ #include -namespace legate { -namespace nvtx { +namespace legate::nvtx { class Range { public: @@ -34,7 +33,6 @@ class Range { nvtxRangeId_t range_; }; -} // namespace nvtx -} // namespace legate +} // namespace legate::nvtx #endif From f09bf3b2d42ae0d3763ed96cb834cb8f9424008c Mon Sep 17 00:00:00 2001 From: Manolis Papadakis Date: Mon, 5 Jun 2023 16:11:27 -0700 Subject: [PATCH 0137/1425] Remove inline-mapped regions from cache on unmapping * Apply 1 suggestion(s) to 1 file(s) * More tests for inline mapping * Remove leftover gmake file * Only unmap once when a PhysicalRegion is used for two LogicalStores * Include foo.h in foo.inl, to help VSCode cope * Remove inline-mapped regions from cache on unmapping Co-authored-by: Wonchan Lee See merge request legate/legate.core.internal!43 --- legate_core_cpp.cmake | 5 +- src/core.mk | 110 -------------------------- src/core/data/logical_store_detail.cc | 2 +- src/core/data/scalar.inl | 5 ++ src/core/data/slice.h | 2 +- src/core/data/store.inl | 5 ++ src/core/mapping/operation.inl | 5 ++ src/core/runtime/context.inl | 1 + src/core/runtime/runtime.cc | 33 +++++--- src/core/runtime/runtime.h | 5 +- src/core/runtime/runtime.inl | 1 + src/core/task/task.inl | 1 + src/core/utilities/buffer_builder.inl | 5 ++ src/core/utilities/deserializer.inl | 5 ++ src/core/utilities/multi_set.h | 55 +++++++++++++ src/core/utilities/multi_set.inl | 53 +++++++++++++ src/core/utilities/tuple.inl | 5 ++ tests/cpp/.gitignore | 1 + tests/cpp/integration/inline_map.cc | 108 +++++++++++++++++++++++++ tests/cpp/run.sh | 2 +- 20 files changed, 284 insertions(+), 125 deletions(-) delete mode 100644 src/core.mk create mode 100644 src/core/utilities/multi_set.h create mode 100644 src/core/utilities/multi_set.inl create mode 100644 tests/cpp/.gitignore create mode 100644 tests/cpp/integration/inline_map.cc diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index bb4355ef51..08231d437b 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -373,6 +373,7 @@ if (legate_core_BUILD_DOCS) src/core/runtime/tracker.h src/core/utilities/debug.h src/core/utilities/dispatch.h + src/core/utilities/multi_set.h # main page src/legate.h ) @@ -492,10 +493,12 @@ install( src/core/utilities/deserializer.inl src/core/utilities/dispatch.h src/core/utilities/machine.h + src/core/utilities/multi_set.h + src/core/utilities/multi_set.inl src/core/utilities/nvtx_help.h src/core/utilities/span.h - src/core/utilities/tuple.inl src/core/utilities/tuple.h + src/core/utilities/tuple.inl src/core/utilities/typedefs.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/legate/core/utilities) diff --git a/src/core.mk b/src/core.mk deleted file mode 100644 index 8cc26842fd..0000000000 --- a/src/core.mk +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -# General source files -GEN_CPU_SRC = core/legate_c.cc \ - core/comm/comm.cc \ - core/comm/comm_cpu.cc \ - core/comm/coll.cc \ - core/data/allocator.cc \ - core/data/logical_store.cc \ - core/data/logical_store_detail.cc \ - core/data/scalar.cc \ - core/data/store.cc \ - core/data/transform.cc \ - core/mapping/base_mapper.cc \ - core/mapping/core_mapper.cc \ - core/mapping/instance_manager.cc \ - core/mapping/mapping.cc \ - core/mapping/task.cc \ - core/partitioning/constraint.cc \ - core/partitioning/constraint_graph.cc \ - core/partitioning/partition.cc \ - core/partitioning/partitioner.cc \ - core/runtime/context.cc \ - core/runtime/launcher.cc \ - core/runtime/launcher_arg.cc \ - core/runtime/operation.cc \ - core/runtime/projection.cc \ - core/runtime/req_analyzer.cc \ - core/runtime/runtime.cc \ - core/runtime/shard.cc \ - core/task/return.cc \ - core/task/task.cc \ - core/utilities/buffer_builder.cc \ - core/utilities/debug.cc \ - core/utilities/deserializer.cc \ - core/utilities/machine.cc \ - core/utilities/linearize.cc - -ifeq ($(strip $(USE_GASNET)),1) -GEN_CPU_SRC += core/comm/alltoall_thread_mpi.cc \ - core/comm/alltoallv_thread_mpi.cc \ - core/comm/gather_thread_mpi.cc \ - core/comm/allgather_thread_mpi.cc \ - core/comm/bcast_thread_mpi.cc -else -GEN_CPU_SRC += core/comm/alltoall_thread_local.cc \ - core/comm/alltoallv_thread_local.cc \ - core/comm/allgather_thread_local.cc -endif - -# Source files for GPUs -GEN_GPU_SRC = core/comm/comm_nccl.cu \ - core/cuda/stream_pool.cu - -# Header files that we need to have installed for client legate libraries -INSTALL_HEADERS = legate.h \ - legate_defines.h \ - legate_preamble.h \ - core/legate_c.h \ - core/comm/communicator.h \ - core/comm/coll.h \ - core/comm/pthread_barrier.h \ - core/cuda/cuda_help.h \ - core/cuda/stream_pool.h \ - core/data/allocator.h \ - core/data/buffer.h \ - core/data/logical_store.h \ - core/data/scalar.h \ - core/data/scalar.inl \ - core/data/store.h \ - core/data/store.inl \ - core/data/transform.h \ - core/mapping/base_mapper.h \ - core/mapping/mapping.h \ - core/mapping/task.h \ - core/mapping/task.inl \ - core/partitioning/constraint.h \ - core/runtime/context.h \ - core/runtime/operation.h \ - core/runtime/projection.h \ - core/runtime/runtime.h \ - core/runtime/runtime.inl \ - core/task/exception.h \ - core/task/return.h \ - core/task/task.h \ - core/utilities/debug.h \ - core/utilities/deserializer.h \ - core/utilities/deserializer.inl \ - core/utilities/dispatch.h \ - core/utilities/machine.h \ - core/utilities/nvtx_help.h \ - core/utilities/span.h \ - core/utilities/tuple.inl \ - core/utilities/tuple.h \ - core/utilities/type_traits.h \ - core/utilities/typedefs.h diff --git a/src/core/data/logical_store_detail.cc b/src/core/data/logical_store_detail.cc index 414a5f7264..57a5124a04 100644 --- a/src/core/data/logical_store_detail.cc +++ b/src/core/data/logical_store_detail.cc @@ -198,7 +198,7 @@ RegionField Storage::map(LibraryContext* context) #ifdef DEBUG_LEGATE assert(Kind::REGION_FIELD == kind_); #endif - return Runtime::get_runtime()->map_region_field(context, region_field_->get_root()); + return Runtime::get_runtime()->map_region_field(context, get_region_field()); } Restrictions Storage::compute_restrictions() const diff --git a/src/core/data/scalar.inl b/src/core/data/scalar.inl index 8f201d5ba4..e4debab7dd 100644 --- a/src/core/data/scalar.inl +++ b/src/core/data/scalar.inl @@ -14,6 +14,11 @@ * */ +#pragma once + +// Useful for IDEs +#include "core/data/scalar.h" + namespace legate { template diff --git a/src/core/data/slice.h b/src/core/data/slice.h index 05915e0b1d..ba0337322b 100644 --- a/src/core/data/slice.h +++ b/src/core/data/slice.h @@ -33,7 +33,7 @@ namespace legate { * from std::slice. */ struct Slice { - static constexpr std::optional OPEN = std::nullopt; + static constexpr std::nullopt_t OPEN = std::nullopt; Slice(std::optional _start = OPEN, std::optional _stop = OPEN) : start(_start), stop(_stop) diff --git a/src/core/data/store.inl b/src/core/data/store.inl index 240813c9e0..09611826e5 100644 --- a/src/core/data/store.inl +++ b/src/core/data/store.inl @@ -14,6 +14,11 @@ * */ +#pragma once + +// Useful for IDEs +#include "core/data/store.h" + namespace legate { template diff --git a/src/core/mapping/operation.inl b/src/core/mapping/operation.inl index 789f6f3fb2..22e4bf9408 100644 --- a/src/core/mapping/operation.inl +++ b/src/core/mapping/operation.inl @@ -14,6 +14,11 @@ * */ +#pragma once + +// Useful for IDEs +#include "core/mapping/operation.h" + namespace legate::mapping { template diff --git a/src/core/runtime/context.inl b/src/core/runtime/context.inl index ca4ce921c2..6de76c2e68 100644 --- a/src/core/runtime/context.inl +++ b/src/core/runtime/context.inl @@ -16,6 +16,7 @@ #pragma once +// Useful for IDEs #include "core/runtime/context.h" namespace legate { diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 05e94502cf..641215e1ee 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -583,41 +583,54 @@ std::shared_ptr Runtime::import_region_field(Legion::Logical Legion::FieldID field_id, uint32_t field_size) { - // TODO: This is a blocking operation. We should instead use index sapces as keys to field + // TODO: This is a blocking operation. We should instead use index spaces as keys to field // managers auto shape = legion_runtime_->get_index_space_domain(legion_context_, region.get_index_space()); auto fld_mgr = runtime_->find_or_create_field_manager(shape, field_size); return fld_mgr->import_field(region, field_id); } -RegionField Runtime::map_region_field(LibraryContext* context, const LogicalRegionField& rf) +RegionField Runtime::map_region_field(LibraryContext* context, LogicalRegionField* rf) { - auto region = rf.region(); - auto field_id = rf.field_id(); + auto root_region = rf->get_root().region(); + auto field_id = rf->field_id(); Legion::PhysicalRegion pr; - RegionFieldID key(region, field_id); + RegionFieldID key(root_region, field_id); auto finder = inline_mapped_.find(key); if (inline_mapped_.end() == finder) { - Legion::RegionRequirement req(region, READ_WRITE, EXCLUSIVE, region); + Legion::RegionRequirement req(root_region, READ_WRITE, EXCLUSIVE, root_region); req.add_field(field_id); auto mapper_id = context->get_mapper_id(); // TODO: We need to pass the metadata about logical store Legion::InlineLauncher launcher(req, mapper_id); - pr = legion_runtime_->map_region(legion_context_, launcher); - inline_mapped_.insert({key, pr}); + pr = legion_runtime_->map_region(legion_context_, launcher); + inline_mapped_[key] = pr; } else pr = finder->second; - return RegionField(rf.dim(), pr, field_id); + physical_region_refs_.add(pr); + return RegionField(rf->dim(), pr, field_id); } void Runtime::unmap_physical_region(Legion::PhysicalRegion pr) { - legion_runtime_->unmap_region(legion_context_, pr); + if (physical_region_refs_.remove(pr)) { + // The last user of this inline mapping was removed, so remove it from our cache and unmap. + std::vector fields; + pr.get_fields(fields); + assert(fields.size() == 1); + RegionFieldID key(pr.get_logical_region(), fields[0]); + auto finder = inline_mapped_.find(key); + assert(finder != inline_mapped_.end() && finder->second == pr); + inline_mapped_.erase(finder); + legion_runtime_->unmap_region(legion_context_, pr); + } } +size_t Runtime::num_inline_mapped() const { return inline_mapped_.size(); } + RegionManager* Runtime::find_or_create_region_manager(const Domain& shape) { auto finder = region_managers_.find(shape); diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 0f48e45d18..1a2708debf 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -30,6 +30,7 @@ #include "core/runtime/resource.h" #include "core/task/exception.h" #include "core/type/type_info.h" +#include "core/utilities/multi_set.h" #include "core/utilities/typedefs.h" /** @defgroup runtime Runtime and library contexts @@ -291,8 +292,9 @@ class Runtime { std::shared_ptr import_region_field(Legion::LogicalRegion region, Legion::FieldID field_id, uint32_t field_size); - RegionField map_region_field(LibraryContext* context, const LogicalRegionField& region_field); + RegionField map_region_field(LibraryContext* context, LogicalRegionField* region_field); void unmap_physical_region(Legion::PhysicalRegion pr); + size_t num_inline_mapped() const; public: RegionManager* find_or_create_region_manager(const Legion::Domain& shape); @@ -421,6 +423,7 @@ class Runtime { private: using RegionFieldID = std::pair; std::map inline_mapped_; + MultiSet physical_region_refs_; uint64_t next_store_id_{1}; uint64_t next_storage_id_{1}; diff --git a/src/core/runtime/runtime.inl b/src/core/runtime/runtime.inl index f56849c843..57a045c8bc 100644 --- a/src/core/runtime/runtime.inl +++ b/src/core/runtime/runtime.inl @@ -16,6 +16,7 @@ #pragma once +// Useful for IDEs #include "core/runtime/runtime.h" namespace legate { diff --git a/src/core/task/task.inl b/src/core/task/task.inl index 797e2659c7..0e19aeb700 100644 --- a/src/core/task/task.inl +++ b/src/core/task/task.inl @@ -16,6 +16,7 @@ #pragma once +// Useful for IDEs #include "core/task/task.h" namespace legate::detail { diff --git a/src/core/utilities/buffer_builder.inl b/src/core/utilities/buffer_builder.inl index a22fc30ed1..3cdb7c850b 100644 --- a/src/core/utilities/buffer_builder.inl +++ b/src/core/utilities/buffer_builder.inl @@ -14,6 +14,11 @@ * */ +#pragma once + +// Useful for IDEs +#include "core/utilities/buffer_builder.h" + namespace legate { template diff --git a/src/core/utilities/deserializer.inl b/src/core/utilities/deserializer.inl index 25563fa6b7..c35a529c6f 100644 --- a/src/core/utilities/deserializer.inl +++ b/src/core/utilities/deserializer.inl @@ -14,6 +14,11 @@ * */ +#pragma once + +// Useful for IDEs +#include "core/utilities/deserializer.h" + namespace legate { template diff --git a/src/core/utilities/multi_set.h b/src/core/utilities/multi_set.h new file mode 100644 index 0000000000..6f049c0a55 --- /dev/null +++ b/src/core/utilities/multi_set.h @@ -0,0 +1,55 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include + +namespace legate { + +/** + * @brief A set variant that allows for multiple instances of each element. + */ +template +class MultiSet { + public: + MultiSet() {} + + public: + /** + * @brief Add a value to the container. + */ + void add(const T& value); + + /** + * @brief Remove an instance of a value from the container (other instances might still remain). + * + * @return Whether this removed the last instance of the value. + */ + bool remove(const T& value); + + /** + * @brief Test whether a value is present in the container (at least once). + */ + bool contains(const T& value) const; + + private: + std::map map_; +}; + +} // namespace legate + +#include "core/utilities/multi_set.inl" diff --git a/src/core/utilities/multi_set.inl b/src/core/utilities/multi_set.inl new file mode 100644 index 0000000000..61f464fd70 --- /dev/null +++ b/src/core/utilities/multi_set.inl @@ -0,0 +1,53 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +// Useful for IDEs +#include "core/utilities/multi_set.h" + +namespace legate { + +template +void MultiSet::add(const T& value) +{ + auto finder = map_.find(value); + if (finder == map_.end()) + map_[value] = 1; + else + finder->second++; +} + +template +bool MultiSet::remove(const T& value) +{ + auto finder = map_.find(value); + assert(finder != map_.end()); + finder->second--; + if (finder->second == 0) { + map_.erase(finder); + return true; + } + return false; +} + +template +bool MultiSet::contains(const T& value) const +{ + return map_.contains(value); +} + +} // namespace legate diff --git a/src/core/utilities/tuple.inl b/src/core/utilities/tuple.inl index 326cc5e99d..77fc1af436 100644 --- a/src/core/utilities/tuple.inl +++ b/src/core/utilities/tuple.inl @@ -14,6 +14,11 @@ * */ +#pragma once + +// Useful for IDEs +#include "core/utilities/tuple.h" + #include #include #include diff --git a/tests/cpp/.gitignore b/tests/cpp/.gitignore new file mode 100644 index 0000000000..567609b123 --- /dev/null +++ b/tests/cpp/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/tests/cpp/integration/inline_map.cc b/tests/cpp/integration/inline_map.cc new file mode 100644 index 0000000000..fea669c84a --- /dev/null +++ b/tests/cpp/integration/inline_map.cc @@ -0,0 +1,108 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "legate.h" + +namespace inline_map { + +static const char* library_name = "test_inline_map"; + +enum TaskOpCode { + ADDER, +}; + +struct AdderTask : public legate::LegateTask { + static const int32_t TASK_ID = ADDER; + static void cpu_variant(legate::TaskContext& context) + { + auto& output = context.outputs()[0]; + auto shape = output.shape<1>(); + auto acc = output.read_write_accessor(shape); + for (legate::PointInRectIterator<1> it(shape); it.valid(); ++it) acc[*it] += 1; + } +}; + +void register_tasks() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + AdderTask::register_variants(context); +} + +void test_mapped_regions_leak(legate::Runtime* runtime, legate::LibraryContext* context) +{ + { + auto l_store = runtime->create_store({5}, legate::int64()); + auto p_store = l_store.get_physical_store(context); + EXPECT_FALSE(p_store->is_future()); + EXPECT_EQ(runtime->num_inline_mapped(), 1); + } + EXPECT_EQ(runtime->num_inline_mapped(), 0); +} + +void test_inline_map_future(legate::Runtime* runtime, legate::LibraryContext* context) +{ + auto l_store = runtime->create_store({1}, legate::int64(), true /*optimize_scalar*/); + auto p_store = l_store.get_physical_store(context); + EXPECT_TRUE(p_store->is_future()); +} + +void test_inline_map_region_and_slice(legate::Runtime* runtime, legate::LibraryContext* context) +{ + auto root_ls = runtime->create_store({5}, legate::int64()); + auto root_ps = root_ls.get_physical_store(context); + EXPECT_FALSE(root_ps->is_future()); + auto slice_ls = root_ls.slice(0, legate::Slice(1)); + auto slice_ps = slice_ls.get_physical_store(context); + EXPECT_FALSE(slice_ps->is_future()); + auto root_acc = root_ps->write_accessor(); + root_acc[2] = 42; + auto slice_acc = slice_ps->read_accessor(); + EXPECT_EQ(slice_acc[1], 42); +} + +void test_inline_map_and_task(legate::Runtime* runtime, legate::LibraryContext* context) +{ + auto l_store = runtime->create_store({5}, legate::int64()); + { + auto p_store = l_store.get_physical_store(context); + auto acc = p_store->write_accessor(); + acc[2] = 42; + } + auto task = runtime->create_task(context, ADDER, {1}); + task->add_input(l_store); + task->add_output(l_store); + runtime->submit(std::move(task)); + auto p_store = l_store.get_physical_store(context); + auto acc = p_store->read_accessor(); + EXPECT_EQ(acc[2], 43); +} + +TEST(Integration, InlineMap) +{ + auto runtime = legate::Runtime::get_runtime(); + legate::Core::perform_registration(); + auto context = runtime->find_library(library_name); + + test_mapped_regions_leak(runtime, context); + test_inline_map_future(runtime, context); + test_inline_map_region_and_slice(runtime, context); + test_inline_map_and_task(runtime, context); +} + +} // namespace inline_map diff --git a/tests/cpp/run.sh b/tests/cpp/run.sh index ffc95617e7..962ef8b319 100755 --- a/tests/cpp/run.sh +++ b/tests/cpp/run.sh @@ -1,4 +1,4 @@ #!/bin/bash cd build -LEGATE_TEST=1 LEGION_DEFAULT_ARGS="-ll:cpu 4" ctest --output-on-failure +REALM_BACKTRACE=1 LEGATE_TEST=1 LEGION_DEFAULT_ARGS="-ll:cpu 4" ctest --output-on-failure "$@" From ddbf13fd747b26a0d817858eaf9764d6200ef35a Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Mon, 5 Jun 2023 20:33:07 -0700 Subject: [PATCH 0138/1425] Pimpl for legate::Runtime * Missing files * Move logical store details to data/detail * Clean-up for callbacks * Expose record_reduction_operator to the API for now * Move internal-only classes to the detail directory * Pimpl for runtime See merge request legate/legate.core.internal!48 --- legate_core_cpp.cmake | 28 +- src/core/comm/comm.cc | 3 +- src/core/comm/comm_cpu.cc | 17 +- src/core/comm/comm_nccl.cu | 16 +- .../data/{ => detail}/logical_region_field.cc | 8 +- .../data/{ => detail}/logical_region_field.h | 6 +- .../logical_store.cc} | 6 +- .../logical_store.h} | 6 +- src/core/data/logical_store.cc | 4 +- src/core/data/logical_store.h | 2 - src/core/data/scalar.cc | 9 +- src/core/data/scalar.h | 12 +- src/core/data/slice.h | 1 + src/core/data/store.cc | 4 +- src/core/partitioning/constraint_solver.cc | 8 +- src/core/partitioning/constraint_solver.h | 4 +- src/core/partitioning/partition.cc | 13 +- src/core/partitioning/partition.h | 2 - src/core/partitioning/partitioner.cc | 16 +- src/core/partitioning/partitioner.h | 9 +- src/core/runtime/context.cc | 6 + src/core/runtime/context.h | 13 +- src/core/runtime/context.inl | 65 +- .../{ => detail}/communicator_manager.cc | 13 +- .../{ => detail}/communicator_manager.h | 5 +- .../runtime/{ => detail}/field_manager.cc | 12 +- src/core/runtime/{ => detail}/field_manager.h | 4 +- src/core/runtime/{ => detail}/launcher.cc | 30 +- src/core/runtime/{ => detail}/launcher.h | 25 +- src/core/runtime/{ => detail}/launcher_arg.cc | 18 +- src/core/runtime/{ => detail}/launcher_arg.h | 20 +- .../runtime/{ => detail}/machine_manager.cc | 6 +- .../runtime/{ => detail}/machine_manager.h | 4 +- .../runtime/{ => detail}/partition_manager.cc | 8 +- .../runtime/{ => detail}/partition_manager.h | 13 +- .../{ => detail}/provenance_manager.cc | 6 +- .../runtime/{ => detail}/provenance_manager.h | 4 +- .../runtime/{ => detail}/region_manager.cc | 18 +- .../runtime/{ => detail}/region_manager.h | 7 +- src/core/runtime/{ => detail}/req_analyzer.cc | 9 +- src/core/runtime/{ => detail}/req_analyzer.h | 4 +- src/core/runtime/detail/runtime.cc | 902 ++++++++++++++++++ src/core/runtime/detail/runtime.h | 247 +++++ src/core/runtime/operation.cc | 40 +- src/core/runtime/operation.h | 30 +- src/core/runtime/runtime.cc | 851 +---------------- src/core/runtime/runtime.h | 198 +--- src/core/runtime/runtime.inl | 18 +- src/core/runtime/tracker.cc | 20 +- src/core/task/return.cc | 124 +-- src/core/type/type_info.cc | 10 +- src/legate.h | 2 - tests/cpp/integration/inline_map.cc | 5 +- tests/cpp/integration/provenance.cc | 33 +- tests/cpp/main.cc | 8 +- 55 files changed, 1567 insertions(+), 1385 deletions(-) rename src/core/data/{ => detail}/logical_region_field.cc (92%) rename src/core/data/{ => detail}/logical_region_field.h (97%) rename src/core/data/{logical_store_detail.cc => detail/logical_store.cc} (99%) rename src/core/data/{logical_store_detail.h => detail/logical_store.h} (98%) rename src/core/runtime/{ => detail}/communicator_manager.cc (91%) rename src/core/runtime/{ => detail}/communicator_manager.h (95%) rename src/core/runtime/{ => detail}/field_manager.cc (90%) rename src/core/runtime/{ => detail}/field_manager.h (95%) rename src/core/runtime/{ => detail}/launcher.cc (94%) rename src/core/runtime/{ => detail}/launcher.h (94%) rename src/core/runtime/{ => detail}/launcher_arg.cc (86%) rename src/core/runtime/{ => detail}/launcher_arg.h (88%) rename src/core/runtime/{ => detail}/machine_manager.cc (92%) rename src/core/runtime/{ => detail}/machine_manager.h (94%) rename src/core/runtime/{ => detail}/partition_manager.cc (98%) rename src/core/runtime/{ => detail}/partition_manager.h (97%) rename src/core/runtime/{ => detail}/provenance_manager.cc (93%) rename src/core/runtime/{ => detail}/provenance_manager.h (94%) rename src/core/runtime/{ => detail}/region_manager.cc (78%) rename src/core/runtime/{ => detail}/region_manager.h (92%) rename src/core/runtime/{ => detail}/req_analyzer.cc (98%) rename src/core/runtime/{ => detail}/req_analyzer.h (98%) create mode 100644 src/core/runtime/detail/runtime.cc create mode 100644 src/core/runtime/detail/runtime.h diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index 08231d437b..9d522fc1ad 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -193,13 +193,13 @@ list(APPEND legate_core_SOURCES src/core/comm/comm_cpu.cc src/core/comm/coll.cc src/core/data/allocator.cc - src/core/data/logical_region_field.cc src/core/data/logical_store.cc - src/core/data/logical_store_detail.cc src/core/data/scalar.cc src/core/data/shape.cc src/core/data/store.cc src/core/data/transform.cc + src/core/data/detail/logical_region_field.cc + src/core/data/detail/logical_store.cc src/core/mapping/base_mapper.cc src/core/mapping/core_mapper.cc src/core/mapping/default_mapper.cc @@ -213,21 +213,22 @@ list(APPEND legate_core_SOURCES src/core/partitioning/partition.cc src/core/partitioning/partitioner.cc src/core/partitioning/restriction.cc - src/core/runtime/communicator_manager.cc src/core/runtime/context.cc - src/core/runtime/field_manager.cc - src/core/runtime/launcher_arg.cc - src/core/runtime/launcher.cc - src/core/runtime/machine_manager.cc src/core/runtime/operation.cc - src/core/runtime/partition_manager.cc src/core/runtime/projection.cc - src/core/runtime/provenance_manager.cc - src/core/runtime/region_manager.cc - src/core/runtime/req_analyzer.cc src/core/runtime/runtime.cc src/core/runtime/tracker.cc src/core/runtime/shard.cc + src/core/runtime/detail/communicator_manager.cc + src/core/runtime/detail/field_manager.cc + src/core/runtime/detail/launcher_arg.cc + src/core/runtime/detail/launcher.cc + src/core/runtime/detail/machine_manager.cc + src/core/runtime/detail/partition_manager.cc + src/core/runtime/detail/provenance_manager.cc + src/core/runtime/detail/region_manager.cc + src/core/runtime/detail/req_analyzer.cc + src/core/runtime/detail/runtime.cc src/core/task/registrar.cc src/core/task/return.cc src/core/task/task.cc @@ -373,7 +374,6 @@ if (legate_core_BUILD_DOCS) src/core/runtime/tracker.h src/core/utilities/debug.h src/core/utilities/dispatch.h - src/core/utilities/multi_set.h # main page src/legate.h ) @@ -460,9 +460,7 @@ install( install( FILES src/core/runtime/context.h src/core/runtime/context.inl - src/core/runtime/machine_manager.h src/core/runtime/operation.h - src/core/runtime/provenance_manager.h src/core/runtime/resource.h src/core/runtime/projection.h src/core/runtime/runtime.h @@ -493,8 +491,6 @@ install( src/core/utilities/deserializer.inl src/core/utilities/dispatch.h src/core/utilities/machine.h - src/core/utilities/multi_set.h - src/core/utilities/multi_set.inl src/core/utilities/nvtx_help.h src/core/utilities/span.h src/core/utilities/tuple.h diff --git a/src/core/comm/comm.cc b/src/core/comm/comm.cc index 99de4e0d3f..c34ba3a49a 100644 --- a/src/core/comm/comm.cc +++ b/src/core/comm/comm.cc @@ -21,8 +21,7 @@ #include "core/comm/comm_cpu.h" #include "env_defaults.h" -#include "core/runtime/communicator_manager.h" -#include "core/runtime/runtime.h" +#include "core/runtime/detail/communicator_manager.h" namespace legate::comm { diff --git a/src/core/comm/comm_cpu.cc b/src/core/comm/comm_cpu.cc index b6beb521a8..04bf889226 100644 --- a/src/core/comm/comm_cpu.cc +++ b/src/core/comm/comm_cpu.cc @@ -15,16 +15,16 @@ */ #include "core/comm/comm_cpu.h" -#include "core/runtime/communicator_manager.h" -#include "core/runtime/launcher.h" -#include "core/runtime/runtime.h" +#include "core/runtime/detail/communicator_manager.h" +#include "core/runtime/detail/launcher.h" +#include "core/runtime/detail/runtime.h" #include "legate.h" #include "core/comm/coll.h" namespace legate::comm::cpu { -class Factory : public CommunicatorFactory { +class Factory : public detail::CommunicatorFactory { public: Factory(const LibraryContext* core_context); @@ -59,13 +59,14 @@ Legion::FutureMap Factory::initialize(const mapping::MachineDesc& machine, uint3 auto comm_id = Legion::Future::from_value(coll::collInitComm()); // Find a mapping of all participants - TaskLauncher init_cpucoll_mapping_launcher( + detail::TaskLauncher init_cpucoll_mapping_launcher( core_context_, machine, LEGATE_CORE_INIT_CPUCOLL_MAPPING_TASK_ID, tag); init_cpucoll_mapping_launcher.add_future(comm_id); auto mapping = init_cpucoll_mapping_launcher.execute(launch_domain); // Then create communicators on participating processors - TaskLauncher init_cpucoll_launcher(core_context_, machine, LEGATE_CORE_INIT_CPUCOLL_TASK_ID, tag); + detail::TaskLauncher init_cpucoll_launcher( + core_context_, machine, LEGATE_CORE_INIT_CPUCOLL_TASK_ID, tag); init_cpucoll_launcher.add_future(comm_id); init_cpucoll_launcher.set_concurrent(true); @@ -82,7 +83,7 @@ void Factory::finalize(const mapping::MachineDesc& machine, auto tag = machine.preferred_target == mapping::TaskTarget::OMP ? LEGATE_OMP_VARIANT : LEGATE_CPU_VARIANT; Domain launch_domain(Rect<1>(Point<1>(0), Point<1>(static_cast(num_tasks) - 1))); - TaskLauncher launcher(core_context_, machine, LEGATE_CORE_FINALIZE_CPUCOLL_TASK_ID, tag); + detail::TaskLauncher launcher(core_context_, machine, LEGATE_CORE_FINALIZE_CPUCOLL_TASK_ID, tag); launcher.set_concurrent(true); launcher.add_future_map(communicator); launcher.execute(launch_domain); @@ -222,7 +223,7 @@ void register_tasks(Legion::Machine machine, void register_factory(const LibraryContext* context) { - auto* comm_mgr = Runtime::get_runtime()->communicator_manager(); + auto* comm_mgr = detail::Runtime::get_runtime()->communicator_manager(); comm_mgr->register_factory("cpu", std::make_unique(context)); } diff --git a/src/core/comm/comm_nccl.cu b/src/core/comm/comm_nccl.cu index 6dad06f9a3..ecafabe496 100644 --- a/src/core/comm/comm_nccl.cu +++ b/src/core/comm/comm_nccl.cu @@ -18,9 +18,9 @@ #include "core/cuda/cuda_help.h" #include "core/cuda/stream_pool.h" #include "core/data/buffer.h" -#include "core/runtime/communicator_manager.h" -#include "core/runtime/launcher.h" -#include "core/runtime/runtime.h" +#include "core/runtime/detail/communicator_manager.h" +#include "core/runtime/detail/launcher.h" +#include "core/runtime/detail/runtime.h" #include "core/utilities/nvtx_help.h" #include "core/utilities/typedefs.h" #include "legate.h" @@ -53,7 +53,7 @@ inline void check_nccl(ncclResult_t error, const char* file, int line) } } -class Factory : public CommunicatorFactory { +class Factory : public detail::CommunicatorFactory { public: Factory(const LibraryContext* core_context); @@ -85,13 +85,13 @@ Legion::FutureMap Factory::initialize(const mapping::MachineDesc& machine, uint3 Domain launch_domain(Rect<1>(Point<1>(0), Point<1>(static_cast(num_tasks) - 1))); // Create a communicator ID - TaskLauncher init_nccl_id_launcher( + detail::TaskLauncher init_nccl_id_launcher( core_context_, machine, LEGATE_CORE_INIT_NCCL_ID_TASK_ID, LEGATE_GPU_VARIANT); init_nccl_id_launcher.set_side_effect(true); auto nccl_id = init_nccl_id_launcher.execute_single(); // Then create the communicators on participating GPUs - TaskLauncher init_nccl_launcher( + detail::TaskLauncher init_nccl_launcher( core_context_, machine, LEGATE_CORE_INIT_NCCL_TASK_ID, LEGATE_GPU_VARIANT); init_nccl_launcher.add_future(nccl_id); init_nccl_launcher.set_concurrent(true); @@ -104,7 +104,7 @@ void Factory::finalize(const mapping::MachineDesc& machine, { Domain launch_domain(Rect<1>(Point<1>(0), Point<1>(static_cast(num_tasks) - 1))); - TaskLauncher launcher( + detail::TaskLauncher launcher( core_context_, machine, LEGATE_CORE_FINALIZE_NCCL_TASK_ID, LEGATE_GPU_VARIANT); launcher.set_concurrent(true); launcher.add_future_map(communicator); @@ -235,7 +235,7 @@ bool needs_barrier() void register_factory(const LibraryContext* context) { - auto* comm_mgr = Runtime::get_runtime()->communicator_manager(); + auto* comm_mgr = detail::Runtime::get_runtime()->communicator_manager(); comm_mgr->register_factory("nccl", std::make_unique(context)); } diff --git a/src/core/data/logical_region_field.cc b/src/core/data/detail/logical_region_field.cc similarity index 92% rename from src/core/data/logical_region_field.cc rename to src/core/data/detail/logical_region_field.cc index 0829a3d682..739cdcee11 100644 --- a/src/core/data/logical_region_field.cc +++ b/src/core/data/detail/logical_region_field.cc @@ -14,11 +14,11 @@ * */ -#include "core/data/logical_region_field.h" +#include "core/data/detail/logical_region_field.h" #include "core/partitioning/partition.h" -#include "core/runtime/runtime.h" +#include "core/runtime/detail/runtime.h" -namespace legate { +namespace legate::detail { LogicalRegionField::LogicalRegionField(const Legion::LogicalRegion& lr, Legion::FieldID fid, @@ -55,4 +55,4 @@ Legion::LogicalPartition LogicalRegionField::get_legion_partition(const Partitio return partition->construct(lr_, complete); } -} // namespace legate +} // namespace legate::detail diff --git a/src/core/data/logical_region_field.h b/src/core/data/detail/logical_region_field.h similarity index 97% rename from src/core/data/logical_region_field.h rename to src/core/data/detail/logical_region_field.h index 8abc0b1c60..0accd7492f 100644 --- a/src/core/data/logical_region_field.h +++ b/src/core/data/detail/logical_region_field.h @@ -24,9 +24,11 @@ #include "core/data/shape.h" namespace legate { - class Partition; class Tiling; +} // namespace legate + +namespace legate::detail { class LogicalRegionField : public std::enable_shared_from_this { public: @@ -60,4 +62,4 @@ class LogicalRegionField : public std::enable_shared_from_this parent_{nullptr}; }; -} // namespace legate +} // namespace legate::detail diff --git a/src/core/data/logical_store_detail.cc b/src/core/data/detail/logical_store.cc similarity index 99% rename from src/core/data/logical_store_detail.cc rename to src/core/data/detail/logical_store.cc index 57a5124a04..0bec15cd2f 100644 --- a/src/core/data/logical_store_detail.cc +++ b/src/core/data/detail/logical_store.cc @@ -14,11 +14,11 @@ * */ -#include "core/data/logical_store_detail.h" +#include "core/data/detail/logical_store.h" #include "core/mapping/machine.h" -#include "core/runtime/partition_manager.h" -#include "core/runtime/req_analyzer.h" +#include "core/runtime/detail/partition_manager.h" +#include "core/runtime/detail/runtime.h" #include "core/type/type_traits.h" #include "core/utilities/buffer_builder.h" #include "core/utilities/dispatch.h" diff --git a/src/core/data/logical_store_detail.h b/src/core/data/detail/logical_store.h similarity index 98% rename from src/core/data/logical_store_detail.h rename to src/core/data/detail/logical_store.h index 5120a2b622..9ad915a6fe 100644 --- a/src/core/data/logical_store_detail.h +++ b/src/core/data/detail/logical_store.h @@ -18,11 +18,12 @@ #include -#include "core/data/logical_region_field.h" +#include "core/data/detail/logical_region_field.h" #include "core/data/slice.h" +#include "core/data/store.h" #include "core/partitioning/partition.h" #include "core/partitioning/restriction.h" -#include "core/runtime/runtime.h" +#include "core/runtime/detail/req_analyzer.h" namespace legate::mapping { class MachineDesc; @@ -30,6 +31,7 @@ class MachineDesc; namespace legate::detail { +class Projection; class StoragePartition; class LogicalStorePartition; diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index 9c81f77fd9..d06c78a4f0 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -16,12 +16,10 @@ #include +#include "core/data/detail/logical_store.h" #include "core/data/logical_store.h" -#include "core/data/logical_store_detail.h" #include "core/data/store.h" #include "core/partitioning/partition.h" -#include "core/runtime/req_analyzer.h" -#include "core/runtime/runtime.h" #include "core/type/type_traits.h" #include "core/utilities/buffer_builder.h" #include "core/utilities/dispatch.h" diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index d44e5ad8ad..663165c173 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -44,11 +44,9 @@ class LogicalStorePartition; namespace legate { -class BufferBuilder; class LibraryContext; class LogicalStorePartition; class Partition; -class Projection; class Runtime; class Store; diff --git a/src/core/data/scalar.cc b/src/core/data/scalar.cc index 04d5bbc739..9a5df97a17 100644 --- a/src/core/data/scalar.cc +++ b/src/core/data/scalar.cc @@ -28,8 +28,15 @@ Scalar::Scalar(Scalar&& other) : own_(other.own_), type_(std::move(other.type_)) other.data_ = nullptr; } -Scalar::Scalar(std::unique_ptr type, const void* data) : type_(std::move(type)), data_(data) +Scalar::Scalar(std::unique_ptr type, const void* data, bool copy) + : type_(std::move(type)), own_(copy) { + if (copy) { + auto buffer = malloc(type_->size()); + memcpy(buffer, data, type_->size()); + data_ = buffer; + } else + data_ = data; } Scalar::Scalar(const std::string& string) : own_(true), type_(string_type()) diff --git a/src/core/data/scalar.h b/src/core/data/scalar.h index bb47c86f8c..718ffd9296 100644 --- a/src/core/data/scalar.h +++ b/src/core/data/scalar.h @@ -31,17 +31,12 @@ class BufferBuilder; /** * @ingroup data - * @brief A type-erased container for scalars and tuples of scalars. + * @brief A type-erased container for scalars * * A Scalar can be owned or shared, depending on whether it owns the backing allocation: * If a `Scalar` is shared, it does not own the allocation and any of its copies are also * shared. If a `Scalar` is owned, it owns the backing allocation and releases it upon * destruction. Any copy of an owned `Scalar` is owned as well. - * - * A `Scalar` that stores a tuple of scalars has an allocation big enough to contain both - * the number of elements and the elements themselves. The number of elements should be - * stored in the first four bytes of the allocation. - * */ class Scalar { public: @@ -51,12 +46,13 @@ class Scalar { /** * @brief Creates a shared `Scalar` with an existing allocation. The caller is responsible - * for passing in a sufficiently big allocation. + * for passing in a sufficiently big allocation. When `copy` is true, the scalar copies the + * data stored in the allocation and becomes owned. * * @param type Type of the scalar(s) * @param data Allocation containing the data. */ - Scalar(std::unique_ptr type, const void* data); + Scalar(std::unique_ptr type, const void* data, bool copy = false); ~Scalar(); public: diff --git a/src/core/data/slice.h b/src/core/data/slice.h index ba0337322b..72280b8ddb 100644 --- a/src/core/data/slice.h +++ b/src/core/data/slice.h @@ -16,6 +16,7 @@ #pragma once +#include #include /** diff --git a/src/core/data/store.cc b/src/core/data/store.cc index 898c8b8770..ba825cb231 100644 --- a/src/core/data/store.cc +++ b/src/core/data/store.cc @@ -17,7 +17,7 @@ #include "core/data/store.h" #include "core/data/buffer.h" -#include "core/runtime/runtime.h" +#include "core/runtime/detail/runtime.h" #include "core/utilities/dispatch.h" #include "core/utilities/machine.h" #include "legate_defines.h" @@ -66,7 +66,7 @@ bool RegionField::valid() const Domain RegionField::domain() const { return dim_dispatch(dim_, get_domain_fn{}, pr_); } -void RegionField::unmap() { Runtime::get_runtime()->unmap_physical_region(pr_); } +void RegionField::unmap() { detail::Runtime::get_runtime()->unmap_physical_region(pr_); } UnboundRegionField::UnboundRegionField(const Legion::OutputRegion& out, Legion::FieldID fid) : out_(out), diff --git a/src/core/partitioning/constraint_solver.cc b/src/core/partitioning/constraint_solver.cc index 86f5fef8a7..cce76d0efe 100644 --- a/src/core/partitioning/constraint_solver.cc +++ b/src/core/partitioning/constraint_solver.cc @@ -18,14 +18,16 @@ #include "legion.h" -#include "core/data/logical_store_detail.h" +#include "core/data/detail/logical_store.h" #include "core/partitioning/constraint.h" #include "core/partitioning/constraint_solver.h" #include "core/runtime/operation.h" namespace legate { - extern Legion::Logger log_legate; +} // namespace legate + +namespace legate::detail { namespace { @@ -204,4 +206,4 @@ const std::vector& ConstraintSolver::partition_symbols() const const std::vector& ConstraintSolver::constraints() const { return constraints_; } -} // namespace legate +} // namespace legate::detail diff --git a/src/core/partitioning/constraint_solver.h b/src/core/partitioning/constraint_solver.h index 08064ce57f..822b125a69 100644 --- a/src/core/partitioning/constraint_solver.h +++ b/src/core/partitioning/constraint_solver.h @@ -24,7 +24,7 @@ #include "core/partitioning/restriction.h" #include "core/utilities/ordered_set.h" -namespace legate { +namespace legate::detail { struct ConstraintSolver { public: @@ -58,4 +58,4 @@ struct ConstraintSolver { std::vector> equiv_classes_{}; }; -} // namespace legate +} // namespace legate::detail diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index 0df708bec8..531ea35137 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -16,12 +16,11 @@ #include -#include "core/data/logical_store_detail.h" +#include "core/data/detail/logical_store.h" #include "core/partitioning/partition.h" -#include "core/runtime/launcher.h" -#include "core/runtime/partition_manager.h" -#include "core/runtime/req_analyzer.h" -#include "core/runtime/runtime.h" +#include "core/runtime/detail/partition_manager.h" +#include "core/runtime/detail/req_analyzer.h" +#include "core/runtime/detail/runtime.h" namespace legate { @@ -126,7 +125,7 @@ bool Tiling::satisfies_restrictions(const Restrictions& restrictions) const Legion::LogicalPartition Tiling::construct(Legion::LogicalRegion region, bool complete) const { auto index_space = region.get_index_space(); - auto runtime = Runtime::get_runtime(); + auto runtime = detail::Runtime::get_runtime(); auto part_mgr = runtime->partition_manager(); auto index_partition = part_mgr->find_index_partition(index_space, *this); if (index_partition != Legion::IndexPartition::NO_PART) @@ -224,7 +223,7 @@ bool Weighted::satisfies_restrictions(const Restrictions& restrictions) const Legion::LogicalPartition Weighted::construct(Legion::LogicalRegion region, bool) const { - auto runtime = Runtime::get_runtime(); + auto runtime = detail::Runtime::get_runtime(); auto part_mgr = runtime->partition_manager(); const auto& index_space = region.get_index_space(); diff --git a/src/core/partitioning/partition.h b/src/core/partitioning/partition.h index 4299803aa8..90cfdc1f59 100644 --- a/src/core/partitioning/partition.h +++ b/src/core/partitioning/partition.h @@ -31,8 +31,6 @@ class Storage; namespace legate { -class Projection; - struct Partition { public: enum class Kind : int32_t { diff --git a/src/core/partitioning/partitioner.cc b/src/core/partitioning/partitioner.cc index e38a6ceba2..7b7bd59c53 100644 --- a/src/core/partitioning/partitioner.cc +++ b/src/core/partitioning/partitioner.cc @@ -15,20 +15,20 @@ */ #include "core/partitioning/partitioner.h" +#include "core/data/detail/logical_store.h" #include "core/data/logical_store.h" -#include "core/data/logical_store_detail.h" #include "core/data/scalar.h" #include "core/partitioning/constraint.h" #include "core/partitioning/constraint_solver.h" #include "core/partitioning/partition.h" -#include "core/runtime/launcher.h" +#include "core/runtime/detail/launcher.h" +#include "core/runtime/detail/runtime.h" #include "core/runtime/operation.h" -#include "core/runtime/runtime.h" -namespace legate { +namespace legate::detail { //////////////////////////////////////////////////// -// legate::LaunchDomainResolver +// legate::detail::LaunchDomainResolver //////////////////////////////////////////////////// class LaunchDomainResolver { @@ -93,7 +93,7 @@ std::unique_ptr LaunchDomainResolver::resolve_launch_domain() const } //////////////////////////////////////////////////// -// legate::Strategy +// legate::detail::Strategy //////////////////////////////////////////////////// Strategy::Strategy() {} @@ -200,7 +200,7 @@ void Strategy::compute_launch_domains() } //////////////////////////////////////////////////// -// legate::Partitioner +// legate::detail::Partitioner //////////////////////////////////////////////////// Partitioner::Partitioner(std::vector&& operations) : operations_(std::move(operations)) @@ -297,4 +297,4 @@ std::vector Partitioner::handle_unbound_stores( return filtered; } -} // namespace legate +} // namespace legate::detail diff --git a/src/core/partitioning/partitioner.h b/src/core/partitioning/partitioner.h index f200b515c7..5d70a1bd6d 100644 --- a/src/core/partitioning/partitioner.h +++ b/src/core/partitioning/partitioner.h @@ -25,11 +25,14 @@ #include "legion.h" namespace legate { +class Operation; +class Partition; +} // namespace legate + +namespace legate::detail { class ConstraintSolver; class LogicalStore; -class Operation; -class Partition; class Partitioner; class Projection; @@ -84,4 +87,4 @@ class Partitioner { std::vector operations_; }; -} // namespace legate +} // namespace legate::detail diff --git a/src/core/runtime/context.cc b/src/core/runtime/context.cc index d6a9268581..350c556b5d 100644 --- a/src/core/runtime/context.cc +++ b/src/core/runtime/context.cc @@ -171,6 +171,12 @@ const TaskInfo* LibraryContext::find_task(int64_t local_task_id) const return tasks_.end() == finder ? nullptr : finder->second.get(); } +void LibraryContext::perform_callback(Legion::RegistrationWithArgsCallbackFnptr callback, + Legion::UntypedBuffer buffer) +{ + Legion::Runtime::perform_registration_callback(callback, buffer, true /*global*/); +} + TaskContext::TaskContext(const Legion::Task* task, const std::vector& regions, Legion::Context context, diff --git a/src/core/runtime/context.h b/src/core/runtime/context.h index 2948c62b7a..88f3507a9d 100644 --- a/src/core/runtime/context.h +++ b/src/core/runtime/context.h @@ -26,7 +26,6 @@ #include "core/comm/communicator.h" #include "core/mapping/machine.h" #include "core/runtime/resource.h" -#include "core/runtime/runtime.h" #include "core/task/return.h" #include "core/task/task_info.h" #include "core/utilities/typedefs.h" @@ -40,11 +39,17 @@ namespace legate::mapping { class Mapper; } // namespace legate::mapping +namespace legate::detail { +class Runtime; +} // namespace legate::detail + namespace legate { class Store; class Scalar; +extern Legion::Logger log_legate; + class InvalidTaskIdException : public std::exception { public: InvalidTaskIdException(const std::string& library_name, @@ -64,7 +69,7 @@ class InvalidTaskIdException : public std::exception { */ class LibraryContext { private: - friend class Runtime; + friend class detail::Runtime; LibraryContext(const std::string& library_name, const ResourceConfig& config, std::unique_ptr mapper); @@ -174,6 +179,10 @@ class LibraryContext { void register_task(int64_t local_task_id, std::unique_ptr task_info); const TaskInfo* find_task(int64_t local_task_id) const; + private: + void perform_callback(Legion::RegistrationWithArgsCallbackFnptr callback, + Legion::UntypedBuffer buffer); + private: Legion::Runtime* runtime_; const std::string library_name_; diff --git a/src/core/runtime/context.inl b/src/core/runtime/context.inl index 6de76c2e68..fe329ebf52 100644 --- a/src/core/runtime/context.inl +++ b/src/core/runtime/context.inl @@ -19,16 +19,10 @@ // Useful for IDEs #include "core/runtime/context.h" -namespace legate { +namespace legate::detail { #ifndef REALM_COMPILER_IS_NVCC -#ifdef LEGATE_USE_CUDA -extern Legion::Logger log_legate; -#endif - -namespace detail { - template void register_reduction_callback(const Legion::RegistrationCallbackArgs& args) { @@ -36,35 +30,8 @@ void register_reduction_callback(const Legion::RegistrationCallbackArgs& args) Legion::Runtime::register_reduction_op(legion_redop_id); } -} // namespace detail - -template -int32_t LibraryContext::register_reduction_operator(int32_t redop_id) -{ - int32_t legion_redop_id = get_reduction_op_id(redop_id); -#ifdef LEGATE_USE_CUDA - log_legate.error("Reduction operators must be registered in a .cu file when CUDA is enabled"); - LEGATE_ABORT; -#endif - auto runtime = Runtime::get_runtime(); - if (runtime->is_in_callback()) - Legion::Runtime::register_reduction_op(legion_redop_id); - else { - runtime->enter_callback(); - Legion::Runtime::perform_registration_callback( - detail::register_reduction_callback, - Legion::UntypedBuffer(&legion_redop_id, sizeof(int32_t)), - true /*global*/); - runtime->exit_callback(); - } - - return legion_redop_id; -} - #else // ifndef REALM_COMPILER_IS_NVCC -namespace detail { - template class CUDAReductionOpWrapper : public T { public: @@ -95,31 +62,23 @@ void register_reduction_callback(const Legion::RegistrationCallbackArgs& args) false); } -} // namespace detail +#endif // ifndef REALM_COMPILER_IS_NVCC + +} // namespace legate::detail + +namespace legate { template int32_t LibraryContext::register_reduction_operator(int32_t redop_id) { int32_t legion_redop_id = get_reduction_op_id(redop_id); - auto runtime = Runtime::get_runtime(); - if (runtime->is_in_callback()) - Legion::Runtime::register_reduction_op( - legion_redop_id, - Realm::ReductionOpUntyped::create_reduction_op>(), - nullptr, - nullptr, - false); - else { - runtime->enter_callback(); - Legion::Runtime::perform_registration_callback( - detail::register_reduction_callback, - Legion::UntypedBuffer(&legion_redop_id, sizeof(int32_t)), - true /*global*/); - runtime->exit_callback(); - } +#if defined(LEGATE_USE_CUDA) && !defined(REALM_COMPILER_IS_NVCC) + log_legate.error("Reduction operators must be registered in a .cu file when CUDA is enabled"); + LEGATE_ABORT; +#endif + perform_callback(detail::register_reduction_callback, + Legion::UntypedBuffer(&legion_redop_id, sizeof(int32_t))); return legion_redop_id; } -#endif // ifndef REALM_COMPILER_IS_NVCC - } // namespace legate diff --git a/src/core/runtime/communicator_manager.cc b/src/core/runtime/detail/communicator_manager.cc similarity index 91% rename from src/core/runtime/communicator_manager.cc rename to src/core/runtime/detail/communicator_manager.cc index 1ca86b8f6f..62b8cbeb16 100644 --- a/src/core/runtime/communicator_manager.cc +++ b/src/core/runtime/detail/communicator_manager.cc @@ -14,12 +14,12 @@ * */ -#include "core/runtime/communicator_manager.h" +#include "core/runtime/detail/communicator_manager.h" -#include "core/runtime/machine_manager.h" -#include "core/runtime/runtime.h" +#include "core/runtime/detail/machine_manager.h" +#include "core/runtime/detail/runtime.h" -namespace legate { +namespace legate::detail { CommunicatorFactory::CommunicatorFactory() {} @@ -45,6 +45,8 @@ void CommunicatorFactory::destroy() { for (auto& [key, communicator] : communicators_) finalize(key.get_machine(), key.desc, communicator); + communicators_.clear(); + nd_aliases_.clear(); } Legion::FutureMap CommunicatorFactory::find_or_create(const mapping::TaskTarget& target, @@ -82,6 +84,7 @@ void CommunicatorManager::register_factory(const std::string& name, void CommunicatorManager::destroy() { for (auto& [_, factory] : factories_) factory->destroy(); + factories_.clear(); } -} // namespace legate +} // namespace legate::detail diff --git a/src/core/runtime/communicator_manager.h b/src/core/runtime/detail/communicator_manager.h similarity index 95% rename from src/core/runtime/communicator_manager.h rename to src/core/runtime/detail/communicator_manager.h index 76c2c75ace..ec6d3498f1 100644 --- a/src/core/runtime/communicator_manager.h +++ b/src/core/runtime/detail/communicator_manager.h @@ -22,8 +22,9 @@ #include "core/mapping/machine.h" #include "core/utilities/typedefs.h" -namespace legate { +namespace legate::detail { +// TODO: We need to expose this eventually so client libraries can register custom communicators class CommunicatorFactory { protected: CommunicatorFactory(); @@ -95,4 +96,4 @@ class CommunicatorManager { std::unordered_map> factories_; }; -} // namespace legate +} // namespace legate::detail diff --git a/src/core/runtime/field_manager.cc b/src/core/runtime/detail/field_manager.cc similarity index 90% rename from src/core/runtime/field_manager.cc rename to src/core/runtime/detail/field_manager.cc index a0de6b1f97..c167e5b418 100644 --- a/src/core/runtime/field_manager.cc +++ b/src/core/runtime/detail/field_manager.cc @@ -14,13 +14,13 @@ * */ -#include "core/runtime/field_manager.h" +#include "core/runtime/detail/field_manager.h" -#include "core/data/logical_region_field.h" -#include "core/runtime/region_manager.h" -#include "core/runtime/runtime.h" +#include "core/data/detail/logical_region_field.h" +#include "core/runtime/detail/region_manager.h" +#include "core/runtime/detail/runtime.h" -namespace legate { +namespace legate::detail { FieldManager::FieldManager(Runtime* runtime, const Domain& shape, uint32_t field_size) : runtime_(runtime), shape_(shape), field_size_(field_size) @@ -66,4 +66,4 @@ std::shared_ptr FieldManager::import_field(const Legion::Log }); } -} // namespace legate +} // namespace legate::detail diff --git a/src/core/runtime/field_manager.h b/src/core/runtime/detail/field_manager.h similarity index 95% rename from src/core/runtime/field_manager.h rename to src/core/runtime/detail/field_manager.h index 3569ec9b02..3e5384ff73 100644 --- a/src/core/runtime/field_manager.h +++ b/src/core/runtime/detail/field_manager.h @@ -20,7 +20,7 @@ #include "core/utilities/typedefs.h" -namespace legate { +namespace legate::detail { class LogicalRegionField; class Runtime; @@ -44,4 +44,4 @@ class FieldManager { std::deque free_fields_; }; -} // namespace legate +} // namespace legate::detail diff --git a/src/core/runtime/launcher.cc b/src/core/runtime/detail/launcher.cc similarity index 94% rename from src/core/runtime/launcher.cc rename to src/core/runtime/detail/launcher.cc index 9a07a7a363..d20c6f0d3f 100644 --- a/src/core/runtime/launcher.cc +++ b/src/core/runtime/detail/launcher.cc @@ -14,21 +14,19 @@ * */ -#include "core/runtime/launcher.h" -#include "core/data/logical_region_field.h" -#include "core/data/logical_store.h" -#include "core/data/logical_store_detail.h" +#include "core/runtime/detail/launcher.h" +#include "core/data/detail/logical_region_field.h" +#include "core/data/detail/logical_store.h" #include "core/data/scalar.h" #include "core/mapping/machine.h" #include "core/runtime/context.h" -#include "core/runtime/launcher_arg.h" -#include "core/runtime/partition_manager.h" -#include "core/runtime/req_analyzer.h" -#include "core/runtime/runtime.h" +#include "core/runtime/detail/launcher_arg.h" +#include "core/runtime/detail/partition_manager.h" +#include "core/runtime/detail/req_analyzer.h" +#include "core/runtime/detail/runtime.h" #include "core/utilities/buffer_builder.h" -#include "core/utilities/dispatch.h" -namespace legate { +namespace legate::detail { TaskLauncher::TaskLauncher(const LibraryContext* library, const mapping::MachineDesc& machine, @@ -79,7 +77,7 @@ void TaskLauncher::add_scalar(const Scalar& scalar) scalars_.push_back(new UntypedScalarArg(scalar)); } -void TaskLauncher::add_input(detail::LogicalStore* store, +void TaskLauncher::add_input(LogicalStore* store, std::unique_ptr proj, Legion::MappingTagID tag, Legion::RegionFlags flags) @@ -87,7 +85,7 @@ void TaskLauncher::add_input(detail::LogicalStore* store, add_store(inputs_, store, std::move(proj), READ_ONLY, tag, flags); } -void TaskLauncher::add_output(detail::LogicalStore* store, +void TaskLauncher::add_output(LogicalStore* store, std::unique_ptr proj, Legion::MappingTagID tag, Legion::RegionFlags flags) @@ -95,7 +93,7 @@ void TaskLauncher::add_output(detail::LogicalStore* store, add_store(outputs_, store, std::move(proj), WRITE_ONLY, tag, flags); } -void TaskLauncher::add_reduction(detail::LogicalStore* store, +void TaskLauncher::add_reduction(LogicalStore* store, std::unique_ptr proj, bool read_write, Legion::MappingTagID tag, @@ -107,7 +105,7 @@ void TaskLauncher::add_reduction(detail::LogicalStore* store, add_store(reductions_, store, std::move(proj), REDUCE, tag, flags); } -void TaskLauncher::add_unbound_output(detail::LogicalStore* store, +void TaskLauncher::add_unbound_output(LogicalStore* store, Legion::FieldSpace field_space, Legion::FieldID field_id) { @@ -156,7 +154,7 @@ Legion::Future TaskLauncher::execute_single() } void TaskLauncher::add_store(std::vector& args, - detail::LogicalStore* store, + LogicalStore* store, std::unique_ptr proj, Legion::PrivilegeMode privilege, Legion::MappingTagID tag, @@ -346,4 +344,4 @@ void TaskLauncher::post_process_unbound_stores(const Legion::FutureMap& result, } } -} // namespace legate +} // namespace legate::detail diff --git a/src/core/runtime/launcher.h b/src/core/runtime/detail/launcher.h similarity index 94% rename from src/core/runtime/launcher.h rename to src/core/runtime/detail/launcher.h index fdfa23b042..ec21d5304b 100644 --- a/src/core/runtime/launcher.h +++ b/src/core/runtime/detail/launcher.h @@ -22,21 +22,20 @@ #include "core/mapping/machine.h" -namespace legate::detail { -class LogicalStore; -} // namespace legate::detail - namespace legate { - -class ArgWrapper; class BufferBuilder; class LibraryContext; +class Scalar; +} // namespace legate + +namespace legate::detail { + +class ArgWrapper; class LogicalStore; class Projection; class OutputRegionArg; class OutputRequirementAnalyzer; class RequirementAnalyzer; -class Scalar; class TaskLauncher { public: @@ -60,20 +59,20 @@ class TaskLauncher { public: void add_scalar(const Scalar& scalar); - void add_input(detail::LogicalStore* store, + void add_input(LogicalStore* store, std::unique_ptr proj, Legion::MappingTagID tag = 0, Legion::RegionFlags flags = LEGION_NO_FLAG); - void add_output(detail::LogicalStore* store, + void add_output(LogicalStore* store, std::unique_ptr proj, Legion::MappingTagID tag = 0, Legion::RegionFlags flags = LEGION_NO_FLAG); - void add_reduction(detail::LogicalStore* store, + void add_reduction(LogicalStore* store, std::unique_ptr proj, bool read_write = false, Legion::MappingTagID tag = 0, Legion::RegionFlags flags = LEGION_NO_FLAG); - void add_unbound_output(detail::LogicalStore* store, + void add_unbound_output(LogicalStore* store, Legion::FieldSpace field_space, Legion::FieldID field_id); @@ -90,7 +89,7 @@ class TaskLauncher { private: void add_store(std::vector& args, - detail::LogicalStore* store, + LogicalStore* store, std::unique_ptr proj, Legion::PrivilegeMode privilege, Legion::MappingTagID tag, @@ -141,4 +140,4 @@ class TaskLauncher { std::vector output_requirements_; }; -} // namespace legate +} // namespace legate::detail diff --git a/src/core/runtime/launcher_arg.cc b/src/core/runtime/detail/launcher_arg.cc similarity index 86% rename from src/core/runtime/launcher_arg.cc rename to src/core/runtime/detail/launcher_arg.cc index 66754dfa2a..18b067e663 100644 --- a/src/core/runtime/launcher_arg.cc +++ b/src/core/runtime/detail/launcher_arg.cc @@ -14,17 +14,17 @@ * */ -#include "core/runtime/launcher_arg.h" -#include "core/data/logical_region_field.h" -#include "core/runtime/launcher.h" -#include "core/runtime/req_analyzer.h" +#include "core/runtime/detail/launcher_arg.h" +#include "core/data/detail/logical_region_field.h" +#include "core/runtime/detail/launcher.h" +#include "core/runtime/detail/req_analyzer.h" -namespace legate { +namespace legate::detail { void UntypedScalarArg::pack(BufferBuilder& buffer) const { scalar_.pack(buffer); } RegionFieldArg::RegionFieldArg(RequirementAnalyzer* analyzer, - detail::LogicalStore* store, + LogicalStore* store, Legion::FieldID field_id, Legion::PrivilegeMode privilege, const ProjectionInfo* proj_info) @@ -48,7 +48,7 @@ void RegionFieldArg::pack(BufferBuilder& buffer) const } OutputRegionArg::OutputRegionArg(OutputRequirementAnalyzer* analyzer, - detail::LogicalStore* store, + LogicalStore* store, Legion::FieldSpace field_space, Legion::FieldID field_id) : analyzer_(analyzer), store_(store), field_space_(field_space), field_id_(field_id) @@ -67,7 +67,7 @@ void OutputRegionArg::pack(BufferBuilder& buffer) const buffer.pack(field_id_); } -FutureStoreArg::FutureStoreArg(detail::LogicalStore* store, +FutureStoreArg::FutureStoreArg(LogicalStore* store, bool read_only, bool has_storage, Legion::ReductionOpID redop) @@ -86,4 +86,4 @@ void FutureStoreArg::pack(BufferBuilder& buffer) const buffer.pack(store_->get_storage()->extents().data()); } -} // namespace legate +} // namespace legate::detail diff --git a/src/core/runtime/launcher_arg.h b/src/core/runtime/detail/launcher_arg.h similarity index 88% rename from src/core/runtime/launcher_arg.h rename to src/core/runtime/detail/launcher_arg.h index 8378b114ce..2d1ba8c895 100644 --- a/src/core/runtime/launcher_arg.h +++ b/src/core/runtime/detail/launcher_arg.h @@ -16,11 +16,11 @@ #pragma once -#include "core/data/logical_store_detail.h" +#include "core/data/detail/logical_store.h" #include "core/data/scalar.h" #include "core/utilities/buffer_builder.h" -namespace legate { +namespace legate::detail { class OutputRequirementAnalyzer; class ProjectionInfo; @@ -63,7 +63,7 @@ struct UntypedScalarArg : public ArgWrapper { struct RegionFieldArg : public ArgWrapper { public: RegionFieldArg(RequirementAnalyzer* analyzer, - detail::LogicalStore* store, + LogicalStore* store, Legion::FieldID field_id, Legion::PrivilegeMode privilege, const ProjectionInfo* proj_info); @@ -76,7 +76,7 @@ struct RegionFieldArg : public ArgWrapper { private: RequirementAnalyzer* analyzer_; - detail::LogicalStore* store_; + LogicalStore* store_; Legion::LogicalRegion region_; Legion::FieldID field_id_; Legion::PrivilegeMode privilege_; @@ -86,7 +86,7 @@ struct RegionFieldArg : public ArgWrapper { struct OutputRegionArg : public ArgWrapper { public: OutputRegionArg(OutputRequirementAnalyzer* analyzer, - detail::LogicalStore* store, + LogicalStore* store, Legion::FieldSpace field_space, Legion::FieldID field_id); @@ -97,14 +97,14 @@ struct OutputRegionArg : public ArgWrapper { virtual ~OutputRegionArg() {} public: - detail::LogicalStore* store() const { return store_; } + LogicalStore* store() const { return store_; } const Legion::FieldSpace& field_space() const { return field_space_; } Legion::FieldID field_id() const { return field_id_; } uint32_t requirement_index() const { return requirement_index_; } private: OutputRequirementAnalyzer* analyzer_; - detail::LogicalStore* store_; + LogicalStore* store_; Legion::FieldSpace field_space_; Legion::FieldID field_id_; mutable uint32_t requirement_index_{-1U}; @@ -112,7 +112,7 @@ struct OutputRegionArg : public ArgWrapper { struct FutureStoreArg : public ArgWrapper { public: - FutureStoreArg(detail::LogicalStore* store, + FutureStoreArg(LogicalStore* store, bool read_only, bool has_storage, Legion::ReductionOpID redop); @@ -124,10 +124,10 @@ struct FutureStoreArg : public ArgWrapper { virtual void pack(BufferBuilder& buffer) const override; private: - detail::LogicalStore* store_; + LogicalStore* store_; bool read_only_; bool has_storage_; Legion::ReductionOpID redop_; }; -} // namespace legate +} // namespace legate::detail diff --git a/src/core/runtime/machine_manager.cc b/src/core/runtime/detail/machine_manager.cc similarity index 92% rename from src/core/runtime/machine_manager.cc rename to src/core/runtime/detail/machine_manager.cc index afcb08a14f..3ed0f00272 100644 --- a/src/core/runtime/machine_manager.cc +++ b/src/core/runtime/detail/machine_manager.cc @@ -14,9 +14,9 @@ * */ -#include "core/runtime/machine_manager.h" +#include "core/runtime/detail/machine_manager.h" -namespace legate { +namespace legate::detail { //////////////////////////////////////////// // legate::MachineManager @@ -45,4 +45,4 @@ void MachineManager::pop_machine() machines_.pop_back(); } -} // namespace legate +} // namespace legate::detail diff --git a/src/core/runtime/machine_manager.h b/src/core/runtime/detail/machine_manager.h similarity index 94% rename from src/core/runtime/machine_manager.h rename to src/core/runtime/detail/machine_manager.h index d086521eaa..8294e04988 100644 --- a/src/core/runtime/machine_manager.h +++ b/src/core/runtime/detail/machine_manager.h @@ -22,7 +22,7 @@ namespace legate::mapping { class MachineDesc; } // namespace legate::mapping -namespace legate { +namespace legate::detail { class MachineManager { public: @@ -37,4 +37,4 @@ class MachineManager { std::vector machines_; }; -} // namespace legate +} // namespace legate::detail diff --git a/src/core/runtime/partition_manager.cc b/src/core/runtime/detail/partition_manager.cc similarity index 98% rename from src/core/runtime/partition_manager.cc rename to src/core/runtime/detail/partition_manager.cc index be650a8e02..f468832e9a 100644 --- a/src/core/runtime/partition_manager.cc +++ b/src/core/runtime/detail/partition_manager.cc @@ -14,15 +14,15 @@ * */ -#include "core/runtime/partition_manager.h" +#include "core/runtime/detail/partition_manager.h" #include "core/legate_c.h" #include "core/mapping/machine.h" #include "core/partitioning/partition.h" #include "core/runtime/context.h" -#include "core/runtime/runtime.h" +#include "core/runtime/detail/runtime.h" -namespace legate { +namespace legate::detail { PartitionManager::PartitionManager(Runtime* runtime, const LibraryContext* context) { @@ -249,4 +249,4 @@ void PartitionManager::record_index_partition(const Legion::IndexSpace& index_sp weighted_cache_[std::make_pair(index_space, weighted)] = index_partition; } -} // namespace legate +} // namespace legate::detail diff --git a/src/core/runtime/partition_manager.h b/src/core/runtime/detail/partition_manager.h similarity index 97% rename from src/core/runtime/partition_manager.h rename to src/core/runtime/detail/partition_manager.h index 4ad8b93354..06cbad6749 100644 --- a/src/core/runtime/partition_manager.h +++ b/src/core/runtime/detail/partition_manager.h @@ -23,16 +23,19 @@ #include "core/data/shape.h" #include "core/partitioning/restriction.h" +namespace legate { +class LibraryContext; +class Tiling; +class Weighted; +} // namespace legate + namespace legate::mapping { class MachineDesc; } // namespace legate::mapping -namespace legate { +namespace legate::detail { -class LibraryContext; class Runtime; -class Tiling; -class Weighted; class PartitionManager { public: @@ -71,4 +74,4 @@ class PartitionManager { std::map weighted_cache_; }; -} // namespace legate +} // namespace legate::detail diff --git a/src/core/runtime/provenance_manager.cc b/src/core/runtime/detail/provenance_manager.cc similarity index 93% rename from src/core/runtime/provenance_manager.cc rename to src/core/runtime/detail/provenance_manager.cc index a37768c34e..5603ed9f35 100644 --- a/src/core/runtime/provenance_manager.cc +++ b/src/core/runtime/detail/provenance_manager.cc @@ -14,12 +14,12 @@ * */ -#include "core/runtime/provenance_manager.h" +#include "core/runtime/detail/provenance_manager.h" #include #include -namespace legate { +namespace legate::detail { static const std::string BOTTOM = ""; @@ -65,4 +65,4 @@ void ProvenanceManager::clear_all() provenance_.push_back(BOTTOM); } -} // namespace legate +} // namespace legate::detail diff --git a/src/core/runtime/provenance_manager.h b/src/core/runtime/detail/provenance_manager.h similarity index 94% rename from src/core/runtime/provenance_manager.h rename to src/core/runtime/detail/provenance_manager.h index b105dc7332..d478365626 100644 --- a/src/core/runtime/provenance_manager.h +++ b/src/core/runtime/detail/provenance_manager.h @@ -19,7 +19,7 @@ #include #include -namespace legate { +namespace legate::detail { class ProvenanceManager { public: @@ -42,4 +42,4 @@ class ProvenanceManager { std::vector provenance_; }; -} // namespace legate +} // namespace legate::detail diff --git a/src/core/runtime/region_manager.cc b/src/core/runtime/detail/region_manager.cc similarity index 78% rename from src/core/runtime/region_manager.cc rename to src/core/runtime/detail/region_manager.cc index 1306f95209..ed982f49b2 100644 --- a/src/core/runtime/region_manager.cc +++ b/src/core/runtime/detail/region_manager.cc @@ -14,16 +14,26 @@ * */ -#include "core/runtime/region_manager.h" -#include "core/runtime/runtime.h" +#include "core/runtime/detail/region_manager.h" +#include "core/runtime/detail/runtime.h" -namespace legate { +namespace legate::detail { + +void RegionManager::ManagerEntry::destroy(Runtime* runtime, bool unordered) +{ + runtime->destroy_region(region, unordered); +} RegionManager::RegionManager(Runtime* runtime, const Domain& shape) : runtime_(runtime), shape_(shape) { } +void RegionManager::destroy(bool unordered) +{ + for (auto& entry : entries_) entry.destroy(runtime_, unordered); +} + void RegionManager::push_entry() { auto is = runtime_->find_or_create_index_space(shape_); @@ -47,4 +57,4 @@ void RegionManager::import_region(const Legion::LogicalRegion& region) entries_.emplace_back(region); } -} // namespace legate +} // namespace legate::detail diff --git a/src/core/runtime/region_manager.h b/src/core/runtime/detail/region_manager.h similarity index 92% rename from src/core/runtime/region_manager.h rename to src/core/runtime/detail/region_manager.h index 51362c9c9c..bcbe024e87 100644 --- a/src/core/runtime/region_manager.h +++ b/src/core/runtime/detail/region_manager.h @@ -18,7 +18,7 @@ #include "core/utilities/typedefs.h" -namespace legate { +namespace legate::detail { class Runtime; @@ -35,12 +35,15 @@ class RegionManager { bool has_space() const { return next_field_id - FIELD_ID_BASE < MAX_NUM_FIELDS; } Legion::FieldID get_next_field_id() { return next_field_id++; } + void destroy(Runtime* runtime, bool unordered); + Legion::LogicalRegion region; Legion::FieldID next_field_id; }; public: RegionManager(Runtime* runtime, const Domain& shape); + void destroy(bool unordered = false); private: const ManagerEntry& active_entry() const { return entries_.back(); } @@ -58,4 +61,4 @@ class RegionManager { std::vector entries_{}; }; -} // namespace legate +} // namespace legate::detail diff --git a/src/core/runtime/req_analyzer.cc b/src/core/runtime/detail/req_analyzer.cc similarity index 98% rename from src/core/runtime/req_analyzer.cc rename to src/core/runtime/detail/req_analyzer.cc index 224e66b014..a1d97a08e3 100644 --- a/src/core/runtime/req_analyzer.cc +++ b/src/core/runtime/detail/req_analyzer.cc @@ -14,11 +14,10 @@ * */ -#include "core/runtime/req_analyzer.h" -#include "core/runtime/launcher.h" -#include "core/runtime/runtime.h" +#include "core/runtime/detail/req_analyzer.h" +#include "core/runtime/detail/runtime.h" -namespace legate { +namespace legate::detail { ///////////// // Projection @@ -304,4 +303,4 @@ void OutputRequirementAnalyzer::populate_output_requirements( } } -} // namespace legate +} // namespace legate::detail diff --git a/src/core/runtime/req_analyzer.h b/src/core/runtime/detail/req_analyzer.h similarity index 98% rename from src/core/runtime/req_analyzer.h rename to src/core/runtime/detail/req_analyzer.h index 8acfb6a7c7..8e152b89d8 100644 --- a/src/core/runtime/req_analyzer.h +++ b/src/core/runtime/detail/req_analyzer.h @@ -19,7 +19,7 @@ #include #include "legate.h" -namespace legate { +namespace legate::detail { class Projection { public: @@ -147,4 +147,4 @@ class OutputRequirementAnalyzer { std::map req_infos_; }; -} // namespace legate +} // namespace legate::detail diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc new file mode 100644 index 0000000000..32e430572e --- /dev/null +++ b/src/core/runtime/detail/runtime.cc @@ -0,0 +1,902 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/runtime/detail/runtime.h" + +#include "core/comm/comm.h" +#include "core/data/detail/logical_region_field.h" +#include "core/data/detail/logical_store.h" +#include "core/mapping/core_mapper.h" +#include "core/mapping/default_mapper.h" +#include "core/partitioning/partitioner.h" +#include "core/runtime/detail/launcher.h" +#include "core/runtime/projection.h" +#include "core/runtime/shard.h" +#include "env_defaults.h" + +namespace legate { + +Logger log_legate("legate"); + +const char* const core_library_name = "legate.core"; + +} // namespace legate + +namespace legate::detail { + +namespace { + +constexpr uint32_t CUSTOM_TYPE_UID_BASE = 1000; +const char* TOPLEVEL_NAME = "Legate Core Toplevel Task"; + +} // namespace + +Runtime::Runtime() + : legion_runtime_(Legion::Runtime::get_runtime()), + next_type_uid_(CUSTOM_TYPE_UID_BASE), + max_pending_exceptions_(extract_env( + "LEGATE_MAX_PENDING_EXCEPTIONS", MAX_PENDING_EXCEPTIONS_DEFAULT, MAX_PENDING_EXCEPTIONS_TEST)) +{ +} + +Runtime::~Runtime() {} + +LibraryContext* Runtime::find_library(const std::string& library_name, + bool can_fail /*=false*/) const +{ + auto finder = libraries_.find(library_name); + if (libraries_.end() == finder) { + if (!can_fail) throw std::out_of_range("Library " + library_name + " does not exist"); + return nullptr; + } + return finder->second; +} + +LibraryContext* Runtime::create_library(const std::string& library_name, + const ResourceConfig& config, + std::unique_ptr mapper) +{ + if (libraries_.find(library_name) != libraries_.end()) + throw std::invalid_argument("Library " + library_name + " already exists"); + + log_legate.debug("Library %s is created", library_name.c_str()); + if (nullptr == mapper) mapper = std::make_unique(); + auto context = new LibraryContext(library_name, config, std::move(mapper)); + libraries_[library_name] = context; + return context; +} + +uint32_t Runtime::get_type_uid() { return next_type_uid_++; } + +void Runtime::record_reduction_operator(int32_t type_uid, int32_t op_kind, int32_t legion_op_id) +{ +#ifdef DEBUG_LEGATE + log_legate.debug("Record reduction op (type_uid: %d, op_kind: %d, legion_op_id: %d)", + type_uid, + op_kind, + legion_op_id); +#endif + auto key = std::make_pair(type_uid, op_kind); + auto finder = reduction_ops_.find(key); + if (finder != reduction_ops_.end()) { + std::stringstream ss; + ss << "Reduction op " << op_kind << " already exists for type " << type_uid; + throw std::invalid_argument(std::move(ss).str()); + } + reduction_ops_.emplace(std::make_pair(key, legion_op_id)); +} + +int32_t Runtime::find_reduction_operator(int32_t type_uid, int32_t op_kind) const +{ + auto key = std::make_pair(type_uid, op_kind); + auto finder = reduction_ops_.find(key); + if (reduction_ops_.end() == finder) { +#ifdef DEBUG_LEGATE + log_legate.debug("Can't find reduction op (type_uid: %d, op_kind: %d)", type_uid, op_kind); +#endif + std::stringstream ss; + ss << "Reduction op " << op_kind << " does not exist for type " << type_uid; + throw std::invalid_argument(std::move(ss).str()); + } +#ifdef DEBUG_LEGATE + log_legate.debug( + "Found reduction op %d (type_uid: %d, op_kind: %d)", finder->second, type_uid, op_kind); +#endif + return finder->second; +} + +void Runtime::enter_callback() { in_callback_ = true; } + +void Runtime::exit_callback() { in_callback_ = false; } + +bool Runtime::is_in_callback() const { return in_callback_; } + +void Runtime::initialize(Legion::Context legion_context) +{ + if (initialized_) throw std::runtime_error("Legate runtime has already been initialized"); + initialized_ = true; + legion_context_ = legion_context; + core_context_ = find_library(core_library_name); + communicator_manager_ = new CommunicatorManager(); + partition_manager_ = new PartitionManager(this, core_context_); + machine_manager_ = new MachineManager(); + provenance_manager_ = new ProvenanceManager(); + Core::retrieve_tunable(legion_context_, legion_runtime_, core_context_); + initialize_toplevel_machine(); + comm::register_builtin_communicator_factories(core_context_); +} + +mapping::MachineDesc Runtime::slice_machine_for_task(LibraryContext* library, int64_t task_id) +{ + auto task_info = library->find_task(task_id); + if (nullptr == task_info) { + std::stringstream ss; + ss << "Library " << library->get_library_name() << " does not have task " << task_id; + throw std::invalid_argument(std::move(ss).str()); + } + + std::vector task_targets; + auto& machine = machine_manager_->get_machine(); + for (const auto& t : machine.valid_targets()) { + if (task_info->has_variant(mapping::to_variant_code(t))) task_targets.push_back(t); + } + auto sliced = machine.only(task_targets); + + if (sliced.empty()) { + std::stringstream ss; + ss << "Task " << task_id << " (" << task_info->name() << ") of library " + << library->get_library_name() << " does not have any valid variant for " + << "the current machine configuration."; + throw std::invalid_argument(ss.str()); + } + return sliced; +} + +// This function should be moved to the library context +std::unique_ptr Runtime::create_task(LibraryContext* library, int64_t task_id) +{ + auto machine = slice_machine_for_task(library, task_id); + auto task = new AutoTask(library, task_id, next_unique_id_++, std::move(machine)); + return std::unique_ptr(task); +} + +std::unique_ptr Runtime::create_task(LibraryContext* library, + int64_t task_id, + const Shape& launch_shape) +{ + auto machine = slice_machine_for_task(library, task_id); + auto task = new ManualTask(library, task_id, launch_shape, next_unique_id_++, std::move(machine)); + return std::unique_ptr(task); +} + +void Runtime::flush_scheduling_window() +{ + if (operations_.size() == 0) return; + + std::vector> to_schedule; + to_schedule.swap(operations_); + schedule(std::move(to_schedule)); +} + +void Runtime::submit(std::unique_ptr op) +{ + operations_.push_back(std::move(op)); + if (operations_.size() >= window_size_) { flush_scheduling_window(); } +} + +void Runtime::schedule(std::vector> operations) +{ + std::vector op_pointers{}; + op_pointers.reserve(operations.size()); + for (auto& op : operations) op_pointers.push_back(op.get()); + + Partitioner partitioner(std::move(op_pointers)); + auto strategy = partitioner.partition_stores(); + + for (auto& op : operations) op->launch(strategy.get()); +} + +std::shared_ptr Runtime::create_store(std::unique_ptr type, int32_t dim) +{ + auto storage = std::make_shared(dim, std::move(type)); + return std::make_shared(std::move(storage)); +} + +std::shared_ptr Runtime::create_store(const Shape& extents, + std::unique_ptr type, + bool optimize_scalar /*=false*/) +{ + auto storage = std::make_shared(extents, std::move(type), optimize_scalar); + return std::make_shared(std::move(storage)); +} + +std::shared_ptr Runtime::create_store(const Scalar& scalar) +{ + Shape extents{1}; + auto future = create_future(scalar.ptr(), scalar.size()); + auto storage = std::make_shared(extents, scalar.type().clone(), future); + return std::make_shared(std::move(storage)); +} + +uint32_t Runtime::max_pending_exceptions() const { return max_pending_exceptions_; } + +void Runtime::set_max_pending_exceptions(uint32_t max_pending_exceptions) +{ + uint32_t old_value = max_pending_exceptions_; + max_pending_exceptions_ = max_pending_exceptions; + if (old_value > max_pending_exceptions_) raise_pending_task_exception(); +} + +void Runtime::raise_pending_task_exception() +{ + auto exn = check_pending_task_exception(); + if (exn.has_value()) throw exn.value(); +} + +std::optional Runtime::check_pending_task_exception() +{ + // If there's already an outstanding exception from the previous scan, we just return that. + if (!outstanding_exceptions_.empty()) { + std::optional result = outstanding_exceptions_.front(); + outstanding_exceptions_.pop_front(); + return result; + } + + // Othrewise, we unpack all pending exceptions and push them to the outstanding exception queue + for (auto& pending_exception : pending_exceptions_) { + auto returned_exception = pending_exception.get_result(); + auto result = returned_exception.to_task_exception(); + if (result.has_value()) outstanding_exceptions_.push_back(result.value()); + } + pending_exceptions_.clear(); + return outstanding_exceptions_.empty() ? std::nullopt : check_pending_task_exception(); +} + +void Runtime::record_pending_exception(const Legion::Future& pending_exception) +{ + pending_exceptions_.push_back(pending_exception); + if (outstanding_exceptions_.size() + pending_exceptions_.size() >= max_pending_exceptions_) + raise_pending_task_exception(); +} + +uint64_t Runtime::get_unique_store_id() { return next_store_id_++; } + +uint64_t Runtime::get_unique_storage_id() { return next_storage_id_++; } + +std::shared_ptr Runtime::create_region_field(const Shape& extents, + uint32_t field_size) +{ + DomainPoint lo, hi; + hi.dim = lo.dim = static_cast(extents.size()); + assert(lo.dim <= LEGION_MAX_DIM); + for (int32_t dim = 0; dim < lo.dim; ++dim) lo[dim] = 0; + for (int32_t dim = 0; dim < lo.dim; ++dim) hi[dim] = extents[dim] - 1; + + Domain shape(lo, hi); + auto fld_mgr = find_or_create_field_manager(shape, field_size); + return fld_mgr->allocate_field(); +} + +std::shared_ptr Runtime::import_region_field(Legion::LogicalRegion region, + Legion::FieldID field_id, + uint32_t field_size) +{ + // TODO: This is a blocking operation. We should instead use index spaces as keys to field + // managers + auto shape = legion_runtime_->get_index_space_domain(legion_context_, region.get_index_space()); + auto fld_mgr = find_or_create_field_manager(shape, field_size); + return fld_mgr->import_field(region, field_id); +} + +RegionField Runtime::map_region_field(LibraryContext* context, const LogicalRegionField* rf) +{ + auto root_region = rf->get_root().region(); + auto field_id = rf->field_id(); + + Legion::PhysicalRegion pr; + + RegionFieldID key(root_region, field_id); + auto finder = inline_mapped_.find(key); + if (inline_mapped_.end() == finder) { + Legion::RegionRequirement req(root_region, READ_WRITE, EXCLUSIVE, root_region); + req.add_field(field_id); + + auto mapper_id = context->get_mapper_id(); + // TODO: We need to pass the metadata about logical store + Legion::InlineLauncher launcher(req, mapper_id); + pr = legion_runtime_->map_region(legion_context_, launcher); + inline_mapped_[key] = pr; + } else + pr = finder->second; + physical_region_refs_.add(pr); + return RegionField(rf->dim(), pr, field_id); +} + +void Runtime::unmap_physical_region(Legion::PhysicalRegion pr) +{ + if (physical_region_refs_.remove(pr)) { + // The last user of this inline mapping was removed, so remove it from our cache and unmap. + std::vector fields; + pr.get_fields(fields); + assert(fields.size() == 1); + RegionFieldID key(pr.get_logical_region(), fields[0]); + auto finder = inline_mapped_.find(key); + assert(finder != inline_mapped_.end() && finder->second == pr); + inline_mapped_.erase(finder); + legion_runtime_->unmap_region(legion_context_, pr); + } +} + +size_t Runtime::num_inline_mapped() const { return inline_mapped_.size(); } + +RegionManager* Runtime::find_or_create_region_manager(const Domain& shape) +{ + auto finder = region_managers_.find(shape); + if (finder != region_managers_.end()) + return finder->second; + else { + auto rgn_mgr = new RegionManager(this, shape); + region_managers_[shape] = rgn_mgr; + return rgn_mgr; + } +} + +FieldManager* Runtime::find_or_create_field_manager(const Domain& shape, uint32_t field_size) +{ + auto key = FieldManagerKey(shape, field_size); + auto finder = field_managers_.find(key); + if (finder != field_managers_.end()) + return finder->second; + else { + auto fld_mgr = new FieldManager(this, shape, field_size); + field_managers_[key] = fld_mgr; + return fld_mgr; + } +} + +PartitionManager* Runtime::partition_manager() const { return partition_manager_; } + +ProvenanceManager* Runtime::provenance_manager() const { return provenance_manager_; } + +Legion::IndexSpace Runtime::find_or_create_index_space(const Domain& shape) +{ + assert(nullptr != legion_context_); + auto finder = index_spaces_.find(shape); + if (finder != index_spaces_.end()) + return finder->second; + else { + auto is = legion_runtime_->create_index_space(legion_context_, shape); + index_spaces_[shape] = is; + return is; + } +} + +Legion::IndexPartition Runtime::create_restricted_partition( + const Legion::IndexSpace& index_space, + const Legion::IndexSpace& color_space, + Legion::PartitionKind kind, + const Legion::DomainTransform& transform, + const Domain& extent) +{ + return legion_runtime_->create_partition_by_restriction( + legion_context_, index_space, color_space, transform, extent, kind); +} + +Legion::IndexPartition Runtime::create_weighted_partition(const Legion::IndexSpace& index_space, + const Legion::IndexSpace& color_space, + const Legion::FutureMap& weights) +{ + return legion_runtime_->create_partition_by_weights( + legion_context_, index_space, weights, color_space); +} + +Legion::FieldSpace Runtime::create_field_space() +{ + assert(nullptr != legion_context_); + return legion_runtime_->create_field_space(legion_context_); +} + +Legion::LogicalRegion Runtime::create_region(const Legion::IndexSpace& index_space, + const Legion::FieldSpace& field_space) +{ + assert(nullptr != legion_context_); + return legion_runtime_->create_logical_region(legion_context_, index_space, field_space); +} + +void Runtime::destroy_region(const Legion::LogicalRegion& logical_region, bool unordered) +{ + assert(nullptr != legion_context_); + legion_runtime_->destroy_logical_region(legion_context_, logical_region, unordered); +} + +Legion::LogicalPartition Runtime::create_logical_partition( + const Legion::LogicalRegion& logical_region, const Legion::IndexPartition& index_partition) +{ + assert(nullptr != legion_context_); + return legion_runtime_->get_logical_partition(legion_context_, logical_region, index_partition); +} + +Legion::LogicalRegion Runtime::get_subregion(const Legion::LogicalPartition& partition, + const Legion::DomainPoint& color) +{ + assert(nullptr != legion_context_); + return legion_runtime_->get_logical_subregion_by_color(legion_context_, partition, color); +} + +Legion::LogicalRegion Runtime::find_parent_region(const Legion::LogicalRegion& region) +{ + auto result = region; + while (legion_runtime_->has_parent_logical_partition(legion_context_, result)) { + auto partition = legion_runtime_->get_parent_logical_partition(legion_context_, result); + result = legion_runtime_->get_parent_logical_region(legion_context_, partition); + } + return result; +} + +Legion::Future Runtime::create_future(const void* data, size_t datalen) const +{ + return Legion::Future::from_untyped_pointer(data, datalen); +} + +Legion::FieldID Runtime::allocate_field(const Legion::FieldSpace& field_space, size_t field_size) +{ + assert(nullptr != legion_context_); + auto allocator = legion_runtime_->create_field_allocator(legion_context_, field_space); + return allocator.allocate_field(field_size); +} + +Legion::FieldID Runtime::allocate_field(const Legion::FieldSpace& field_space, + Legion::FieldID field_id, + size_t field_size) +{ + assert(nullptr != legion_context_); + auto allocator = legion_runtime_->create_field_allocator(legion_context_, field_space); + return allocator.allocate_field(field_size, field_id); +} + +Domain Runtime::get_index_space_domain(const Legion::IndexSpace& index_space) const +{ + assert(nullptr != legion_context_); + return legion_runtime_->get_index_space_domain(legion_context_, index_space); +} + +namespace { + +Legion::DomainPoint _delinearize_future_map(const DomainPoint& point, + const Domain& domain, + const Domain& range) +{ + assert(range.dim == 1); + DomainPoint result; + result.dim = 1; + + int32_t ndim = domain.dim; + int64_t idx = point[0]; + for (int32_t dim = 1; dim < ndim; ++dim) { + int64_t extent = domain.rect_data[dim + ndim] - domain.rect_data[dim] + 1; + idx = idx * extent + point[dim]; + } + result[0] = idx; + return result; +} + +} // namespace + +Legion::FutureMap Runtime::delinearize_future_map(const Legion::FutureMap& future_map, + const Legion::IndexSpace& new_domain) const +{ + return legion_runtime_->transform_future_map( + legion_context_, future_map, new_domain, _delinearize_future_map); +} + +std::pair Runtime::create_barriers(size_t num_tasks) +{ + auto arrival_barrier = legion_runtime_->create_phase_barrier(legion_context_, num_tasks); + auto wait_barrier = legion_runtime_->advance_phase_barrier(legion_context_, arrival_barrier); + return std::make_pair(arrival_barrier, wait_barrier); +} + +void Runtime::destroy_barrier(Legion::PhaseBarrier barrier) +{ + legion_runtime_->destroy_phase_barrier(legion_context_, barrier); +} + +Legion::Future Runtime::get_tunable(Legion::MapperID mapper_id, int64_t tunable_id, size_t size) +{ + Legion::TunableLauncher launcher(tunable_id, mapper_id, 0, size); + return legion_runtime_->select_tunable_value(legion_context_, launcher); +} + +Legion::Future Runtime::dispatch(Legion::TaskLauncher* launcher, + std::vector* output_requirements) +{ + assert(nullptr != legion_context_); + return legion_runtime_->execute_task(legion_context_, *launcher, output_requirements); +} + +Legion::FutureMap Runtime::dispatch(Legion::IndexTaskLauncher* launcher, + std::vector* output_requirements) +{ + assert(nullptr != legion_context_); + return legion_runtime_->execute_index_space(legion_context_, *launcher, output_requirements); +} + +Legion::Future Runtime::extract_scalar(const Legion::Future& result, uint32_t idx) const +{ + auto& machine = get_machine(); + auto& provenance = provenance_manager()->get_provenance(); + auto variant = mapping::to_variant_code(machine.preferred_target); + TaskLauncher launcher( + core_context_, machine, provenance, LEGATE_CORE_EXTRACT_SCALAR_TASK_ID, variant); + launcher.add_future(result); + launcher.add_scalar(Scalar(idx)); + return launcher.execute_single(); +} + +Legion::FutureMap Runtime::extract_scalar(const Legion::FutureMap& result, + uint32_t idx, + const Legion::Domain& launch_domain) const +{ + auto& machine = get_machine(); + auto& provenance = provenance_manager()->get_provenance(); + auto variant = mapping::to_variant_code(machine.preferred_target); + TaskLauncher launcher( + core_context_, machine, provenance, LEGATE_CORE_EXTRACT_SCALAR_TASK_ID, variant); + launcher.add_future_map(result); + launcher.add_scalar(Scalar(idx)); + return launcher.execute(launch_domain); +} + +Legion::Future Runtime::reduce_future_map(const Legion::FutureMap& future_map, + int32_t reduction_op) const +{ + return legion_runtime_->reduce_future_map(legion_context_, + future_map, + reduction_op, + false /*deterministic*/, + core_context_->get_mapper_id()); +} + +Legion::Future Runtime::reduce_exception_future_map(const Legion::FutureMap& future_map) const +{ + auto reduction_op = core_context_->get_reduction_op_id(LEGATE_CORE_JOIN_EXCEPTION_OP); + return legion_runtime_->reduce_future_map(legion_context_, + future_map, + reduction_op, + false /*deterministic*/, + core_context_->get_mapper_id(), + LEGATE_CORE_JOIN_EXCEPTION_TAG); +} + +void Runtime::issue_execution_fence(bool block /*=false*/) +{ + flush_scheduling_window(); + // FIXME: This needs to be a Legate operation + auto future = legion_runtime_->issue_execution_fence(legion_context_); + if (block) future.wait(); +} + +void Runtime::initialize_toplevel_machine() +{ + auto mapper_id = core_context_->get_mapper_id(); + auto num_nodes = get_tunable(mapper_id, LEGATE_CORE_TUNABLE_NUM_NODES); + + auto num_gpus = get_tunable(mapper_id, LEGATE_CORE_TUNABLE_TOTAL_GPUS); + auto num_omps = get_tunable(mapper_id, LEGATE_CORE_TUNABLE_TOTAL_OMPS); + auto num_cpus = get_tunable(mapper_id, LEGATE_CORE_TUNABLE_TOTAL_CPUS); + + auto create_range = [&num_nodes](int32_t num_procs) { + auto per_node_count = num_procs / num_nodes; + return mapping::ProcessorRange(0, num_procs, per_node_count); + }; + + mapping::MachineDesc machine({{mapping::TaskTarget::GPU, create_range(num_gpus)}, + {mapping::TaskTarget::OMP, create_range(num_omps)}, + {mapping::TaskTarget::CPU, create_range(num_cpus)}}); +#ifdef DEBUG_LEGATE + assert(machine_manager_ != nullptr); +#endif + + machine_manager_->push_machine(std::move(machine)); +} + +const mapping::MachineDesc& Runtime::get_machine() const +{ +#ifdef DEBUG_LEGATE + assert(machine_manager_ != nullptr); +#endif + return machine_manager_->get_machine(); +} + +Legion::ProjectionID Runtime::get_projection(int32_t src_ndim, const proj::SymbolicPoint& point) +{ +#ifdef DEBUG_LEGATE + log_legate.debug() << "Query projection {src_ndim: " << src_ndim << ", point: " << point << "}"; +#endif + + if (is_identity(src_ndim, point)) { +#ifdef DEBUG_LEGATE + log_legate.debug() << "Identity projection {src_ndim: " << src_ndim << ", point: " << point + << "}"; +#endif + return 0; + } + + ProjectionDesc key(src_ndim, point); + auto finder = registered_projections_.find(key); + if (registered_projections_.end() != finder) return finder->second; + + auto proj_id = core_context_->get_projection_id(next_projection_id_++); + + auto ndim = point.size(); + std::vector dims; + std::vector weights; + std::vector offsets; + for (auto& expr : point.data()) { + dims.push_back(expr.dim()); + weights.push_back(expr.weight()); + offsets.push_back(expr.offset()); + } + legate_register_affine_projection_functor( + src_ndim, ndim, dims.data(), weights.data(), offsets.data(), proj_id); + registered_projections_[key] = proj_id; + +#ifdef DEBUG_LEGATE + log_legate.debug() << "Register projection " << proj_id << " {src_ndim: " << src_ndim + << ", point: " << point << "}"; +#endif + + return proj_id; +} + +Legion::ProjectionID Runtime::get_delinearizing_projection() +{ + return core_context_->get_projection_id(LEGATE_CORE_DELINEARIZE_PROJ_ID); +} + +CommunicatorManager* Runtime::communicator_manager() const { return communicator_manager_; } + +MachineManager* Runtime::machine_manager() const { return machine_manager_; } + +/*static*/ int32_t Runtime::start(int32_t argc, char** argv) +{ + Legion::Runtime::initialize(&argc, &argv, true /*filter legion and realm args*/); + + Legion::Runtime::add_registration_callback(registration_callback); + + auto result = Legion::Runtime::start(argc, argv, true); + if (result != 0) { + log_legate.error("Legion Runtime failed to start."); + return result; + } + + // Get the runtime now that we've started it + auto legion_runtime = Legion::Runtime::get_runtime(); + + // Then we can make this thread into an implicit top-level task + auto legion_context = legion_runtime->begin_implicit_task(LEGATE_CORE_TOPLEVEL_TASK_ID, + 0 /*mapper id*/, + Processor::LOC_PROC, + TOPLEVEL_NAME, + true /*control replicable*/); + + // We can now initialize the Legate runtime with the Legion context + Runtime::get_runtime()->initialize(legion_context); + + return result; +} + +int32_t Runtime::finish() +{ + destroy(); + + // Mark that we are done excecuting the top-level task + // After this call the context is no longer valid + Legion::Runtime::get_runtime()->finish_implicit_task(legion_context_); + + // The previous call is asynchronous so we still need to + // wait for the shutdown of the runtime to complete + return Legion::Runtime::wait_for_shutdown(); +} + +/*static*/ Runtime* Runtime::get_runtime() +{ + static auto runtime = std::make_unique(); + return runtime.get(); +} + +void Runtime::destroy() +{ + // Make sure all Legate ops finish + issue_execution_fence(true); + + // Clean up resources used by Legate + for (auto& [_, context] : libraries_) delete context; + libraries_.clear(); + + communicator_manager_->destroy(); + delete communicator_manager_; + delete machine_manager_; + delete partition_manager_; + delete provenance_manager_; + + for (auto& [_, region_manager] : region_managers_) { + region_manager->destroy(true /*unordered*/); + delete region_manager; + } + region_managers_.clear(); + + for (auto& [_, field_manager] : field_managers_) delete field_manager; + field_managers_.clear(); + + for (auto& [_, index_space] : index_spaces_) + legion_runtime_->destroy_index_space(legion_context_, index_space, true /*unordered*/); + index_spaces_.clear(); + + issue_execution_fence(); +} + +static void extract_scalar_task( + const void* args, size_t arglen, const void* userdata, size_t userlen, Legion::Processor p) +{ + // Legion preamble + const Legion::Task* task; + const std::vector* regions; + Legion::Context legion_context; + Legion::Runtime* runtime; + Legion::Runtime::legion_task_preamble(args, arglen, p, task, regions, legion_context, runtime); + + Core::show_progress(task, legion_context, runtime); + + TaskContext context(task, *regions, legion_context, runtime); + auto idx = context.scalars()[0].value(); + auto value_and_size = ReturnValues::extract(task->futures[0], idx); + + // Legion postamble + value_and_size.finalize(legion_context); +} + +void register_legate_core_tasks(Legion::Machine machine, + Legion::Runtime* runtime, + const LibraryContext* context) +{ + auto extract_scalar_task_id = context->get_task_id(LEGATE_CORE_EXTRACT_SCALAR_TASK_ID); + const char* extract_scalar_task_name = "core::extract_scalar"; + runtime->attach_name( + extract_scalar_task_id, extract_scalar_task_name, false /*mutable*/, true /*local only*/); + + auto make_registrar = [&](auto task_id, auto* task_name, auto proc_kind) { + Legion::TaskVariantRegistrar registrar(task_id, task_name); + registrar.add_constraint(Legion::ProcessorConstraint(proc_kind)); + registrar.set_leaf(true); + registrar.global_registration = false; + return registrar; + }; + + // Register the task variants + auto register_extract_scalar = [&](auto proc_kind, auto variant_id) { + auto registrar = make_registrar(extract_scalar_task_id, extract_scalar_task_name, proc_kind); + Legion::CodeDescriptor desc(extract_scalar_task); + runtime->register_task_variant( + registrar, desc, nullptr, 0, LEGATE_MAX_SIZE_SCALAR_RETURN, variant_id); + }; + register_extract_scalar(Processor::LOC_PROC, LEGATE_CPU_VARIANT); +#ifdef LEGATE_USE_CUDA + register_extract_scalar(Processor::TOC_PROC, LEGATE_GPU_VARIANT); +#endif +#ifdef LEGATE_USE_OPENMP + register_extract_scalar(Processor::OMP_PROC, LEGATE_OMP_VARIANT); +#endif + comm::register_tasks(machine, runtime, context); +} + +#define BUILTIN_REDOP_ID(OP, TYPE_CODE) \ + (LEGION_REDOP_BASE + (OP)*LEGION_TYPE_TOTAL + (static_cast(TYPE_CODE))) + +#define RECORD(OP, TYPE_CODE) \ + PrimitiveType(TYPE_CODE).record_reduction_operator(OP, BUILTIN_REDOP_ID(OP, TYPE_CODE)); + +#define RECORD_INT(OP) \ + RECORD(OP, Type::Code::BOOL) \ + RECORD(OP, Type::Code::INT8) \ + RECORD(OP, Type::Code::INT16) \ + RECORD(OP, Type::Code::INT32) \ + RECORD(OP, Type::Code::INT64) \ + RECORD(OP, Type::Code::UINT8) \ + RECORD(OP, Type::Code::UINT16) \ + RECORD(OP, Type::Code::UINT32) \ + RECORD(OP, Type::Code::UINT64) + +#define RECORD_FLOAT(OP) \ + RECORD(OP, Type::Code::FLOAT16) \ + RECORD(OP, Type::Code::FLOAT32) \ + RECORD(OP, Type::Code::FLOAT64) + +#define RECORD_COMPLEX(OP) RECORD(OP, Type::Code::COMPLEX64) + +#define RECORD_ALL(OP) \ + RECORD_INT(OP) \ + RECORD_FLOAT(OP) \ + RECORD_COMPLEX(OP) + +void register_builtin_reduction_ops() +{ + RECORD_ALL(ADD_LT) + RECORD(ADD_LT, Type::Code::COMPLEX128) + RECORD_ALL(SUB_LT) + RECORD_ALL(MUL_LT) + RECORD_ALL(DIV_LT) + + RECORD_INT(MAX_LT) + RECORD_FLOAT(MAX_LT) + + RECORD_INT(MIN_LT) + RECORD_FLOAT(MIN_LT) + + RECORD_INT(OR_LT) + RECORD_INT(AND_LT) + RECORD_INT(XOR_LT) +} + +extern void register_exception_reduction_op(Legion::Runtime* runtime, + const LibraryContext* context); + +void core_library_registration(Legion::Machine machine, + Legion::Runtime* legion_runtime, + const std::set& local_procs) +{ + ResourceConfig config; + config.max_tasks = LEGATE_CORE_NUM_TASK_IDS; + config.max_projections = LEGATE_CORE_MAX_FUNCTOR_ID; + // We register one sharding functor for each new projection functor + config.max_shardings = LEGATE_CORE_MAX_FUNCTOR_ID; + config.max_reduction_ops = LEGATE_CORE_MAX_REDUCTION_OP_ID; + + auto runtime = Runtime::get_runtime(); + auto core_lib = runtime->create_library(core_library_name, config); + + register_legate_core_tasks(machine, legion_runtime, core_lib); + + register_legate_core_mapper(machine, legion_runtime, core_lib); + + register_builtin_reduction_ops(); + + register_exception_reduction_op(legion_runtime, core_lib); + + register_legate_core_projection_functors(legion_runtime, core_lib); + + register_legate_core_sharding_functors(legion_runtime, core_lib); +} + +void registration_callback(Legion::Machine machine, + Legion::Runtime* legion_runtime, + const std::set& local_procs) +{ + core_library_registration(machine, legion_runtime, local_procs); + + Core::parse_config(); +} + +void registration_callback_for_python(Legion::Machine machine, + Legion::Runtime* legion_runtime, + const std::set& local_procs) +{ + core_library_registration(machine, legion_runtime, local_procs); + + Runtime::get_runtime()->initialize(Legion::Runtime::get_context()); +} + +} // namespace legate::detail diff --git a/src/core/runtime/detail/runtime.h b/src/core/runtime/detail/runtime.h new file mode 100644 index 0000000000..9592a593d1 --- /dev/null +++ b/src/core/runtime/detail/runtime.h @@ -0,0 +1,247 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include +#include + +#include "core/data/shape.h" +#include "core/data/store.h" +#include "core/mapping/machine.h" +#include "core/runtime/detail/communicator_manager.h" +#include "core/runtime/detail/field_manager.h" +#include "core/runtime/detail/machine_manager.h" +#include "core/runtime/detail/partition_manager.h" +#include "core/runtime/detail/provenance_manager.h" +#include "core/runtime/detail/region_manager.h" +#include "core/runtime/resource.h" +#include "core/task/exception.h" +#include "core/type/type_info.h" +#include "core/utilities/multi_set.h" + +namespace legate { +class AutoTask; +class LibraryContext; +class ManualTask; +class Operation; +} // namespace legate + +namespace legate::detail { + +class LogicalRegionField; +class LogicalStore; + +class Runtime { + public: + Runtime(); + ~Runtime(); + + public: + LibraryContext* find_library(const std::string& library_name, bool can_fail = false) const; + LibraryContext* create_library(const std::string& library_name, + const ResourceConfig& config = ResourceConfig{}, + std::unique_ptr mapper = nullptr); + + public: + uint32_t get_type_uid(); + void record_reduction_operator(int32_t type_uid, int32_t op_kind, int32_t legion_op_id); + int32_t find_reduction_operator(int32_t type_uid, int32_t op_kind) const; + + public: + void enter_callback(); + void exit_callback(); + bool is_in_callback() const; + + public: + void initialize(Legion::Context legion_context); + + public: + mapping::MachineDesc slice_machine_for_task(LibraryContext* library, int64_t task_id); + std::unique_ptr create_task(LibraryContext* library, int64_t task_id); + std::unique_ptr create_task(LibraryContext* library, + int64_t task_id, + const Shape& launch_shape); + void flush_scheduling_window(); + void submit(std::unique_ptr op); + + public: + std::shared_ptr create_store(std::unique_ptr type, int32_t dim = 1); + std::shared_ptr create_store(const Shape& extents, + std::unique_ptr type, + bool optimize_scalar = false); + std::shared_ptr create_store(const Scalar& scalar); + + public: + uint32_t max_pending_exceptions() const; + void set_max_pending_exceptions(uint32_t max_pending_exceptions); + void raise_pending_task_exception(); + std::optional check_pending_task_exception(); + void record_pending_exception(const Legion::Future& pending_exception); + + public: + uint64_t get_unique_store_id(); + uint64_t get_unique_storage_id(); + + public: + std::shared_ptr create_region_field(const Shape& extents, + uint32_t field_size); + std::shared_ptr import_region_field(Legion::LogicalRegion region, + Legion::FieldID field_id, + uint32_t field_size); + RegionField map_region_field(LibraryContext* context, const LogicalRegionField* region_field); + void unmap_physical_region(Legion::PhysicalRegion pr); + size_t num_inline_mapped() const; + + public: + RegionManager* find_or_create_region_manager(const Legion::Domain& shape); + FieldManager* find_or_create_field_manager(const Legion::Domain& shape, uint32_t field_size); + CommunicatorManager* communicator_manager() const; + MachineManager* machine_manager() const; + PartitionManager* partition_manager() const; + ProvenanceManager* provenance_manager() const; + + public: + Legion::IndexSpace find_or_create_index_space(const Legion::Domain& shape); + Legion::IndexPartition create_restricted_partition(const Legion::IndexSpace& index_space, + const Legion::IndexSpace& color_space, + Legion::PartitionKind kind, + const Legion::DomainTransform& transform, + const Legion::Domain& extent); + Legion::IndexPartition create_weighted_partition(const Legion::IndexSpace& index_space, + const Legion::IndexSpace& color_space, + const Legion::FutureMap& weights); + Legion::FieldSpace create_field_space(); + Legion::LogicalRegion create_region(const Legion::IndexSpace& index_space, + const Legion::FieldSpace& field_space); + void destroy_region(const Legion::LogicalRegion& logical_region, bool unordered = false); + Legion::LogicalPartition create_logical_partition(const Legion::LogicalRegion& logical_region, + const Legion::IndexPartition& index_partition); + Legion::LogicalRegion get_subregion(const Legion::LogicalPartition& partition, + const Legion::DomainPoint& color); + Legion::LogicalRegion find_parent_region(const Legion::LogicalRegion& region); + Legion::Future create_future(const void* data, size_t datalen) const; + Legion::FieldID allocate_field(const Legion::FieldSpace& field_space, size_t field_size); + Legion::FieldID allocate_field(const Legion::FieldSpace& field_space, + Legion::FieldID field_id, + size_t field_size); + Legion::Domain get_index_space_domain(const Legion::IndexSpace& index_space) const; + Legion::FutureMap delinearize_future_map(const Legion::FutureMap& future_map, + const Legion::IndexSpace& new_domain) const; + std::pair create_barriers(size_t num_tasks); + void destroy_barrier(Legion::PhaseBarrier barrier); + Legion::Future get_tunable(Legion::MapperID mapper_id, int64_t tunable_id, size_t size); + template + T get_tunable(Legion::MapperID mapper_id, int64_t tunable_id) + { + return get_tunable(mapper_id, tunable_id, sizeof(T)).get_result(); + } + + public: + Legion::Future dispatch(Legion::TaskLauncher* launcher, + std::vector* output_requirements = nullptr); + Legion::FutureMap dispatch(Legion::IndexTaskLauncher* launcher, + std::vector* output_requirements = nullptr); + + public: + Legion::Future extract_scalar(const Legion::Future& result, uint32_t idx) const; + Legion::FutureMap extract_scalar(const Legion::FutureMap& result, + uint32_t idx, + const Legion::Domain& launch_domain) const; + Legion::Future reduce_future_map(const Legion::FutureMap& future_map, int32_t reduction_op) const; + Legion::Future reduce_exception_future_map(const Legion::FutureMap& future_map) const; + + public: + void issue_execution_fence(bool block = false); + + public: + void initialize_toplevel_machine(); + const mapping::MachineDesc& get_machine() const; + + public: + Legion::ProjectionID get_projection(int32_t src_ndim, const proj::SymbolicPoint& point); + Legion::ProjectionID get_delinearizing_projection(); + + private: + void schedule(std::vector> operations); + + public: + static Runtime* get_runtime(); + static int32_t start(int32_t argc, char** argv); + bool initialized() const { return initialized_; } + void destroy(); + int32_t finish(); + + private: + bool initialized_{false}; + Legion::Runtime* legion_runtime_{nullptr}; + Legion::Context legion_context_{nullptr}; + LibraryContext* core_context_{nullptr}; + + private: + using FieldManagerKey = std::pair; + std::map field_managers_; + std::map region_managers_; + CommunicatorManager* communicator_manager_{nullptr}; + MachineManager* machine_manager_{nullptr}; + PartitionManager* partition_manager_{nullptr}; + ProvenanceManager* provenance_manager_{nullptr}; + + private: + std::map index_spaces_; + + private: + using ProjectionDesc = std::pair; + int64_t next_projection_id_{LEGATE_CORE_FIRST_DYNAMIC_FUNCTOR_ID}; + std::map registered_projections_; + + private: + std::vector> operations_; + size_t window_size_{1}; + uint64_t next_unique_id_{0}; + + private: + using RegionFieldID = std::pair; + std::map inline_mapped_; + MultiSet physical_region_refs_; + uint64_t next_store_id_{1}; + uint64_t next_storage_id_{1}; + + private: + bool in_callback_{false}; + + private: + std::map libraries_{}; + + private: + uint32_t next_type_uid_; + std::map, int32_t> reduction_ops_{}; + + private: + uint32_t max_pending_exceptions_; + std::vector pending_exceptions_{}; + std::deque outstanding_exceptions_{}; +}; + +void registration_callback(Legion::Machine machine, + Legion::Runtime* legion_runtime, + const std::set& local_procs); + +void registration_callback_for_python(Legion::Machine machine, + Legion::Runtime* legion_runtime, + const std::set& local_procs); + +} // namespace legate::detail diff --git a/src/core/runtime/operation.cc b/src/core/runtime/operation.cc index 14736e7e0b..06df29427e 100644 --- a/src/core/runtime/operation.cc +++ b/src/core/runtime/operation.cc @@ -19,17 +19,18 @@ #include #include -#include "core/data/logical_store_detail.h" +#include "core/data/detail/logical_store.h" #include "core/data/scalar.h" #include "core/partitioning/constraint.h" #include "core/partitioning/constraint_solver.h" #include "core/partitioning/partition.h" #include "core/partitioning/partitioner.h" -#include "core/runtime/communicator_manager.h" #include "core/runtime/context.h" -#include "core/runtime/launcher.h" -#include "core/runtime/req_analyzer.h" -#include "core/runtime/runtime.h" +#include "core/runtime/detail/communicator_manager.h" +#include "core/runtime/detail/launcher.h" +#include "core/runtime/detail/provenance_manager.h" +#include "core/runtime/detail/req_analyzer.h" +#include "core/runtime/detail/runtime.h" namespace legate { @@ -41,7 +42,7 @@ Operation::Operation(LibraryContext* library, uint64_t unique_id, mapping::Machi : library_(library), unique_id_(unique_id), machine_(std::move(machine)), - provenance_(Runtime::get_runtime()->provenance_manager()->get_provenance()) + provenance_(detail::Runtime::get_runtime()->provenance_manager()->get_provenance()) { } @@ -85,14 +86,14 @@ void Task::throws_exception(bool can_throw_exception) void Task::add_communicator(const std::string& name) { - auto* comm_mgr = Runtime::get_runtime()->communicator_manager(); + auto* comm_mgr = detail::Runtime::get_runtime()->communicator_manager(); communicator_factories_.push_back(comm_mgr->find_factory(name)); } -void Task::launch(Strategy* p_strategy) +void Task::launch(detail::Strategy* p_strategy) { auto& strategy = *p_strategy; - TaskLauncher launcher(library_, machine_, provenance_, task_id_); + detail::TaskLauncher launcher(library_, machine_, provenance_, task_id_); const auto* launch_domain = strategy.launch_domain(this); // Add input stores @@ -126,7 +127,7 @@ void Task::launch(Strategy* p_strategy) } // Add unbound output stores - auto* runtime = Runtime::get_runtime(); + auto* runtime = detail::Runtime::get_runtime(); for (auto& [store, var] : outputs_) { if (!store->unbound()) continue; auto field_space = strategy.find_field_space(var); @@ -181,7 +182,7 @@ void Task::demux_scalar_stores(const Legion::Future& result) auto [store, _] = reductions_[scalar_reductions_.front()]; store->set_future(result); } else if (can_throw_exception_) { - auto* runtime = Runtime::get_runtime(); + auto* runtime = detail::Runtime::get_runtime(); runtime->record_pending_exception(result); } #ifdef DEBUG_LEGATE @@ -190,7 +191,7 @@ void Task::demux_scalar_stores(const Legion::Future& result) } #endif } else { - auto* runtime = Runtime::get_runtime(); + auto* runtime = detail::Runtime::get_runtime(); uint32_t idx = num_unbound_outs; for (const auto& out_idx : scalar_outputs_) { auto [store, _] = outputs_[out_idx]; @@ -216,14 +217,14 @@ void Task::demux_scalar_stores(const Legion::FutureMap& result, const Domain& la auto total = num_scalar_reds + num_unbound_outs + static_cast(can_throw_exception_); if (0 == total) return; - auto* runtime = Runtime::get_runtime(); + auto* runtime = detail::Runtime::get_runtime(); if (1 == total) { if (1 == num_scalar_reds) { auto red_idx = scalar_reductions_.front(); auto [store, _] = reductions_[red_idx]; store->set_future(runtime->reduce_future_map(result, reduction_ops_[red_idx])); } else if (can_throw_exception_) { - auto* runtime = Runtime::get_runtime(); + auto* runtime = detail::Runtime::get_runtime(); runtime->record_pending_exception(runtime->reduce_exception_future_map(result)); } #ifdef DEBUG_LEGATE @@ -302,7 +303,7 @@ void AutoTask::add_constraint(std::unique_ptr constraint) constraints_.push_back(std::move(constraint)); } -void AutoTask::add_to_solver(ConstraintSolver& solver) const +void AutoTask::add_to_solver(detail::ConstraintSolver& solver) const { for (auto& constraint : constraints_) solver.add_constraint(constraint.get()); for (auto& [_, symb] : outputs_) solver.add_partition_symbol(symb); @@ -321,7 +322,8 @@ ManualTask::ManualTask(LibraryContext* library, const Shape& launch_shape, uint64_t unique_id, mapping::MachineDesc&& machine) - : Task(library, task_id, unique_id, std::move(machine)), strategy_(std::make_unique()) + : Task(library, task_id, unique_id, std::move(machine)), + strategy_(std::make_unique()) { strategy_->set_launch_shape(this, launch_shape); } @@ -376,14 +378,14 @@ void ManualTask::add_store(std::vector& store_args, store_args.push_back(StoreArg(store_impl.get(), partition_symbol)); all_stores_.insert(std::move(store_impl)); if (store.unbound()) { - auto field_space = Runtime::get_runtime()->create_field_space(); + auto field_space = detail::Runtime::get_runtime()->create_field_space(); strategy_->insert(partition_symbol, std::move(partition), field_space); } else strategy_->insert(partition_symbol, std::move(partition)); } -void ManualTask::launch(Strategy*) { Task::launch(strategy_.get()); } +void ManualTask::launch(detail::Strategy*) { Task::launch(strategy_.get()); } -void ManualTask::add_to_solver(ConstraintSolver& constraint_graph) const {} +void ManualTask::add_to_solver(detail::ConstraintSolver& constraint_graph) const {} } // namespace legate diff --git a/src/core/runtime/operation.h b/src/core/runtime/operation.h index 2cc001791c..124d29048c 100644 --- a/src/core/runtime/operation.h +++ b/src/core/runtime/operation.h @@ -33,18 +33,18 @@ */ namespace legate::detail { +class CommunicatorFactory; +class ConstraintSolver; class LogicalStore; +class Strategy; +class Runtime; } // namespace legate::detail namespace legate { -class CommunicatorFactory; class Constraint; -class ConstraintSolver; class LibraryContext; -class Runtime; class Scalar; -class Strategy; /** * @ingroup op @@ -59,9 +59,9 @@ class Operation { virtual ~Operation() {} public: - virtual void add_to_solver(ConstraintSolver& constraint_graph) const = 0; - virtual void launch(Strategy* strategy) = 0; - virtual std::string to_string() const = 0; + virtual void add_to_solver(detail::ConstraintSolver& constraint_graph) const = 0; + virtual void launch(detail::Strategy* strategy) = 0; + virtual std::string to_string() const = 0; public: /** @@ -160,7 +160,7 @@ class Task : public Operation { void add_communicator(const std::string& name); public: - virtual void launch(Strategy* strategy) override; + virtual void launch(detail::Strategy* strategy) override; private: void demux_scalar_stores(const Legion::Future& result); @@ -178,7 +178,7 @@ class Task : public Operation { std::vector unbound_outputs_{}; std::vector scalar_outputs_{}; std::vector scalar_reductions_{}; - std::vector communicator_factories_{}; + std::vector communicator_factories_{}; }; /** @@ -187,7 +187,7 @@ class Task : public Operation { */ class AutoTask : public Task { public: - friend class Runtime; + friend class detail::Runtime; AutoTask(LibraryContext* library, int64_t task_id, uint64_t unique_id, @@ -243,7 +243,7 @@ class AutoTask : public Task { * @param constraint A partitioning constraint */ void add_constraint(std::unique_ptr constraint); - void add_to_solver(ConstraintSolver& constraint_graph) const override; + void add_to_solver(detail::ConstraintSolver& constraint_graph) const override; private: std::vector> constraints_{}; @@ -255,7 +255,7 @@ class AutoTask : public Task { */ class ManualTask : public Task { private: - friend class Runtime; + friend class detail::Runtime; ManualTask(LibraryContext* library, int64_t task_id, const Shape& launch_shape, @@ -319,13 +319,13 @@ class ManualTask : public Task { std::shared_ptr partition); public: - void launch(Strategy* strategy) override; + void launch(detail::Strategy* strategy) override; public: - void add_to_solver(ConstraintSolver& constraint_graph) const override; + void add_to_solver(detail::ConstraintSolver& constraint_graph) const override; private: - std::unique_ptr strategy_; + std::unique_ptr strategy_; }; } // namespace legate diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 641215e1ee..110f910410 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -1,4 +1,4 @@ -/* Copyright 2021-2022 NVIDIA Corporation +/* Copyright 2021-2023 NVIDIA Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,44 +14,19 @@ * */ -#include -#include - -#include "core/comm/comm.h" -#include "core/data/logical_region_field.h" -#include "core/data/logical_store.h" -#include "core/data/logical_store_detail.h" -#include "core/mapping/core_mapper.h" -#include "core/mapping/default_mapper.h" -#include "core/mapping/machine.h" -#include "core/partitioning/partition.h" -#include "core/partitioning/partitioner.h" -#include "core/runtime/communicator_manager.h" +#include "core/runtime/runtime.h" + #include "core/runtime/context.h" -#include "core/runtime/field_manager.h" -#include "core/runtime/launcher.h" -#include "core/runtime/machine_manager.h" -#include "core/runtime/partition_manager.h" -#include "core/runtime/projection.h" -#include "core/runtime/provenance_manager.h" -#include "core/runtime/region_manager.h" -#include "core/runtime/shard.h" -#include "core/task/return.h" -#include "core/task/task.h" -#include "core/type/type_info.h" -#include "core/utilities/deserializer.h" -#include "core/utilities/machine.h" -#include "core/utilities/nvtx_help.h" -#include "env_defaults.h" -#include "legate.h" +#include "core/runtime/detail/runtime.h" +#include "core/runtime/operation.h" namespace legate { -Logger log_legate("legate"); +extern Logger log_legate; // This is the unique string name for our library which can be used // from both C++ and Python to generate IDs -static const char* const core_library_name = "legate.core"; +extern const char* const core_library_name; /*static*/ bool Core::show_progress_requested = false; @@ -103,26 +78,6 @@ static const char* const core_library_name = "legate.core"; parse_variable("LEGATE_LOG_MAPPING", log_mapping_decisions); } -static void extract_scalar_task( - const void* args, size_t arglen, const void* userdata, size_t userlen, Legion::Processor p) -{ - // Legion preamble - const Legion::Task* task; - const std::vector* regions; - Legion::Context legion_context; - Legion::Runtime* runtime; - Legion::Runtime::legion_task_preamble(args, arglen, p, task, regions, legion_context, runtime); - - Core::show_progress(task, legion_context, runtime); - - TaskContext context(task, *regions, legion_context, runtime); - auto idx = context.scalars()[0].value(); - auto value_and_size = ReturnValues::extract(task->futures[0], idx); - - // Legion postamble - value_and_size.finalize(legion_context); -} - /*static*/ void Core::shutdown(void) { // Nothing to do here yet... @@ -172,324 +127,47 @@ static void extract_scalar_task( Core::has_socket_mem = fut.get_result(); } -void register_legate_core_tasks(Legion::Machine machine, - Legion::Runtime* runtime, - const LibraryContext* context) +/*static*/ void Core::perform_callback(Legion::RegistrationCallbackFnptr callback) { - auto extract_scalar_task_id = context->get_task_id(LEGATE_CORE_EXTRACT_SCALAR_TASK_ID); - const char* extract_scalar_task_name = "core::extract_scalar"; - runtime->attach_name( - extract_scalar_task_id, extract_scalar_task_name, false /*mutable*/, true /*local only*/); - - auto make_registrar = [&](auto task_id, auto* task_name, auto proc_kind) { - Legion::TaskVariantRegistrar registrar(task_id, task_name); - registrar.add_constraint(Legion::ProcessorConstraint(proc_kind)); - registrar.set_leaf(true); - registrar.global_registration = false; - return registrar; - }; - - // Register the task variants - auto register_extract_scalar = [&](auto proc_kind, auto variant_id) { - auto registrar = make_registrar(extract_scalar_task_id, extract_scalar_task_name, proc_kind); - Legion::CodeDescriptor desc(extract_scalar_task); - runtime->register_task_variant( - registrar, desc, nullptr, 0, LEGATE_MAX_SIZE_SCALAR_RETURN, variant_id); - }; - register_extract_scalar(Processor::LOC_PROC, LEGATE_CPU_VARIANT); -#ifdef LEGATE_USE_CUDA - register_extract_scalar(Processor::TOC_PROC, LEGATE_GPU_VARIANT); -#endif -#ifdef LEGATE_USE_OPENMP - register_extract_scalar(Processor::OMP_PROC, LEGATE_OMP_VARIANT); -#endif - comm::register_tasks(machine, runtime, context); -} - -extern void register_exception_reduction_op(Legion::Runtime* runtime, - const LibraryContext* context); - -#define BUILTIN_REDOP_ID(OP, TYPE_CODE) \ - (LEGION_REDOP_BASE + (OP)*LEGION_TYPE_TOTAL + (static_cast(TYPE_CODE))) - -#define RECORD(OP, TYPE_CODE) \ - PrimitiveType(TYPE_CODE).record_reduction_operator(OP, BUILTIN_REDOP_ID(OP, TYPE_CODE)); - -#define RECORD_INT(OP) \ - RECORD(OP, Type::Code::BOOL) \ - RECORD(OP, Type::Code::INT8) \ - RECORD(OP, Type::Code::INT16) \ - RECORD(OP, Type::Code::INT32) \ - RECORD(OP, Type::Code::INT64) \ - RECORD(OP, Type::Code::UINT8) \ - RECORD(OP, Type::Code::UINT16) \ - RECORD(OP, Type::Code::UINT32) \ - RECORD(OP, Type::Code::UINT64) - -#define RECORD_FLOAT(OP) \ - RECORD(OP, Type::Code::FLOAT16) \ - RECORD(OP, Type::Code::FLOAT32) \ - RECORD(OP, Type::Code::FLOAT64) - -#define RECORD_COMPLEX(OP) RECORD(OP, Type::Code::COMPLEX64) - -#define RECORD_ALL(OP) \ - RECORD_INT(OP) \ - RECORD_FLOAT(OP) \ - RECORD_COMPLEX(OP) - -void register_builtin_reduction_ops() -{ - RECORD_ALL(ADD_LT) - RECORD(ADD_LT, Type::Code::COMPLEX128) - RECORD_ALL(SUB_LT) - RECORD_ALL(MUL_LT) - RECORD_ALL(DIV_LT) - - RECORD_INT(MAX_LT) - RECORD_FLOAT(MAX_LT) - - RECORD_INT(MIN_LT) - RECORD_FLOAT(MIN_LT) - - RECORD_INT(OR_LT) - RECORD_INT(AND_LT) - RECORD_INT(XOR_LT) -} - -/*static*/ void core_library_registration(Legion::Machine machine, - Legion::Runtime* legion_runtime, - const std::set& local_procs) -{ - Runtime::create_runtime(legion_runtime); - - ResourceConfig config; - config.max_tasks = LEGATE_CORE_NUM_TASK_IDS; - config.max_projections = LEGATE_CORE_MAX_FUNCTOR_ID; - // We register one sharding functor for each new projection functor - config.max_shardings = LEGATE_CORE_MAX_FUNCTOR_ID; - config.max_reduction_ops = LEGATE_CORE_MAX_REDUCTION_OP_ID; - - auto runtime = Runtime::get_runtime(); - auto core_lib = runtime->create_library(core_library_name, config); - - register_legate_core_tasks(machine, legion_runtime, core_lib); - - register_legate_core_mapper(machine, legion_runtime, core_lib); - - register_builtin_reduction_ops(); - - register_exception_reduction_op(legion_runtime, core_lib); - - register_legate_core_projection_functors(legion_runtime, core_lib); - - register_legate_core_sharding_functors(legion_runtime, core_lib); -} - -/*static*/ void core_library_registration_callback(Legion::Machine machine, - Legion::Runtime* legion_runtime, - const std::set& local_procs) -{ - core_library_registration(machine, legion_runtime, local_procs); - - Runtime::get_runtime()->post_startup_initialization(Legion::Runtime::get_context()); -} - -/*static*/ void core_library_bootstrapping_callback(Legion::Machine machine, - Legion::Runtime* legion_runtime, - const std::set& local_procs) -{ - core_library_registration(machine, legion_runtime, local_procs); - - Core::parse_config(); -} - -//////////////////////////////////////////////////// -// legate::Runtime -//////////////////////////////////////////////////// - -/*static*/ Runtime* Runtime::runtime_; - -namespace { - -constexpr uint32_t CUSTOM_TYPE_UID_BASE = 1000; - -} // namespace - -Runtime::Runtime(Legion::Runtime* legion_runtime) - : legion_runtime_(legion_runtime), - next_type_uid_(CUSTOM_TYPE_UID_BASE), - max_pending_exceptions_(extract_env( - "LEGATE_MAX_PENDING_EXCEPTIONS", MAX_PENDING_EXCEPTIONS_DEFAULT, MAX_PENDING_EXCEPTIONS_TEST)) -{ -} - -Runtime::~Runtime() -{ - for (auto& [_, context] : libraries_) delete context; - delete communicator_manager_; - delete machine_manager_; - delete partition_manager_; - delete provenance_manager_; + Legion::Runtime::perform_registration_callback(callback, true /*global*/); } LibraryContext* Runtime::find_library(const std::string& library_name, bool can_fail /*=false*/) const { - auto finder = libraries_.find(library_name); - if (libraries_.end() == finder) { - if (!can_fail) throw std::out_of_range("Library " + library_name + " does not exist"); - return nullptr; - } - return finder->second; + return impl_->find_library(library_name, can_fail); } LibraryContext* Runtime::create_library(const std::string& library_name, const ResourceConfig& config, std::unique_ptr mapper) { - if (libraries_.find(library_name) != libraries_.end()) - throw std::invalid_argument("Library " + library_name + " already exists"); - - log_legate.debug("Library %s is created", library_name.c_str()); - if (nullptr == mapper) mapper = std::make_unique(); - auto context = new LibraryContext(library_name, config, std::move(mapper)); - libraries_[library_name] = context; - return context; + return impl_->create_library(library_name, config, std::move(mapper)); } -uint32_t Runtime::get_type_uid() { return next_type_uid_++; } - void Runtime::record_reduction_operator(int32_t type_uid, int32_t op_kind, int32_t legion_op_id) { -#ifdef DEBUG_LEGATE - log_legate.debug("Record reduction op (type_uid: %d, op_kind: %d, legion_op_id: %d)", - type_uid, - op_kind, - legion_op_id); -#endif - auto key = std::make_pair(type_uid, op_kind); - auto finder = reduction_ops_.find(key); - if (finder != reduction_ops_.end()) { - std::stringstream ss; - ss << "Reduction op " << op_kind << " already exists for type " << type_uid; - throw std::invalid_argument(std::move(ss).str()); - } - reduction_ops_.emplace(std::make_pair(key, legion_op_id)); -} - -int32_t Runtime::find_reduction_operator(int32_t type_uid, int32_t op_kind) const -{ - auto key = std::make_pair(type_uid, op_kind); - auto finder = reduction_ops_.find(key); - if (reduction_ops_.end() == finder) { -#ifdef DEBUG_LEGATE - log_legate.debug("Can't find reduction op (type_uid: %d, op_kind: %d)", type_uid, op_kind); -#endif - std::stringstream ss; - ss << "Reduction op " << op_kind << " does not exist for type " << type_uid; - throw std::invalid_argument(std::move(ss).str()); - } -#ifdef DEBUG_LEGATE - log_legate.debug( - "Found reduction op %d (type_uid: %d, op_kind: %d)", finder->second, type_uid, op_kind); -#endif - return finder->second; -} - -void Runtime::enter_callback() { in_callback_ = true; } - -void Runtime::exit_callback() { in_callback_ = false; } - -bool Runtime::is_in_callback() const { return in_callback_; } - -void Runtime::post_startup_initialization(Legion::Context legion_context) -{ - legion_context_ = legion_context; - core_context_ = find_library(core_library_name); - communicator_manager_ = new CommunicatorManager(); - partition_manager_ = new PartitionManager(this, core_context_); - machine_manager_ = new MachineManager(); - provenance_manager_ = new ProvenanceManager(); - Core::retrieve_tunable(legion_context_, legion_runtime_, core_context_); - initialize_toplevel_machine(); - comm::register_builtin_communicator_factories(core_context_); -} - -mapping::MachineDesc Runtime::slice_machine_for_task(LibraryContext* library, int64_t task_id) -{ - auto task_info = library->find_task(task_id); - if (nullptr == task_info) { - std::stringstream ss; - ss << "Library " << library->get_library_name() << " does not have task " << task_id; - throw std::invalid_argument(std::move(ss).str()); - } - - std::vector task_targets; - auto& machine = machine_manager_->get_machine(); - for (const auto& t : machine.valid_targets()) { - if (task_info->has_variant(mapping::to_variant_code(t))) task_targets.push_back(t); - } - auto sliced = machine.only(task_targets); - - if (sliced.empty()) { - std::stringstream ss; - ss << "Task " << task_id << " (" << task_info->name() << ") of library " - << library->get_library_name() << " does not have any valid variant for " - << "the current machine configuration."; - throw std::invalid_argument(ss.str()); - } - return sliced; + impl_->record_reduction_operator(type_uid, op_kind, legion_op_id); } // This function should be moved to the library context std::unique_ptr Runtime::create_task(LibraryContext* library, int64_t task_id) { - auto machine = slice_machine_for_task(library, task_id); - auto task = new AutoTask(library, task_id, next_unique_id_++, std::move(machine)); - return std::unique_ptr(task); + return impl_->create_task(library, task_id); } std::unique_ptr Runtime::create_task(LibraryContext* library, int64_t task_id, const Shape& launch_shape) { - auto machine = slice_machine_for_task(library, task_id); - auto task = new ManualTask(library, task_id, launch_shape, next_unique_id_++, std::move(machine)); - return std::unique_ptr(task); -} - -void Runtime::flush_scheduling_window() -{ - if (operations_.size() == 0) return; - - std::vector> to_schedule; - to_schedule.swap(operations_); - schedule(std::move(to_schedule)); + return impl_->create_task(library, task_id, launch_shape); } -void Runtime::submit(std::unique_ptr op) -{ - operations_.push_back(std::move(op)); - if (operations_.size() >= window_size_) { flush_scheduling_window(); } -} - -void Runtime::schedule(std::vector> operations) -{ - std::vector op_pointers{}; - op_pointers.reserve(operations.size()); - for (auto& op : operations) op_pointers.push_back(op.get()); - - Partitioner partitioner(std::move(op_pointers)); - auto strategy = partitioner.partition_stores(); - - for (auto& op : operations) op->launch(strategy.get()); -} +void Runtime::submit(std::unique_ptr op) { impl_->submit(std::move(op)); } LogicalStore Runtime::create_store(std::unique_ptr type, int32_t dim) { - auto storage = std::make_shared(dim, std::move(type)); - return LogicalStore(std::make_shared(std::move(storage))); + return LogicalStore(impl_->create_store(std::move(type), dim)); } LogicalStore Runtime::create_store(const Type& type, int32_t dim) @@ -501,8 +179,7 @@ LogicalStore Runtime::create_store(const Shape& extents, std::unique_ptr type, bool optimize_scalar /*=false*/) { - auto storage = std::make_shared(extents, std::move(type), optimize_scalar); - return LogicalStore(std::make_shared(std::move(storage))); + return LogicalStore(impl_->create_store(extents, std::move(type), optimize_scalar)); } LogicalStore Runtime::create_store(const Shape& extents, @@ -514,500 +191,48 @@ LogicalStore Runtime::create_store(const Shape& extents, LogicalStore Runtime::create_store(const Scalar& scalar) { - Shape extents{1}; - auto future = create_future(scalar.ptr(), scalar.size()); - auto storage = std::make_shared(extents, scalar.type().clone(), future); - return LogicalStore(std::make_shared(std::move(storage))); + return LogicalStore(impl_->create_store(scalar)); } -uint32_t Runtime::max_pending_exceptions() const { return max_pending_exceptions_; } +uint32_t Runtime::max_pending_exceptions() const { return impl_->max_pending_exceptions(); } void Runtime::set_max_pending_exceptions(uint32_t max_pending_exceptions) { - uint32_t old_value = max_pending_exceptions_; - max_pending_exceptions_ = max_pending_exceptions; - if (old_value > max_pending_exceptions_) raise_pending_task_exception(); + impl_->set_max_pending_exceptions(max_pending_exceptions); } -void Runtime::raise_pending_task_exception() -{ - auto exn = check_pending_task_exception(); - if (exn.has_value()) throw exn.value(); -} +void Runtime::raise_pending_task_exception() { impl_->raise_pending_task_exception(); } std::optional Runtime::check_pending_task_exception() { - // If there's already an outstanding exception from the previous scan, we just return that. - if (!outstanding_exceptions_.empty()) { - std::optional result = outstanding_exceptions_.front(); - outstanding_exceptions_.pop_front(); - return result; - } - - // Othrewise, we unpack all pending exceptions and push them to the outstanding exception queue - for (auto& pending_exception : pending_exceptions_) { - auto returned_exception = pending_exception.get_result(); - auto result = returned_exception.to_task_exception(); - if (result.has_value()) outstanding_exceptions_.push_back(result.value()); - } - pending_exceptions_.clear(); - return outstanding_exceptions_.empty() ? std::nullopt : check_pending_task_exception(); -} - -void Runtime::record_pending_exception(const Legion::Future& pending_exception) -{ - pending_exceptions_.push_back(pending_exception); - if (outstanding_exceptions_.size() + pending_exceptions_.size() >= max_pending_exceptions_) - raise_pending_task_exception(); -} - -uint64_t Runtime::get_unique_store_id() { return next_store_id_++; } - -uint64_t Runtime::get_unique_storage_id() { return next_storage_id_++; } - -std::shared_ptr Runtime::create_region_field(const Shape& extents, - uint32_t field_size) -{ - DomainPoint lo, hi; - hi.dim = lo.dim = static_cast(extents.size()); - assert(lo.dim <= LEGION_MAX_DIM); - for (int32_t dim = 0; dim < lo.dim; ++dim) lo[dim] = 0; - for (int32_t dim = 0; dim < lo.dim; ++dim) hi[dim] = extents[dim] - 1; - - Domain shape(lo, hi); - auto fld_mgr = runtime_->find_or_create_field_manager(shape, field_size); - return fld_mgr->allocate_field(); -} - -std::shared_ptr Runtime::import_region_field(Legion::LogicalRegion region, - Legion::FieldID field_id, - uint32_t field_size) -{ - // TODO: This is a blocking operation. We should instead use index spaces as keys to field - // managers - auto shape = legion_runtime_->get_index_space_domain(legion_context_, region.get_index_space()); - auto fld_mgr = runtime_->find_or_create_field_manager(shape, field_size); - return fld_mgr->import_field(region, field_id); -} - -RegionField Runtime::map_region_field(LibraryContext* context, LogicalRegionField* rf) -{ - auto root_region = rf->get_root().region(); - auto field_id = rf->field_id(); - - Legion::PhysicalRegion pr; - - RegionFieldID key(root_region, field_id); - auto finder = inline_mapped_.find(key); - if (inline_mapped_.end() == finder) { - Legion::RegionRequirement req(root_region, READ_WRITE, EXCLUSIVE, root_region); - req.add_field(field_id); - - auto mapper_id = context->get_mapper_id(); - // TODO: We need to pass the metadata about logical store - Legion::InlineLauncher launcher(req, mapper_id); - pr = legion_runtime_->map_region(legion_context_, launcher); - inline_mapped_[key] = pr; - } else - pr = finder->second; - physical_region_refs_.add(pr); - return RegionField(rf->dim(), pr, field_id); + return impl_->check_pending_task_exception(); } -void Runtime::unmap_physical_region(Legion::PhysicalRegion pr) -{ - if (physical_region_refs_.remove(pr)) { - // The last user of this inline mapping was removed, so remove it from our cache and unmap. - std::vector fields; - pr.get_fields(fields); - assert(fields.size() == 1); - RegionFieldID key(pr.get_logical_region(), fields[0]); - auto finder = inline_mapped_.find(key); - assert(finder != inline_mapped_.end() && finder->second == pr); - inline_mapped_.erase(finder); - legion_runtime_->unmap_region(legion_context_, pr); - } -} +void Runtime::issue_execution_fence(bool block /*=false*/) { impl_->issue_execution_fence(block); } -size_t Runtime::num_inline_mapped() const { return inline_mapped_.size(); } +const mapping::MachineDesc& Runtime::get_machine() const { return impl_->get_machine(); } -RegionManager* Runtime::find_or_create_region_manager(const Domain& shape) +/*static*/ Runtime* Runtime::get_runtime() { - auto finder = region_managers_.find(shape); - if (finder != region_managers_.end()) - return finder->second; - else { - auto rgn_mgr = new RegionManager(this, shape); - region_managers_[shape] = rgn_mgr; - return rgn_mgr; - } -} + static Runtime* the_runtime{nullptr}; + if (nullptr == the_runtime) { + auto* impl = detail::Runtime::get_runtime(); + if (!impl->initialized()) + throw std::runtime_error( + "Legate runtime has not been initialized. Please invoke legate::start to use the runtime"); -FieldManager* Runtime::find_or_create_field_manager(const Domain& shape, uint32_t field_size) -{ - auto key = FieldManagerKey(shape, field_size); - auto finder = field_managers_.find(key); - if (finder != field_managers_.end()) - return finder->second; - else { - auto fld_mgr = new FieldManager(this, shape, field_size); - field_managers_[key] = fld_mgr; - return fld_mgr; + the_runtime = new Runtime(impl); } + return the_runtime; } -PartitionManager* Runtime::partition_manager() const { return partition_manager_; } - -ProvenanceManager* Runtime::provenance_manager() const { return provenance_manager_; } - -Legion::IndexSpace Runtime::find_or_create_index_space(const Domain& shape) -{ - assert(nullptr != legion_context_); - auto finder = index_spaces_.find(shape); - if (finder != index_spaces_.end()) - return finder->second; - else { - auto is = legion_runtime_->create_index_space(legion_context_, shape); - index_spaces_[shape] = is; - return is; - } -} - -Legion::IndexPartition Runtime::create_restricted_partition( - const Legion::IndexSpace& index_space, - const Legion::IndexSpace& color_space, - Legion::PartitionKind kind, - const Legion::DomainTransform& transform, - const Domain& extent) -{ - return legion_runtime_->create_partition_by_restriction( - legion_context_, index_space, color_space, transform, extent, kind); -} - -Legion::IndexPartition Runtime::create_weighted_partition(const Legion::IndexSpace& index_space, - const Legion::IndexSpace& color_space, - const Legion::FutureMap& weights) -{ - return legion_runtime_->create_partition_by_weights( - legion_context_, index_space, weights, color_space); -} - -Legion::FieldSpace Runtime::create_field_space() -{ - assert(nullptr != legion_context_); - return legion_runtime_->create_field_space(legion_context_); -} - -Legion::LogicalRegion Runtime::create_region(const Legion::IndexSpace& index_space, - const Legion::FieldSpace& field_space) -{ - assert(nullptr != legion_context_); - return legion_runtime_->create_logical_region(legion_context_, index_space, field_space); -} - -Legion::LogicalPartition Runtime::create_logical_partition( - const Legion::LogicalRegion& logical_region, const Legion::IndexPartition& index_partition) -{ - assert(nullptr != legion_context_); - return legion_runtime_->get_logical_partition(legion_context_, logical_region, index_partition); -} - -Legion::LogicalRegion Runtime::get_subregion(const Legion::LogicalPartition& partition, - const Legion::DomainPoint& color) -{ - assert(nullptr != legion_context_); - return legion_runtime_->get_logical_subregion_by_color(legion_context_, partition, color); -} - -Legion::LogicalRegion Runtime::find_parent_region(const Legion::LogicalRegion& region) -{ - auto result = region; - while (legion_runtime_->has_parent_logical_partition(legion_context_, result)) { - auto partition = legion_runtime_->get_parent_logical_partition(legion_context_, result); - result = legion_runtime_->get_parent_logical_region(legion_context_, partition); - } - return result; -} - -Legion::Future Runtime::create_future(const void* data, size_t datalen) const -{ - return Legion::Future::from_untyped_pointer(data, datalen); -} - -Legion::FieldID Runtime::allocate_field(const Legion::FieldSpace& field_space, size_t field_size) -{ - assert(nullptr != legion_context_); - auto allocator = legion_runtime_->create_field_allocator(legion_context_, field_space); - return allocator.allocate_field(field_size); -} - -Legion::FieldID Runtime::allocate_field(const Legion::FieldSpace& field_space, - Legion::FieldID field_id, - size_t field_size) -{ - assert(nullptr != legion_context_); - auto allocator = legion_runtime_->create_field_allocator(legion_context_, field_space); - return allocator.allocate_field(field_size, field_id); -} - -Domain Runtime::get_index_space_domain(const Legion::IndexSpace& index_space) const -{ - assert(nullptr != legion_context_); - return legion_runtime_->get_index_space_domain(legion_context_, index_space); -} - -namespace { - -Legion::DomainPoint _delinearize_future_map(const DomainPoint& point, - const Domain& domain, - const Domain& range) -{ - assert(range.dim == 1); - DomainPoint result; - result.dim = 1; - - int32_t ndim = domain.dim; - int64_t idx = point[0]; - for (int32_t dim = 1; dim < ndim; ++dim) { - int64_t extent = domain.rect_data[dim + ndim] - domain.rect_data[dim] + 1; - idx = idx * extent + point[dim]; - } - result[0] = idx; - return result; -} - -} // namespace - -Legion::FutureMap Runtime::delinearize_future_map(const Legion::FutureMap& future_map, - const Legion::IndexSpace& new_domain) const -{ - return legion_runtime_->transform_future_map( - legion_context_, future_map, new_domain, _delinearize_future_map); -} - -std::pair Runtime::create_barriers(size_t num_tasks) -{ - auto arrival_barrier = legion_runtime_->create_phase_barrier(legion_context_, num_tasks); - auto wait_barrier = legion_runtime_->advance_phase_barrier(legion_context_, arrival_barrier); - return std::make_pair(arrival_barrier, wait_barrier); -} - -void Runtime::destroy_barrier(Legion::PhaseBarrier barrier) -{ - legion_runtime_->destroy_phase_barrier(legion_context_, barrier); -} - -Legion::Future Runtime::dispatch(Legion::TaskLauncher* launcher, - std::vector* output_requirements) -{ - assert(nullptr != legion_context_); - return legion_runtime_->execute_task(legion_context_, *launcher, output_requirements); -} - -Legion::FutureMap Runtime::dispatch(Legion::IndexTaskLauncher* launcher, - std::vector* output_requirements) -{ - assert(nullptr != legion_context_); - return legion_runtime_->execute_index_space(legion_context_, *launcher, output_requirements); -} - -Legion::Future Runtime::extract_scalar(const Legion::Future& result, uint32_t idx) const -{ - auto& machine = get_machine(); - auto& provenance = provenance_manager()->get_provenance(); - auto variant = mapping::to_variant_code(machine.preferred_target); - TaskLauncher launcher( - core_context_, machine, provenance, LEGATE_CORE_EXTRACT_SCALAR_TASK_ID, variant); - launcher.add_future(result); - launcher.add_scalar(Scalar(idx)); - return launcher.execute_single(); -} - -Legion::FutureMap Runtime::extract_scalar(const Legion::FutureMap& result, - uint32_t idx, - const Legion::Domain& launch_domain) const -{ - auto& machine = get_machine(); - auto& provenance = provenance_manager()->get_provenance(); - auto variant = mapping::to_variant_code(machine.preferred_target); - TaskLauncher launcher( - core_context_, machine, provenance, LEGATE_CORE_EXTRACT_SCALAR_TASK_ID, variant); - launcher.add_future_map(result); - launcher.add_scalar(Scalar(idx)); - return launcher.execute(launch_domain); -} - -Legion::Future Runtime::reduce_future_map(const Legion::FutureMap& future_map, - int32_t reduction_op) const -{ - return legion_runtime_->reduce_future_map(legion_context_, - future_map, - reduction_op, - false /*deterministic*/, - core_context_->get_mapper_id()); -} - -Legion::Future Runtime::reduce_exception_future_map(const Legion::FutureMap& future_map) const -{ - auto reduction_op = core_context_->get_reduction_op_id(LEGATE_CORE_JOIN_EXCEPTION_OP); - return legion_runtime_->reduce_future_map(legion_context_, - future_map, - reduction_op, - false /*deterministic*/, - core_context_->get_mapper_id(), - LEGATE_CORE_JOIN_EXCEPTION_TAG); -} - -void Runtime::issue_execution_fence(bool block /*=false*/) -{ - flush_scheduling_window(); - // FIXME: This needs to be a Legate operation - auto future = legion_runtime_->issue_execution_fence(legion_context_); - if (block) future.wait(); -} - -void Runtime::initialize_toplevel_machine() -{ - auto mapper_id = core_context_->get_mapper_id(); - auto num_nodes = get_tunable(mapper_id, LEGATE_CORE_TUNABLE_NUM_NODES); - - auto num_gpus = get_tunable(mapper_id, LEGATE_CORE_TUNABLE_TOTAL_GPUS); - auto num_omps = get_tunable(mapper_id, LEGATE_CORE_TUNABLE_TOTAL_OMPS); - auto num_cpus = get_tunable(mapper_id, LEGATE_CORE_TUNABLE_TOTAL_CPUS); - - auto create_range = [&num_nodes](int32_t num_procs) { - auto per_node_count = num_procs / num_nodes; - return mapping::ProcessorRange(0, num_procs, per_node_count); - }; - - mapping::MachineDesc machine({{mapping::TaskTarget::GPU, create_range(num_gpus)}, - {mapping::TaskTarget::OMP, create_range(num_omps)}, - {mapping::TaskTarget::CPU, create_range(num_cpus)}}); -#ifdef DEBUG_LEGATE - assert(machine_manager_ != nullptr); -#endif - - machine_manager_->push_machine(std::move(machine)); -} - -const mapping::MachineDesc& Runtime::get_machine() const -{ -#ifdef DEBUG_LEGATE - assert(machine_manager_ != nullptr); -#endif - return machine_manager_->get_machine(); -} - -Legion::ProjectionID Runtime::get_projection(int32_t src_ndim, const proj::SymbolicPoint& point) -{ -#ifdef DEBUG_LEGATE - log_legate.debug() << "Query projection {src_ndim: " << src_ndim << ", point: " << point << "}"; -#endif - - if (is_identity(src_ndim, point)) { -#ifdef DEBUG_LEGATE - log_legate.debug() << "Identity projection {src_ndim: " << src_ndim << ", point: " << point - << "}"; -#endif - return 0; - } - - ProjectionDesc key(src_ndim, point); - auto finder = registered_projections_.find(key); - if (registered_projections_.end() != finder) return finder->second; - - auto proj_id = core_context_->get_projection_id(next_projection_id_++); - - auto ndim = point.size(); - std::vector dims; - std::vector weights; - std::vector offsets; - for (auto& expr : point.data()) { - dims.push_back(expr.dim()); - weights.push_back(expr.weight()); - offsets.push_back(expr.offset()); - } - legate_register_affine_projection_functor( - src_ndim, ndim, dims.data(), weights.data(), offsets.data(), proj_id); - registered_projections_[key] = proj_id; - -#ifdef DEBUG_LEGATE - log_legate.debug() << "Register projection " << proj_id << " {src_ndim: " << src_ndim - << ", point: " << point << "}"; -#endif - - return proj_id; -} - -Legion::ProjectionID Runtime::get_delinearizing_projection() -{ - return core_context_->get_projection_id(LEGATE_CORE_DELINEARIZE_PROJ_ID); -} - -CommunicatorManager* Runtime::communicator_manager() const { return communicator_manager_; } - -MachineManager* Runtime::machine_manager() const { return machine_manager_; } - -/*static*/ void Runtime::initialize(int32_t argc, char** argv) -{ - Legion::Runtime::initialize(&argc, &argv, true /*filter legion and realm args*/); - Legion::Runtime::add_registration_callback(legate::core_library_bootstrapping_callback); -} - -/*static*/ int32_t Runtime::start(int32_t argc, char** argv) -{ - auto result = Legion::Runtime::start(argc, argv, true); - if (result != 0) { - log_legate.error("Legion Runtime failed to start."); - return result; - } - - // Get the runtime now that we've started it - auto runtime = Legion::Runtime::get_runtime(); - - // Then we can make this thread into an implicit top-level task - const char* toplevel_task_name = "Legate Core Toplevel Task"; - auto ctx = runtime->begin_implicit_task(LEGATE_CORE_TOPLEVEL_TASK_ID, - 0 /*mapper id*/, - Processor::LOC_PROC, - toplevel_task_name, - true /*control replicable*/); - Runtime::get_runtime()->post_startup_initialization(ctx); - - return result; -} - -int32_t Runtime::wait_for_shutdown() -{ - destroy(); - - // Mark that we are done excecuting the top-level task - // After this call the context is no longer valid - Legion::Runtime::get_runtime()->finish_implicit_task(legion_context_); - - // The previous call is asynchronous so we still need to - // wait for the shutdown of the runtime to complete - return Legion::Runtime::wait_for_shutdown(); -} - -/*static*/ Runtime* Runtime::get_runtime() { return Runtime::runtime_; } - -/*static*/ void Runtime::create_runtime(Legion::Runtime* legion_runtime) -{ - runtime_ = new Runtime(legion_runtime); -} - -void Runtime::destroy() -{ - issue_execution_fence(); - communicator_manager_->destroy(); -} +Runtime::~Runtime() {} -void initialize(int32_t argc, char** argv) { Runtime::initialize(argc, argv); } +Runtime::Runtime(detail::Runtime* impl) : impl_(impl) {} -int32_t start(int32_t argc, char** argv) { return Runtime::start(argc, argv); } +int32_t start(int32_t argc, char** argv) { return detail::Runtime::start(argc, argv); } -int32_t wait_for_shutdown() { return Runtime::get_runtime()->wait_for_shutdown(); } +int32_t finish() { return detail::Runtime::get_runtime()->finish(); } } // namespace legate @@ -1017,7 +242,7 @@ void legate_core_perform_registration() { // Tell the runtime about our registration callback so we can register ourselves // Make sure it is global so this shared object always gets loaded on all nodes - Legion::Runtime::perform_registration_callback(legate::core_library_registration_callback, + Legion::Runtime::perform_registration_callback(legate::detail::registration_callback_for_python, true /*global*/); } } diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 1a2708debf..271ab246dd 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -1,4 +1,4 @@ -/* Copyright 2021-2022 NVIDIA Corporation +/* Copyright 2021-2023 NVIDIA Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,18 +19,11 @@ #include #include -#include "legion.h" - -#include - +#include "core/data/logical_store.h" #include "core/data/shape.h" #include "core/data/store.h" -#include "core/legate_c.h" -#include "core/mapping/machine.h" #include "core/runtime/resource.h" #include "core/task/exception.h" -#include "core/type/type_info.h" -#include "core/utilities/multi_set.h" #include "core/utilities/typedefs.h" /** @defgroup runtime Runtime and library contexts @@ -44,6 +37,7 @@ namespace legate { class LibraryContext; class Scalar; +class Type; extern uint32_t extract_env(const char* env_name, const uint32_t default_value, @@ -80,6 +74,9 @@ struct Core { template static void perform_registration(); + private: + static void perform_callback(Legion::RegistrationCallbackFnptr callback); + public: // Configuration settings static bool show_progress_requested; @@ -90,17 +87,8 @@ struct Core { }; class AutoTask; -class CommunicatorManager; -class FieldManager; -class LogicalRegionField; -class LogicalStore; -class MachineManager; class ManualTask; class Operation; -class PartitionManager; -class ProvenanceManager; -class RegionManager; -class Tiling; /** * @ingroup runtime @@ -111,15 +99,12 @@ class Tiling; * and communicator management. Legate libraries are free of all these details about * distribute programming and can focus on their domain logics. */ -class Runtime { - public: - Runtime(Legion::Runtime* legion_runtime); - ~Runtime(); - public: - friend void initialize(int32_t argc, char** argv); - friend int32_t start(int32_t argc, char** argv); +namespace detail { +class Runtime; +} // namespace detail +class Runtime { public: /** * @brief Find a library @@ -152,26 +137,9 @@ class Runtime { LibraryContext* create_library(const std::string& library_name, const ResourceConfig& config = ResourceConfig{}, std::unique_ptr mapper = nullptr); - - public: - uint32_t get_type_uid(); void record_reduction_operator(int32_t type_uid, int32_t op_kind, int32_t legion_op_id); - int32_t find_reduction_operator(int32_t type_uid, int32_t op_kind) const; - - public: - void enter_callback(); - void exit_callback(); - bool is_in_callback() const; - - public: - void post_startup_initialization(Legion::Context legion_context); public: - template - T get_tunable(Legion::MapperID mapper_id, int64_t tunable_id); - - public: - mapping::MachineDesc slice_machine_for_task(LibraryContext* library, int64_t task_id); /** * @brief Create an AutoTask * @@ -193,7 +161,6 @@ class Runtime { std::unique_ptr create_task(LibraryContext* library, int64_t task_id, const Shape& launch_shape); - void flush_scheduling_window(); /** * @brief Submits an operation for execution * @@ -280,72 +247,6 @@ class Runtime { * @brief Returns the first pending exception. */ std::optional check_pending_task_exception(); - void record_pending_exception(const Legion::Future& pending_exception); - - public: - uint64_t get_unique_store_id(); - uint64_t get_unique_storage_id(); - - public: - std::shared_ptr create_region_field(const Shape& extents, - uint32_t field_size); - std::shared_ptr import_region_field(Legion::LogicalRegion region, - Legion::FieldID field_id, - uint32_t field_size); - RegionField map_region_field(LibraryContext* context, LogicalRegionField* region_field); - void unmap_physical_region(Legion::PhysicalRegion pr); - size_t num_inline_mapped() const; - - public: - RegionManager* find_or_create_region_manager(const Legion::Domain& shape); - FieldManager* find_or_create_field_manager(const Legion::Domain& shape, uint32_t field_size); - CommunicatorManager* communicator_manager() const; - MachineManager* machine_manager() const; - PartitionManager* partition_manager() const; - ProvenanceManager* provenance_manager() const; - - public: - Legion::IndexSpace find_or_create_index_space(const Legion::Domain& shape); - Legion::IndexPartition create_restricted_partition(const Legion::IndexSpace& index_space, - const Legion::IndexSpace& color_space, - Legion::PartitionKind kind, - const Legion::DomainTransform& transform, - const Legion::Domain& extent); - Legion::IndexPartition create_weighted_partition(const Legion::IndexSpace& index_space, - const Legion::IndexSpace& color_space, - const Legion::FutureMap& weights); - Legion::FieldSpace create_field_space(); - Legion::LogicalRegion create_region(const Legion::IndexSpace& index_space, - const Legion::FieldSpace& field_space); - Legion::LogicalPartition create_logical_partition(const Legion::LogicalRegion& logical_region, - const Legion::IndexPartition& index_partition); - Legion::LogicalRegion get_subregion(const Legion::LogicalPartition& partition, - const Legion::DomainPoint& color); - Legion::LogicalRegion find_parent_region(const Legion::LogicalRegion& region); - Legion::Future create_future(const void* data, size_t datalen) const; - Legion::FieldID allocate_field(const Legion::FieldSpace& field_space, size_t field_size); - Legion::FieldID allocate_field(const Legion::FieldSpace& field_space, - Legion::FieldID field_id, - size_t field_size); - Legion::Domain get_index_space_domain(const Legion::IndexSpace& index_space) const; - Legion::FutureMap delinearize_future_map(const Legion::FutureMap& future_map, - const Legion::IndexSpace& new_domain) const; - std::pair create_barriers(size_t num_tasks); - void destroy_barrier(Legion::PhaseBarrier barrier); - - public: - Legion::Future dispatch(Legion::TaskLauncher* launcher, - std::vector* output_requirements = nullptr); - Legion::FutureMap dispatch(Legion::IndexTaskLauncher* launcher, - std::vector* output_requirements = nullptr); - - public: - Legion::Future extract_scalar(const Legion::Future& result, uint32_t idx) const; - Legion::FutureMap extract_scalar(const Legion::FutureMap& result, - uint32_t idx, - const Legion::Domain& launch_domain) const; - Legion::Future reduce_future_map(const Legion::FutureMap& future_map, int32_t reduction_op) const; - Legion::Future reduce_exception_future_map(const Legion::FutureMap& future_map) const; public: /** @@ -360,7 +261,6 @@ class Runtime { void issue_execution_fence(bool block = false); public: - void initialize_toplevel_machine(); /** * @brief Returns the machine of the current scope * @@ -368,17 +268,6 @@ class Runtime { */ const mapping::MachineDesc& get_machine() const; - public: - Legion::ProjectionID get_projection(int32_t src_ndim, const proj::SymbolicPoint& point); - Legion::ProjectionID get_delinearizing_projection(); - - private: - void schedule(std::vector> operations); - - public: - static void initialize(int32_t argc, char** argv); - static int32_t start(int32_t argc, char** argv); - public: /** * @brief Returns a singleton runtime object @@ -386,71 +275,14 @@ class Runtime { * @return The runtime object */ static Runtime* get_runtime(); - static void create_runtime(Legion::Runtime* legion_runtime); - void destroy(); - int32_t wait_for_shutdown(); - - private: - static Runtime* runtime_; - - private: - Legion::Runtime* legion_runtime_; - Legion::Context legion_context_{nullptr}; - LibraryContext* core_context_{nullptr}; - - private: - using FieldManagerKey = std::pair; - std::map field_managers_; - std::map region_managers_; - CommunicatorManager* communicator_manager_{nullptr}; - MachineManager* machine_manager_{nullptr}; - PartitionManager* partition_manager_{nullptr}; - ProvenanceManager* provenance_manager_{nullptr}; - - private: - std::map index_spaces_; - - private: - using ProjectionDesc = std::pair; - int64_t next_projection_id_{LEGATE_CORE_FIRST_DYNAMIC_FUNCTOR_ID}; - std::map registered_projections_; - - private: - std::vector> operations_; - size_t window_size_{1}; - uint64_t next_unique_id_{0}; + detail::Runtime* impl() { return impl_; } private: - using RegionFieldID = std::pair; - std::map inline_mapped_; - MultiSet physical_region_refs_; - uint64_t next_store_id_{1}; - uint64_t next_storage_id_{1}; - - private: - bool in_callback_{false}; - - private: - std::map libraries_{}; - - private: - uint32_t next_type_uid_; - std::map, int32_t> reduction_ops_{}; - - private: - uint32_t max_pending_exceptions_; - std::vector pending_exceptions_{}; - std::deque outstanding_exceptions_{}; + Runtime(detail::Runtime* runtime); + ~Runtime(); + detail::Runtime* impl_{nullptr}; }; -/** - * @brief Initializes the Legate runtime - * - * @param argc Number of command-line flags - * @param argv Command-line flags - */ -void initialize(int32_t argc, char** argv); - /** * @brief Starts the Legate runtime * @@ -470,7 +302,7 @@ int32_t start(int32_t argc, char** argv); * * @return Non-zero value when the runtime encountered a failure, 0 otherwise */ -int32_t wait_for_shutdown(); +int32_t finish(); } // namespace legate diff --git a/src/core/runtime/runtime.inl b/src/core/runtime/runtime.inl index 57a045c8bc..f10a7e8746 100644 --- a/src/core/runtime/runtime.inl +++ b/src/core/runtime/runtime.inl @@ -21,14 +21,6 @@ namespace legate { -template -T Runtime::get_tunable(Legion::MapperID mapper_id, int64_t tunable_id) -{ - Legion::TunableLauncher launcher(tunable_id, mapper_id, 0, sizeof(T)); - auto future = legion_runtime_->select_tunable_value(legion_context_, launcher); - return future.get_result(); -} - namespace detail { template @@ -44,15 +36,7 @@ void invoke_legate_registration_callback(Legion::Machine, template /*static*/ void Core::perform_registration() { - auto runtime = Runtime::get_runtime(); - if (runtime->is_in_callback()) - CALLBACK(); - else { - runtime->enter_callback(); - Legion::Runtime::perform_registration_callback( - detail::invoke_legate_registration_callback, true /*global*/); - runtime->exit_callback(); - } + perform_callback(detail::invoke_legate_registration_callback); } } // namespace legate diff --git a/src/core/runtime/tracker.cc b/src/core/runtime/tracker.cc index 1a96e914a9..e96652f9b9 100644 --- a/src/core/runtime/tracker.cc +++ b/src/core/runtime/tracker.cc @@ -16,9 +16,9 @@ #include "core/runtime/tracker.h" -#include "core/runtime/machine_manager.h" -#include "core/runtime/provenance_manager.h" -#include "core/runtime/runtime.h" +#include "core/runtime/detail/machine_manager.h" +#include "core/runtime/detail/provenance_manager.h" +#include "core/runtime/detail/runtime.h" namespace legate { @@ -28,19 +28,19 @@ namespace legate { ProvenanceTracker::ProvenanceTracker(const std::string& p) { - auto* runtime = Runtime::get_runtime(); + auto* runtime = detail::Runtime::get_runtime(); runtime->provenance_manager()->push_provenance(p); } ProvenanceTracker::~ProvenanceTracker() { - auto* runtime = Runtime::get_runtime(); + auto* runtime = detail::Runtime::get_runtime(); runtime->provenance_manager()->pop_provenance(); } const std::string& ProvenanceTracker::get_current_provenance() const { - return Runtime::get_runtime()->provenance_manager()->get_provenance(); + return detail::Runtime::get_runtime()->provenance_manager()->get_provenance(); } //////////////////////////////////////////// @@ -49,8 +49,8 @@ const std::string& ProvenanceTracker::get_current_provenance() const MachineTracker::MachineTracker(const mapping::MachineDesc& machine) { - auto* runtime = Runtime::get_runtime(); - auto result = machine & Runtime::get_runtime()->get_machine(); + auto* runtime = detail::Runtime::get_runtime(); + auto result = machine & runtime->get_machine(); if (result.count() == 0) throw std::runtime_error("Empty machines cannot be used for resource scoping"); runtime->machine_manager()->push_machine(std::move(result)); @@ -58,13 +58,13 @@ MachineTracker::MachineTracker(const mapping::MachineDesc& machine) MachineTracker::~MachineTracker() { - auto* runtime = Runtime::get_runtime(); + auto* runtime = detail::Runtime::get_runtime(); runtime->machine_manager()->pop_machine(); } const mapping::MachineDesc& MachineTracker::get_current_machine() const { - return Runtime::get_runtime()->get_machine(); + return detail::Runtime::get_runtime()->get_machine(); } } // namespace legate diff --git a/src/core/task/return.cc b/src/core/task/return.cc index 4b68c3b66d..4d5ffc7aa8 100644 --- a/src/core/task/return.cc +++ b/src/core/task/return.cc @@ -67,65 +67,6 @@ const void* ReturnValue::ptr() const return acc.ptr(0); } -struct JoinReturnedException { - using LHS = ReturnedException; - using RHS = LHS; - - static const ReturnedException identity; - - template - static void apply(LHS& lhs, RHS rhs) - { -#ifdef DEBUG_LEGATE - assert(EXCLUSIVE); -#endif - if (lhs.raised() || !rhs.raised()) return; - lhs = rhs; - } - - template - static void fold(RHS& rhs1, RHS rhs2) - { -#ifdef DEBUG_LEGATE - assert(EXCLUSIVE); -#endif - if (rhs1.raised() || !rhs2.raised()) return; - rhs1 = rhs2; - } -}; - -/*static*/ const ReturnedException JoinReturnedException::identity; - -static void pack_returned_exception(const ReturnedException& value, void*& ptr, size_t& size) -{ - auto new_size = value.legion_buffer_size(); - if (new_size > size) { - size = new_size; - ptr = realloc(ptr, new_size); - } - value.legion_serialize(ptr); -} - -static void returned_exception_init(const Legion::ReductionOp* reduction_op, - void*& ptr, - size_t& size) -{ - pack_returned_exception(JoinReturnedException::identity, ptr, size); -} - -static void returned_exception_fold(const Legion::ReductionOp* reduction_op, - void*& lhs_ptr, - size_t& lhs_size, - const void* rhs_ptr) - -{ - ReturnedException lhs, rhs; - lhs.legion_deserialize(lhs_ptr); - rhs.legion_deserialize(rhs_ptr); - JoinReturnedException::fold(lhs, rhs); - pack_returned_exception(lhs, lhs_ptr, lhs_size); -} - ReturnedException::ReturnedException(int32_t index, const std::string& error_message) : raised_(true), index_(index), error_message_(error_message) { @@ -331,6 +272,69 @@ void ReturnValues::finalize(Legion::Context legion_context) const return_buffer.finalize(legion_context); } +} // namespace legate + +namespace legate::detail { + +struct JoinReturnedException { + using LHS = ReturnedException; + using RHS = LHS; + + static const ReturnedException identity; + + template + static void apply(LHS& lhs, RHS rhs) + { +#ifdef DEBUG_LEGATE + assert(EXCLUSIVE); +#endif + if (lhs.raised() || !rhs.raised()) return; + lhs = rhs; + } + + template + static void fold(RHS& rhs1, RHS rhs2) + { +#ifdef DEBUG_LEGATE + assert(EXCLUSIVE); +#endif + if (rhs1.raised() || !rhs2.raised()) return; + rhs1 = rhs2; + } +}; + +/*static*/ const ReturnedException JoinReturnedException::identity; + +static void pack_returned_exception(const ReturnedException& value, void*& ptr, size_t& size) +{ + auto new_size = value.legion_buffer_size(); + if (new_size > size) { + size = new_size; + ptr = realloc(ptr, new_size); + } + value.legion_serialize(ptr); +} + +static void returned_exception_init(const Legion::ReductionOp* reduction_op, + void*& ptr, + size_t& size) +{ + pack_returned_exception(JoinReturnedException::identity, ptr, size); +} + +static void returned_exception_fold(const Legion::ReductionOp* reduction_op, + void*& lhs_ptr, + size_t& lhs_size, + const void* rhs_ptr) + +{ + ReturnedException lhs, rhs; + lhs.legion_deserialize(lhs_ptr); + rhs.legion_deserialize(rhs_ptr); + JoinReturnedException::fold(lhs, rhs); + pack_returned_exception(lhs, lhs_ptr, lhs_size); +} + void register_exception_reduction_op(Legion::Runtime* runtime, const LibraryContext* context) { auto redop_id = context->get_reduction_op_id(LEGATE_CORE_JOIN_EXCEPTION_OP); @@ -339,4 +343,4 @@ void register_exception_reduction_op(Legion::Runtime* runtime, const LibraryCont redop_id, redop, returned_exception_init, returned_exception_fold); } -} // namespace legate +} // namespace legate::detail diff --git a/src/core/type/type_info.cc b/src/core/type/type_info.cc index 420d70f5fc..bb3fa4b76c 100644 --- a/src/core/type/type_info.cc +++ b/src/core/type/type_info.cc @@ -17,7 +17,7 @@ #include #include -#include "core/runtime/runtime.h" +#include "core/runtime/detail/runtime.h" #include "core/type/type_info.h" #include "core/type/type_traits.h" #include "core/utilities/buffer_builder.h" @@ -69,12 +69,12 @@ Type::Type(Code c) : code(c) {} void Type::record_reduction_operator(int32_t op_kind, int32_t global_op_id) const { - Runtime::get_runtime()->record_reduction_operator(uid(), op_kind, global_op_id); + detail::Runtime::get_runtime()->record_reduction_operator(uid(), op_kind, global_op_id); } int32_t Type::find_reduction_operator(int32_t op_kind) const { - return Runtime::get_runtime()->find_reduction_operator(uid(), op_kind); + return detail::Runtime::get_runtime()->find_reduction_operator(uid(), op_kind); } int32_t Type::find_reduction_operator(ReductionOpKind op_kind) const @@ -255,14 +255,14 @@ std::unique_ptr fixed_array_type(std::unique_ptr element_type, uint32_t N) noexcept(false) { return std::make_unique( - Runtime::get_runtime()->get_type_uid(), std::move(element_type), N); + detail::Runtime::get_runtime()->get_type_uid(), std::move(element_type), N); } std::unique_ptr struct_type(std::vector>&& field_types, bool align) noexcept(false) { return std::make_unique( - Runtime::get_runtime()->get_type_uid(), std::move(field_types), align); + detail::Runtime::get_runtime()->get_type_uid(), std::move(field_types), align); } std::ostream& operator<<(std::ostream& ostream, const Type::Code& code) diff --git a/src/legate.h b/src/legate.h index 5da162f91c..9305dce523 100644 --- a/src/legate.h +++ b/src/legate.h @@ -32,9 +32,7 @@ #include "core/mapping/mapping.h" #include "core/partitioning/constraint.h" #include "core/partitioning/partition.h" -#include "core/runtime/machine_manager.h" #include "core/runtime/operation.h" -#include "core/runtime/provenance_manager.h" #include "core/runtime/runtime.h" #include "core/runtime/tracker.h" #include "core/task/exception.h" diff --git a/tests/cpp/integration/inline_map.cc b/tests/cpp/integration/inline_map.cc index fea669c84a..823af879ae 100644 --- a/tests/cpp/integration/inline_map.cc +++ b/tests/cpp/integration/inline_map.cc @@ -16,6 +16,7 @@ #include +#include "core/runtime/detail/runtime.h" #include "legate.h" namespace inline_map { @@ -50,9 +51,9 @@ void test_mapped_regions_leak(legate::Runtime* runtime, legate::LibraryContext* auto l_store = runtime->create_store({5}, legate::int64()); auto p_store = l_store.get_physical_store(context); EXPECT_FALSE(p_store->is_future()); - EXPECT_EQ(runtime->num_inline_mapped(), 1); + EXPECT_EQ(runtime->impl()->num_inline_mapped(), 1); } - EXPECT_EQ(runtime->num_inline_mapped(), 0); + EXPECT_EQ(runtime->impl()->num_inline_mapped(), 0); } void test_inline_map_future(legate::Runtime* runtime, legate::LibraryContext* context) diff --git a/tests/cpp/integration/provenance.cc b/tests/cpp/integration/provenance.cc index cc4241bcfa..00a9b3c2c5 100644 --- a/tests/cpp/integration/provenance.cc +++ b/tests/cpp/integration/provenance.cc @@ -16,13 +16,12 @@ #include -#include "core/mapping/mapping.h" +#include "core/runtime/detail/runtime.h" #include "legate.h" namespace provenance { -static const char* library_name = "provenance"; -static legate::Logger logger(library_name); +static const char* library_name = "test_provenance"; enum TaskIDs { PROVENANCE = 0, @@ -51,7 +50,7 @@ void test_manual_provenance(legate::LibraryContext* context) { auto runtime = legate::Runtime::get_runtime(); std::string provenance = "test_manual_provenance"; - runtime->provenance_manager()->set_provenance(provenance); + runtime->impl()->provenance_manager()->set_provenance(provenance); // auto task auto task = runtime->create_task(context, PROVENANCE); task->add_scalar_arg(legate::Scalar(provenance)); @@ -62,8 +61,8 @@ void test_push_provenance(legate::LibraryContext* context) { auto runtime = legate::Runtime::get_runtime(); std::string provenance = "test_push_provenance"; - runtime->provenance_manager()->push_provenance(provenance); - EXPECT_EQ(runtime->provenance_manager()->get_provenance(), provenance); + runtime->impl()->provenance_manager()->push_provenance(provenance); + EXPECT_EQ(runtime->impl()->provenance_manager()->get_provenance(), provenance); // auto task auto task = runtime->create_task(context, PROVENANCE); task->add_scalar_arg(legate::Scalar(provenance)); @@ -73,9 +72,9 @@ void test_push_provenance(legate::LibraryContext* context) void test_pop_provenance(legate::LibraryContext* context) { auto runtime = legate::Runtime::get_runtime(); - runtime->provenance_manager()->clear_all(); - runtime->provenance_manager()->push_provenance("some provenance for provenance task"); - runtime->provenance_manager()->pop_provenance(); + runtime->impl()->provenance_manager()->clear_all(); + runtime->impl()->provenance_manager()->push_provenance("some provenance for provenance task"); + runtime->impl()->provenance_manager()->pop_provenance(); // auto task auto task = runtime->create_task(context, PROVENANCE); std::string provenance = ""; @@ -86,17 +85,17 @@ void test_pop_provenance(legate::LibraryContext* context) void test_underflow(legate::LibraryContext* context) { auto runtime = legate::Runtime::get_runtime(); - runtime->provenance_manager()->clear_all(); - runtime->provenance_manager()->set_provenance("some provenance for provenance task"); - EXPECT_THROW(runtime->provenance_manager()->pop_provenance(), std::runtime_error); + runtime->impl()->provenance_manager()->clear_all(); + runtime->impl()->provenance_manager()->set_provenance("some provenance for provenance task"); + EXPECT_THROW(runtime->impl()->provenance_manager()->pop_provenance(), std::runtime_error); } void test_clear_provenance(legate::LibraryContext* context) { auto runtime = legate::Runtime::get_runtime(); - runtime->provenance_manager()->push_provenance("provenance for provenance task"); - runtime->provenance_manager()->push_provenance("another provenance"); - runtime->provenance_manager()->clear_all(); + runtime->impl()->provenance_manager()->push_provenance("provenance for provenance task"); + runtime->impl()->provenance_manager()->push_provenance("another provenance"); + runtime->impl()->provenance_manager()->clear_all(); // auto task auto task = runtime->create_task(context, PROVENANCE); std::string provenance = ""; @@ -110,7 +109,7 @@ void test_provenance_tracker(legate::LibraryContext* context) auto runtime = legate::Runtime::get_runtime(); // auto task auto task = runtime->create_task(context, PROVENANCE); - std::string provenance = "provenance.cc:109"; + std::string provenance = "provenance.cc:108"; task->add_scalar_arg(legate::Scalar(provenance)); runtime->submit(std::move(task)); } @@ -122,7 +121,7 @@ void test_nested_provenance_tracker(legate::LibraryContext* context) // The provenance string used by test_provenance_tracker should be popped out at this point auto runtime = legate::Runtime::get_runtime(); auto task = runtime->create_task(context, PROVENANCE); - std::string provenance = "provenance.cc:120"; + std::string provenance = "provenance.cc:119"; task->add_scalar_arg(legate::Scalar(provenance)); runtime->submit(std::move(task)); } diff --git a/tests/cpp/main.cc b/tests/cpp/main.cc index 747c335690..b5d5945ba8 100644 --- a/tests/cpp/main.cc +++ b/tests/cpp/main.cc @@ -21,12 +21,8 @@ class Environment : public ::testing::Environment { public: Environment(int argc, char** argv) : argc_(argc), argv_(argv) {} - void SetUp() override - { - legate::initialize(argc_, argv_); - EXPECT_EQ(legate::start(argc_, argv_), 0); - } - void TearDown() override { EXPECT_EQ(legate::wait_for_shutdown(), 0); } + void SetUp() override { EXPECT_EQ(legate::start(argc_, argv_), 0); } + void TearDown() override { EXPECT_EQ(legate::finish(), 0); } private: int argc_; From 4bda0af7472cd95b62e8a25ee53c0221ab153724 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Mon, 5 Jun 2023 22:37:17 -0700 Subject: [PATCH 0139/1425] Missing cmake file for nccl * Missing cmake file for nccl See merge request legate/legate.core.internal!49 --- tests/cpp/cmake/thirdparty/get_nccl.cmake | 34 +++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 tests/cpp/cmake/thirdparty/get_nccl.cmake diff --git a/tests/cpp/cmake/thirdparty/get_nccl.cmake b/tests/cpp/cmake/thirdparty/get_nccl.cmake new file mode 100644 index 0000000000..1aee52b6f5 --- /dev/null +++ b/tests/cpp/cmake/thirdparty/get_nccl.cmake @@ -0,0 +1,34 @@ +#============================================================================= +# Copyright 2022 NVIDIA Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +function(find_or_configure_nccl) + + if(TARGET NCCL::NCCL) + return() + endif() + + rapids_find_generate_module(NCCL + HEADER_NAMES nccl.h + LIBRARY_NAMES nccl + ) + + # Currently NCCL has no CMake build-system so we require + # it built and installed on the machine already + rapids_find_package(NCCL REQUIRED) + +endfunction() + +find_or_configure_nccl() From 3a5eb35357d2c31c9d0170a6fe301a1e5c260264 Mon Sep 17 00:00:00 2001 From: Vladislav Zhurba Date: Tue, 6 Jun 2023 10:28:28 -0700 Subject: [PATCH 0140/1425] Use cuNumeric arrays instead of custom * Rebase main fix * Leverage operator bool * Apply review * Use static helper to make IOArray * Use more robust method to find legate/cunumeric * Use cuNumeric arrays instead of custom See merge request legate/legate.core.internal!34 --- .gitignore | 2 + examples/cpp/CMakeLists.txt | 3 +- examples/cpp/build.sh | 8 ++- examples/cpp/io/io.cc | 99 +++++++++++-------------------- examples/cpp/io/task_io.cc | 98 ------------------------------ examples/cpp/io/task_io.h | 24 -------- examples/cpp/main.cc | 9 +-- src/core/partitioning/partition.h | 15 ++++- 8 files changed, 62 insertions(+), 196 deletions(-) diff --git a/.gitignore b/.gitignore index 9b9a465c9f..df14f41238 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,5 @@ _cmake_test_compile !cmake/versions.json legate.core.code-workspace *.prof +examples/cpp/build +tests/cpp/build diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt index 439fd11051..f41886acbe 100644 --- a/examples/cpp/CMakeLists.txt +++ b/examples/cpp/CMakeLists.txt @@ -32,6 +32,7 @@ set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) FetchContent_MakeAvailable(googletest) find_package(legate_core REQUIRED) +find_package(cunumeric REQUIRED) enable_testing() @@ -41,7 +42,7 @@ file(GLOB io_SRC ${PROJECT_SOURCE_DIR}/io/*.cc) add_executable(cpp_tests ${main_SRC} ${hello_SRC} ${io_SRC}) -target_link_libraries(cpp_tests PRIVATE legate::core PRIVATE GTest::gtest) +target_link_libraries(cpp_tests PRIVATE legate::core PRIVATE cunumeric::cunumeric PRIVATE GTest::gtest) include(GoogleTest) gtest_discover_tests(cpp_tests) diff --git a/examples/cpp/build.sh b/examples/cpp/build.sh index dd0fb9706a..3c3d8d2fcb 100755 --- a/examples/cpp/build.sh +++ b/examples/cpp/build.sh @@ -1,5 +1,9 @@ #!/bin/bash rm -rf build -cmake -B build -S . -D legate_core_ROOT=../../build -D CMAKE_BUILD_TYPE=Debug -cmake --build build -j 4 +legate_root=`python -c 'import legate.install_info as i; from pathlib import Path; print(Path(i.libpath).parent.resolve())'` +cunumeric_root=`python -c 'import cunumeric.install_info as i; from pathlib import Path; print(Path(i.libpath).parent.resolve())'` +echo "Using Legate at $legate_root" +echo "Using cuNumeric at $cunumeric_root" +cmake -B build -S . -D cunumeric_DIR=$cunumeric_root -D legate_core_ROOT=$legate_root -D CMAKE_BUILD_TYPE=Debug +cmake --build build -j 8 diff --git a/examples/cpp/io/io.cc b/examples/cpp/io/io.cc index f82b5f323e..89379e349d 100644 --- a/examples/cpp/io/io.cc +++ b/examples/cpp/io/io.cc @@ -18,6 +18,7 @@ #include #include "core/mapping/mapping.h" +#include "cunumeric.h" #include "legate.h" #include "task_io.h" @@ -100,56 +101,15 @@ class IOArray : public Array { task->add_scalar_arg(legate::Scalar(std::vector{tile_shape, tile_shape})); runtime->submit(std::move(task)); } -}; - -legate::LogicalStore iota(legate::LibraryContext* context, uint32_t size) -{ - auto runtime = legate::Runtime::get_runtime(); - auto task = runtime->create_task(context, task::legateio::IOTA); - auto output = runtime->create_store({size}, legate::int8(), true); - auto part = task->declare_partition(); - task->add_output(output, part); - runtime->submit(std::move(task)); - return output; -} - -legate::LogicalStore iota_2d(legate::LibraryContext* context, uint32_t size) -{ - auto runtime = legate::Runtime::get_runtime(); - auto task = runtime->create_task(context, task::legateio::IOTA_2D); - auto output = runtime->create_store({size, size}, legate::int8(), true); - auto part = task->declare_partition(); - task->add_output(output, part); - runtime->submit(std::move(task)); - return output; -} -void array_equal(legate::LibraryContext* context, IOArray c1, IOArray c2) -{ - int8_t bytearray = 1; - auto runtime = legate::Runtime::get_runtime(); - auto task_id = - c1.store().extents().size() == 1 ? task::legateio::EQUAL : task::legateio::EQUAL_2D; - auto task = runtime->create_task(context, task_id); - auto output = runtime->create_store(legate::Scalar(legate::int8(), &bytearray)); - auto redop = c1.store().type().find_reduction_operator(legate::ReductionOpKind::MUL); - - auto part1 = task->declare_partition(); - auto part2 = task->declare_partition(); - auto part3 = task->declare_partition(); - - task->add_input(c1.store(), part1); - task->add_input(c2.store(), part2); - task->add_reduction(output, redop, part3); - runtime->submit(std::move(task)); - - auto p_scalar = output.get_physical_store(context); - auto acc = p_scalar->read_accessor(); - int8_t output_value = static_cast(acc[{0}]); - EXPECT_EQ(output_value, 1); -} + static IOArray from_store(legate::LogicalStore store) + { + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(task::legateio::library_name); + return IOArray(context, store); + } +}; -// TODO: return struct from an interface supporting legate arrays IOArray read_file(legate::LibraryContext* context, std::string filename, std::unique_ptr dtype) @@ -166,7 +126,6 @@ IOArray read_file(legate::LibraryContext* context, return IOArray(context, output); } -// TODO: return struct from an interface supporting legate arrays IOArray read_file_parallel(legate::LibraryContext* context, std::string filename, std::unique_ptr dtype, @@ -304,7 +263,7 @@ IOArray read_even_tiles(legate::LibraryContext* context, std::string path) // Unlike AutoTask, manually parallelized tasks don't update the "key" // partition for their outputs. - output.set_key_partition(output_partition.partition().get()); + output.set_key_partition(runtime->get_machine(), output_partition.partition().get()); return IOArray(context, output); } @@ -316,11 +275,11 @@ TEST(Example, SingleFileIO) auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(task::legateio::library_name); - int n = 10; + uint32_t n = 10; std::string filename = "test.dat"; - auto c1_store = iota(context, n); - IOArray c1(context, c1_store); + auto src = cunumeric::arange(n).as_type(legate::int8()); + auto c1 = IOArray::from_store(src.get_store()); // Dump the IOArray to a file c1.to_file(filename); @@ -331,17 +290,16 @@ TEST(Example, SingleFileIO) // Read the file into a IOArray IOArray c2 = read_file(context, filename, legate::int8()); - c2.to_file(filename); - array_equal(context, c1, c2); + EXPECT_TRUE(cunumeric::array_equal(src, cunumeric::as_array(c2.store()))); // Read the file into a IOArray with a fixed degree of parallelism IOArray c3 = read_file_parallel(context, filename, legate::int8(), 2); - array_equal(context, c1, c3); + EXPECT_TRUE(cunumeric::array_equal(src, cunumeric::as_array(c3.store()))); // Read the file into a IOArray with the library-chosen degree of // parallelism IOArray c4 = read_file_parallel(context, filename, legate::int8()); - array_equal(context, c1, c4); + EXPECT_TRUE(cunumeric::array_equal(src, cunumeric::as_array(c4.store()))); } TEST(Example, EvenTilesIO) @@ -355,8 +313,11 @@ TEST(Example, EvenTilesIO) uint32_t store_shape = 8; uint32_t tile_shape = 3; - auto c1_store = iota_2d(context, store_shape); - IOArray c1(context, c1_store); + // Use cuNumeric to generate a random array to dump to a dataset + auto src = cunumeric::random({store_shape, store_shape}).as_type(legate::int8()); + + // Construct an IOArray from the cuNumeric ndarray + auto c1 = IOArray::from_store(src.get_store()); // Dump the IOArray to a dataset of even tiles c1.to_even_tiles(dataset_name, tile_shape); @@ -365,8 +326,15 @@ TEST(Example, EvenTilesIO) // Read the dataset into an IOArray IOArray c2 = read_even_tiles(context, dataset_name); - // TODO: Ideally we collapse the shapes to be the same - array_equal(context, c1, c2); + + // Convert the IOArray into a cuNumeric ndarray and perform a binary + // operation, just to confirm in the profile that the partition from the + // reader tasks is reused in the downstream tasks. + auto empty = + cunumeric::full({store_shape, store_shape}, cunumeric::Scalar(static_cast(0))); + auto c2_cunumeric = + cunumeric::add(cunumeric::as_array(c2.store()).as_type(legate::int64()), empty); + EXPECT_TRUE(cunumeric::array_equal(src, c2_cunumeric.as_type(legate::int8()))); } TEST(Example, UnevenTilesIO) @@ -379,8 +347,11 @@ TEST(Example, UnevenTilesIO) std::string dataset_name = "uneven_datafiles"; uint32_t store_shape = 8; - auto c1_store = iota_2d(context, store_shape); - IOArray c1(context, c1_store); + // Use cuNumeric to generate a random array to dump to a dataset + auto src = cunumeric::random({store_shape, store_shape}).as_type(legate::int8()); + + // Construct an IOArray from the cuNumeric ndarray + auto c1 = IOArray::from_store(src.get_store()); // Dump the IOArray to a dataset of even tiles c1.to_uneven_tiles(dataset_name); @@ -389,7 +360,7 @@ TEST(Example, UnevenTilesIO) // Read the dataset into an IOArray IOArray c2 = read_uneven_tiles(context, dataset_name); - array_equal(context, c1, c2); + EXPECT_TRUE(cunumeric::array_equal(src, cunumeric::as_array(c2.store()))); } } // namespace legateio diff --git a/examples/cpp/io/task_io.cc b/examples/cpp/io/task_io.cc index d6154db7db..7cf4446a01 100644 --- a/examples/cpp/io/task_io.cc +++ b/examples/cpp/io/task_io.cc @@ -37,10 +37,6 @@ void register_tasks() WriteEvenTilesTask::register_variants(context); WriteFileTask::register_variants(context); WriteUnevenTilesTask::register_variants(context); - IotaTask::register_variants(context); - Iota2DTask::register_variants(context); - EqualTask::register_variants(context); - EqualTask2D::register_variants(context); } namespace utils { @@ -366,100 +362,6 @@ struct header_write_fn { utils::write_to_file(context, dirname, input); } -/*static*/ void IotaTask::cpu_variant(legate::TaskContext& context) -{ - legate::Store& output = context.outputs().at(0); - legate::Rect<1> output_shape = output.shape<1>(); - auto out = output.write_accessor(); - - logger.debug() << "Iota task [" << output_shape.lo << "," << output_shape.hi << "]"; - - // i is a global index for the complete array - for (size_t i = output_shape.lo; i <= output_shape.hi; ++i) { out[i] = i + 1; } -} - -/*static*/ void Iota2DTask::cpu_variant(legate::TaskContext& context) -{ - legate::Store& output = context.outputs().at(0); - legate::Rect<2> output_shape = output.shape<2>(); - auto out = output.write_accessor(); - - logger.debug() << "Iota 2d task [" << output_shape.lo << "," << output_shape.hi << "]"; - - // i is a global index for the complete array - for (auto i = output_shape.lo[0]; i <= output_shape.hi[0]; ++i) { - for (auto j = output_shape.lo[1]; i <= output_shape.hi[1]; ++i) { - out[i][j] = i * output_shape.hi[0] + j; - } - } -} - -/*static*/ void EqualTask::cpu_variant(legate::TaskContext& context) -{ - legate::Store& input0 = context.inputs().at(0); - legate::Store& input1 = context.inputs().at(1); - - legate::Rect<1> input0_shape = input0.shape<1>(); - legate::Rect<1> input1_shape = input1.shape<1>(); - - auto in0 = input0.read_accessor(); - auto in1 = input1.read_accessor(); - - logger.debug() << "Equal task in0 [" << input0_shape.lo << "," << input0_shape.hi << "], in1 [" - << input1_shape.lo << "," << input1_shape.hi << "]"; - - using Reduce = Legion::ProdReduction; - legate::Store& output = context.reductions().at(0); - auto out = output.reduce_accessor(); - - if (input0_shape.lo != input1_shape.lo || input0_shape.hi != input1_shape.hi) { - logger.debug() << "Shape different"; - out.reduce(0, false); - return; - } - - for (size_t i = input0_shape.lo; i <= input0_shape.hi; ++i) { - if (in0[i] != in1[i]) { - logger.debug() << "Value different"; - out.reduce(0, false); - return; - } - } - out.reduce(0, true); -} - -/*static*/ void EqualTask2D::cpu_variant(legate::TaskContext& context) -{ - legate::Store& input0 = context.inputs().at(0); - legate::Store& input1 = context.inputs().at(1); - - legate::Rect<2> input0_shape = input0.shape<2>(); - legate::Rect<2> input1_shape = input1.shape<2>(); - - auto in0 = input0.read_accessor(); - auto in1 = input1.read_accessor(); - - logger.debug() << "Equal task 2D in0 [" << input0_shape.lo[0] << "," << input0_shape.lo[1] - << "] to [" << input0_shape.hi[0] << "," << input0_shape.hi[1] << "]"; - logger.debug() << "Equal task 2D in1 [" << input1_shape.lo[0] << "," << input1_shape.lo[1] - << "] to [" << input1_shape.hi[0] << "," << input1_shape.hi[1] << "]"; - - using Reduce = Legion::ProdReduction; - legate::Store& output = context.reductions().at(0); - auto out = output.reduce_accessor(); - - for (size_t i = input1_shape.lo[0]; i <= input1_shape.hi[0]; ++i) { - for (size_t j = input1_shape.lo[1]; j <= input1_shape.hi[1]; ++j) { - if (in0[i][j] != in1[i][j]) { - logger.debug() << "Value different"; - out.reduce(0, false); - return; - } - } - } - out.reduce(0, true); -} - } // namespace legateio } // namespace task diff --git a/examples/cpp/io/task_io.h b/examples/cpp/io/task_io.h index 9554384a1e..a3e16c62ce 100644 --- a/examples/cpp/io/task_io.h +++ b/examples/cpp/io/task_io.h @@ -31,10 +31,6 @@ enum LegateIOOpCode { WRITE_EVEN_TILES = 4, WRITE_FILE = 5, WRITE_UNEVEN_TILES = 6, - IOTA = 7, - IOTA_2D = 8, - EQUAL = 9, - EQUAL_2D = 10, }; static const char* library_name = "legateio"; @@ -72,26 +68,6 @@ struct WriteUnevenTilesTask : public legate::LegateTask { static void cpu_variant(legate::TaskContext& context); }; -struct IotaTask : public legate::LegateTask { - static const int32_t TASK_ID = IOTA; - static void cpu_variant(legate::TaskContext& context); -}; - -struct Iota2DTask : public legate::LegateTask { - static const int32_t TASK_ID = IOTA_2D; - static void cpu_variant(legate::TaskContext& context); -}; - -struct EqualTask : public legate::LegateTask { - static const int32_t TASK_ID = EQUAL; - static void cpu_variant(legate::TaskContext& context); -}; - -struct EqualTask2D : public legate::LegateTask { - static const int32_t TASK_ID = EQUAL_2D; - static void cpu_variant(legate::TaskContext& context); -}; - } // namespace legateio } // namespace task diff --git a/examples/cpp/main.cc b/examples/cpp/main.cc index 747c335690..22a8f331e1 100644 --- a/examples/cpp/main.cc +++ b/examples/cpp/main.cc @@ -15,18 +15,15 @@ */ #include +#include "cunumeric.h" #include "legate.h" class Environment : public ::testing::Environment { public: Environment(int argc, char** argv) : argc_(argc), argv_(argv) {} - void SetUp() override - { - legate::initialize(argc_, argv_); - EXPECT_EQ(legate::start(argc_, argv_), 0); - } - void TearDown() override { EXPECT_EQ(legate::wait_for_shutdown(), 0); } + void SetUp() override { EXPECT_EQ(legate::start(argc_, argv_), 0); } + void TearDown() override { EXPECT_EQ(legate::finish(), 0); } private: int argc_; diff --git a/src/core/partitioning/partition.h b/src/core/partitioning/partition.h index 90cfdc1f59..6ce5e1e72c 100644 --- a/src/core/partitioning/partition.h +++ b/src/core/partitioning/partition.h @@ -64,6 +64,9 @@ struct Partition { public: virtual std::string to_string() const = 0; + + public: + virtual const Shape& color_shape() const = 0; }; class NoPartition : public Partition { @@ -90,6 +93,13 @@ class NoPartition : public Partition { public: std::string to_string() const override; + + public: + const Shape& color_shape() const override + { + assert(false); + throw std::invalid_argument("NoPartition doesn't support color_shape"); + } }; class Tiling : public Partition { @@ -126,7 +136,7 @@ class Tiling : public Partition { public: const Shape& tile_shape() const { return tile_shape_; } - const Shape& color_shape() const { return color_shape_; } + const Shape& color_shape() const override { return color_shape_; } const tuple& offsets() const { return offsets_; } public: @@ -171,6 +181,9 @@ class Weighted : public Partition { public: std::string to_string() const override; + public: + const Shape& color_shape() const override { return color_shape_; } + private: Legion::FutureMap weights_; Domain color_domain_; From 88c85d0babe21afe23c64ed3b371d52294314798 Mon Sep 17 00:00:00 2001 From: Parag Kulkarni Date: Wed, 7 Jun 2023 02:10:59 -0700 Subject: [PATCH 0141/1425] Add CI and relevant build scripts (continued) * - Modify files: - build-all: Add Test build script. - tests/cpp/CMakeLists.txt: Add CPack related changes. - Add files: - .gitlab-ci.yml (CI) - build-cpp-test (Build the CPP test) See merge request legate/legate.core.internal!50 --- .gitlab-ci.yml | 63 +++++++++++++++++++ .../home/coder/.local/bin/build-all | 1 + .../home/coder/.local/bin/build-cpp-test | 23 +++++++ tests/cpp/CMakeLists.txt | 6 +- 4 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 .gitlab-ci.yml create mode 100755 continuous_integration/home/coder/.local/bin/build-cpp-test diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000..e1c62e0396 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,63 @@ +image: + name: "rapidsai/devcontainers:23.06-cpp-cuda11.8-mambaforge-ubuntu22.04" + +stages: + - build + +Build:LegateCore: + stage: build + + tags: + - cpu_only + - gpu/disabled + + variables: + # Set the following two vars to `nthreads - 1` + JOBS: 1 + CMAKE_BUILD_PARALLEL_LEVEL: 1 + PYTHONDONTWRITEBYTECODE: 1 + DEFAULT_CONDA_ENV: legate + # Add these if we want to speed up builds with sccache + # SCCACHE_REGION: us-east-2 + # SCCACHE_BUCKET: rapids-sccache-devs + # SCCACHE_S3_KEY_PREFIX: legate-cunumeric-dev + # VAULT_HOST: ${{ secrets.PERSONAL_ACCESS_TOKEN && 'https://vault.ops.k8s.rapids.ai' || '' }} + # GH_TOKEN: "${{ secrets.PERSONAL_ACCESS_TOKEN }}" + # GITHUB_TOKEN: "${{ secrets.PERSONAL_ACCESS_TOKEN }}" + + before_script: + - cp -R $CI_PROJECT_DIR ~coder/legate + - mkdir -p ~coder/.local + - cp -R $CI_PROJECT_DIR/continuous_integration/home/coder/.local/bin ~coder/.local/. + - cp -ar $CI_PROJECT_DIR/continuous_integration/home/coder/.gitconfig /home/coder/; + - chown -R coder:coder /home/coder/ + - mkdir -p /tmp/out; + - chown -R coder:coder /tmp/out + + script: + # Create conda env + - su coder -c 'cd ~/; exec entrypoint make-conda-env' + + # Build legate.core C++ library + - su coder -c 'cd ~/; exec entrypoint build-legate-cpp' + + # Build legate.core Python Wheel + - su coder -c 'cd ~/; exec entrypoint build-legate-wheel' + + # Build legate.core Conda Package + - su coder -c 'cd ~/; exec entrypoint build-legate-conda' + + # Build legate.core C++ test + - su coder -c 'cd ~/; exec entrypoint build-cpp-test' + + after_script: + # Copy the artifacts + - mkdir -p $CI_PROJECT_DIR/artifacts + - cp /tmp/out/* $CI_PROJECT_DIR/artifacts/ + + artifacts: + name: "$CI_COMMIT_REF_NAME" + paths: + - artifacts + exclude: + - artifacts/env*.yaml diff --git a/continuous_integration/home/coder/.local/bin/build-all b/continuous_integration/home/coder/.local/bin/build-all index 2a91bcb298..9f1288c84c 100755 --- a/continuous_integration/home/coder/.local/bin/build-all +++ b/continuous_integration/home/coder/.local/bin/build-all @@ -12,6 +12,7 @@ build_all() { build-legate-cpp; build-legate-wheel; build-legate-conda; + build-cpp-test; } (build_all "$@"); diff --git a/continuous_integration/home/coder/.local/bin/build-cpp-test b/continuous_integration/home/coder/.local/bin/build-cpp-test new file mode 100755 index 0000000000..ba0048093c --- /dev/null +++ b/continuous_integration/home/coder/.local/bin/build-cpp-test @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +build_cpp_tests() { + set -ex; + + conda deactivate + + cd ~/legate/tests/cpp + + rm -rf build + cmake -B build -S . -D legate_core_ROOT=../../build -D CMAKE_BUILD_TYPE=Debug + cmake --build build -v -j 4 + + ( + cd ~/legate/tests/cpp/build; + cpack -G TGZ; + cp cpp_tests*.tar.gz /tmp/out/; + ); + + { set +x; } 2>/dev/null; +} + +build_cpp_tests "$@"; diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt index 4da5a893db..4bc45b35e3 100644 --- a/tests/cpp/CMakeLists.txt +++ b/tests/cpp/CMakeLists.txt @@ -66,6 +66,10 @@ target_link_libraries(cpp_tests PRIVATE NCCL::NCCL) endif() include(GoogleTest) -gtest_discover_tests(cpp_tests) +gtest_discover_tests(cpp_tests DISCOVERY_MODE PRE_TEST WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + +include(CPack) +include(GNUInstallDirs) install(TARGETS cpp_tests DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/cmake-install") +install( TARGETS cpp_tests DESTINATION ${CMAKE_INSTALL_BINDIR}) From 21a9b23c82bcf7c92b997f7eec35a8349dc1f0ed Mon Sep 17 00:00:00 2001 From: Manolis Papadakis Date: Wed, 7 Jun 2023 13:30:26 -0700 Subject: [PATCH 0142/1425] Fix a typo --- src/core/data/detail/logical_store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/data/detail/logical_store.cc b/src/core/data/detail/logical_store.cc index 0bec15cd2f..dc3fabe74a 100644 --- a/src/core/data/detail/logical_store.cc +++ b/src/core/data/detail/logical_store.cc @@ -612,7 +612,7 @@ bool LogicalStore::has_key_partition(const mapping::MachineDesc& machine, key_partition_->satisfies_restrictions(restrictions)) return true; else - return transform_->is_convertible() & + return transform_->is_convertible() && storage_->find_key_partition(machine, transform_->invert(restrictions)) != nullptr; } From b21ed19561287f920ec737cf6829a86e98e23fd0 Mon Sep 17 00:00:00 2001 From: Irina Demeshko Date: Thu, 8 Jun 2023 20:00:13 -0700 Subject: [PATCH 0143/1425] Non-task operations * Massive rewrite for the copy tests * Adds shortcuts to the runtime API * Special case for a vector of scalars to avoid default ctor * Make sure operations are serialized when their outputs are unpartitioned * Add a mode to MachineDesc::slice that keeps the unsliced ranges * Remove Scalar's default ctor and add two useful ctors * Built-in point_type and rect_type * Make sure reset_key_partition resets partition for both storage and store * Make sure cunumeric is initialized * Catch up API changes in examples and tests * No requirement analyzers in copy launcher and clean up requirement analyzer * Minor clean-up for constraint generation in copy * Checks to prevent partitions from being reused for different stores and minor test clean-up * Minor clean-up for strategy, task, and copy * Clean up fill tests * Minor clean-up for fill and fill launcher * Remove obsolete functions * Unify Projection and ProjectionInfo and fix the deallocation issue * Split out Projection * Fix a bug in create_region_requirement * Reorganize operation and launcher classes * addressing comments + removing redundant test * addressing comments * replacing create_fill with issue_fill * adding scatter-gather test * removing uneccessary checks * improving copy test * fixing constraints logic * addressing comments for tests * addressing more of the PR comments * addressing some of the PR comments * addressing some of the PR comments * addressing some of the PR comments * fixing errors after the merge * adding another test * adding logic for partitioning constraints * formatting * added logic for scatter copies * removing CopyAnalyzer * renaming retun_region_req* fubctions and templating them * adding test for Fill * implementing Fill * code clean-up * removing debug output + adding more tests * making copy to work for a simple case * creating CopyAnalyzer * towards implementing Copy * towards implementing Copy operation Co-authored-by: Wonchan Lee See merge request legate/legate.core.internal!36 --- examples/cpp/hello/hello.cc | 1 - examples/cpp/hello/hello_print.cc | 1 - examples/cpp/io/io.cc | 1 - examples/cpp/main.cc | 6 +- legate_core_cpp.cmake | 9 +- src/core/comm/comm_cpu.cc | 2 +- src/core/comm/comm_nccl.cu | 2 +- src/core/data/detail/logical_store.cc | 26 +- src/core/data/detail/logical_store.h | 11 +- src/core/data/scalar.h | 17 +- src/core/data/scalar.inl | 16 ++ src/core/mapping/machine.cc | 16 +- src/core/mapping/machine.h | 6 +- src/core/partitioning/constraint_solver.cc | 10 +- src/core/partitioning/constraint_solver.h | 5 +- src/core/partitioning/partition.cc | 1 - src/core/partitioning/partitioner.cc | 27 +- src/core/partitioning/partitioner.h | 6 +- src/core/runtime/copy.cc | 212 +++++++++++++++ src/core/runtime/detail/copy_launcher.cc | 256 ++++++++++++++++++ src/core/runtime/detail/copy_launcher.h | 98 +++++++ src/core/runtime/detail/fill.cc | 67 +++++ src/core/runtime/detail/fill.h | 47 ++++ src/core/runtime/detail/fill_launcher.cc | 110 ++++++++ src/core/runtime/detail/fill_launcher.h | 67 +++++ src/core/runtime/detail/launcher_arg.cc | 7 +- src/core/runtime/detail/launcher_arg.h | 24 +- src/core/runtime/detail/projection.cc | 92 +++++++ src/core/runtime/detail/projection.h | 52 ++++ src/core/runtime/detail/req_analyzer.cc | 163 +++-------- src/core/runtime/detail/req_analyzer.h | 68 +---- src/core/runtime/detail/runtime.cc | 49 +++- src/core/runtime/detail/runtime.h | 16 +- .../detail/{launcher.cc => task_launcher.cc} | 43 +-- .../detail/{launcher.h => task_launcher.h} | 24 +- src/core/runtime/operation.h | 60 +++- src/core/runtime/runtime.cc | 17 ++ src/core/runtime/runtime.h | 40 ++- src/core/runtime/{operation.cc => task.cc} | 56 ++-- src/core/type/type_info.cc | 21 ++ src/core/type/type_info.h | 16 ++ src/core/utilities/deserializer.h | 8 +- src/core/utilities/deserializer.inl | 7 +- src/core/utilities/tuple.h | 1 + src/core/utilities/tuple.inl | 6 + tests/cpp/integration/copy_failure.cc | 112 ++++++++ tests/cpp/integration/copy_gather.cc | 188 +++++++++++++ tests/cpp/integration/copy_gather_scatter.cc | 228 ++++++++++++++++ tests/cpp/integration/copy_normal.cc | 163 +++++++++++ tests/cpp/integration/copy_scatter.cc | 214 +++++++++++++++ tests/cpp/integration/copy_util.inl | 179 ++++++++++++ tests/cpp/integration/fill.cc | 194 +++++++++++++ tests/cpp/integration/machine_scope.cc | 1 - tests/cpp/integration/manual_simple.cc | 1 - tests/cpp/integration/multi_scalar_out.cc | 1 - 55 files changed, 2718 insertions(+), 353 deletions(-) create mode 100644 src/core/runtime/copy.cc create mode 100644 src/core/runtime/detail/copy_launcher.cc create mode 100644 src/core/runtime/detail/copy_launcher.h create mode 100644 src/core/runtime/detail/fill.cc create mode 100644 src/core/runtime/detail/fill.h create mode 100644 src/core/runtime/detail/fill_launcher.cc create mode 100644 src/core/runtime/detail/fill_launcher.h create mode 100644 src/core/runtime/detail/projection.cc create mode 100644 src/core/runtime/detail/projection.h rename src/core/runtime/detail/{launcher.cc => task_launcher.cc} (88%) rename src/core/runtime/detail/{launcher.h => task_launcher.h} (82%) rename src/core/runtime/{operation.cc => task.cc} (89%) create mode 100644 tests/cpp/integration/copy_failure.cc create mode 100644 tests/cpp/integration/copy_gather.cc create mode 100644 tests/cpp/integration/copy_gather_scatter.cc create mode 100644 tests/cpp/integration/copy_normal.cc create mode 100644 tests/cpp/integration/copy_scatter.cc create mode 100644 tests/cpp/integration/copy_util.inl create mode 100644 tests/cpp/integration/fill.cc diff --git a/examples/cpp/hello/hello.cc b/examples/cpp/hello/hello.cc index 7b32c41a70..9bd0434422 100644 --- a/examples/cpp/hello/hello.cc +++ b/examples/cpp/hello/hello.cc @@ -16,7 +16,6 @@ #include -#include "core/mapping/mapping.h" #include "legate.h" #include "task_hello.h" diff --git a/examples/cpp/hello/hello_print.cc b/examples/cpp/hello/hello_print.cc index 1a7db58098..1c47d3e8fb 100644 --- a/examples/cpp/hello/hello_print.cc +++ b/examples/cpp/hello/hello_print.cc @@ -16,7 +16,6 @@ #include -#include "core/mapping/mapping.h" #include "legate.h" #include "task_hello.h" diff --git a/examples/cpp/io/io.cc b/examples/cpp/io/io.cc index 89379e349d..4aed904d1d 100644 --- a/examples/cpp/io/io.cc +++ b/examples/cpp/io/io.cc @@ -17,7 +17,6 @@ #include #include -#include "core/mapping/mapping.h" #include "cunumeric.h" #include "legate.h" #include "task_io.h" diff --git a/examples/cpp/main.cc b/examples/cpp/main.cc index 22a8f331e1..34dcccd138 100644 --- a/examples/cpp/main.cc +++ b/examples/cpp/main.cc @@ -22,7 +22,11 @@ class Environment : public ::testing::Environment { public: Environment(int argc, char** argv) : argc_(argc), argv_(argv) {} - void SetUp() override { EXPECT_EQ(legate::start(argc_, argv_), 0); } + void SetUp() override + { + EXPECT_EQ(legate::start(argc_, argv_), 0); + cunumeric::initialize(argc_, argv_); + } void TearDown() override { EXPECT_EQ(legate::finish(), 0); } private: diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index 9d522fc1ad..9987bded70 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -214,21 +214,26 @@ list(APPEND legate_core_SOURCES src/core/partitioning/partitioner.cc src/core/partitioning/restriction.cc src/core/runtime/context.cc - src/core/runtime/operation.cc + src/core/runtime/copy.cc src/core/runtime/projection.cc src/core/runtime/runtime.cc + src/core/runtime/task.cc src/core/runtime/tracker.cc src/core/runtime/shard.cc src/core/runtime/detail/communicator_manager.cc + src/core/runtime/detail/copy_launcher.cc src/core/runtime/detail/field_manager.cc + src/core/runtime/detail/fill.cc + src/core/runtime/detail/fill_launcher.cc src/core/runtime/detail/launcher_arg.cc - src/core/runtime/detail/launcher.cc src/core/runtime/detail/machine_manager.cc src/core/runtime/detail/partition_manager.cc + src/core/runtime/detail/projection.cc src/core/runtime/detail/provenance_manager.cc src/core/runtime/detail/region_manager.cc src/core/runtime/detail/req_analyzer.cc src/core/runtime/detail/runtime.cc + src/core/runtime/detail/task_launcher.cc src/core/task/registrar.cc src/core/task/return.cc src/core/task/task.cc diff --git a/src/core/comm/comm_cpu.cc b/src/core/comm/comm_cpu.cc index 04bf889226..2cdfdef96f 100644 --- a/src/core/comm/comm_cpu.cc +++ b/src/core/comm/comm_cpu.cc @@ -16,8 +16,8 @@ #include "core/comm/comm_cpu.h" #include "core/runtime/detail/communicator_manager.h" -#include "core/runtime/detail/launcher.h" #include "core/runtime/detail/runtime.h" +#include "core/runtime/detail/task_launcher.h" #include "legate.h" #include "core/comm/coll.h" diff --git a/src/core/comm/comm_nccl.cu b/src/core/comm/comm_nccl.cu index ecafabe496..3da787b16d 100644 --- a/src/core/comm/comm_nccl.cu +++ b/src/core/comm/comm_nccl.cu @@ -19,8 +19,8 @@ #include "core/cuda/stream_pool.h" #include "core/data/buffer.h" #include "core/runtime/detail/communicator_manager.h" -#include "core/runtime/detail/launcher.h" #include "core/runtime/detail/runtime.h" +#include "core/runtime/detail/task_launcher.h" #include "core/utilities/nvtx_help.h" #include "core/utilities/typedefs.h" #include "legate.h" diff --git a/src/core/data/detail/logical_store.cc b/src/core/data/detail/logical_store.cc index dc3fabe74a..9e0f8bebe5 100644 --- a/src/core/data/detail/logical_store.cc +++ b/src/core/data/detail/logical_store.cc @@ -18,6 +18,7 @@ #include "core/mapping/machine.h" #include "core/runtime/detail/partition_manager.h" +#include "core/runtime/detail/projection.h" #include "core/runtime/detail/runtime.h" #include "core/type/type_traits.h" #include "core/utilities/buffer_builder.h" @@ -624,7 +625,13 @@ void LogicalStore::set_key_partition(const mapping::MachineDesc& machine, storage_->set_key_partition(machine, std::move(inverted)); } -void LogicalStore::reset_key_partition() { storage_->reset_key_partition(); } +void LogicalStore::reset_key_partition() +{ + // Need to flush scheduling window to make this effective + Runtime::get_runtime()->flush_scheduling_window(); + key_partition_ = nullptr; + storage_->reset_key_partition(); +} std::shared_ptr LogicalStore::create_partition( std::shared_ptr partition, std::optional complete) @@ -632,7 +639,8 @@ std::shared_ptr LogicalStore::create_partition( if (unbound()) { throw std::invalid_argument("Unbound store cannot be manually partitioned"); } auto storage_partition = storage_->create_partition(transform_->invert(partition.get()), complete); - return std::make_shared(std::move(storage_partition), shared_from_this()); + return std::make_shared( + std::move(partition), std::move(storage_partition), shared_from_this()); } void LogicalStore::pack(BufferBuilder& buffer) const @@ -662,22 +670,26 @@ std::string LogicalStore::to_string() const //////////////////////////////////////////////////// // legate::detail::LogicalStorePartition //////////////////////////////////////////////////// -LogicalStorePartition::LogicalStorePartition(std::shared_ptr storage_partition, +LogicalStorePartition::LogicalStorePartition(std::shared_ptr partition, + std::shared_ptr storage_partition, std::shared_ptr store) - : storage_partition_(std::move(storage_partition)), store_(std::move(store)) + : partition_(std::move(partition)), + storage_partition_(std::move(storage_partition)), + store_(std::move(store)) { } -std::unique_ptr LogicalStorePartition::create_projection(const Domain* launch_domain) +std::unique_ptr LogicalStorePartition::create_projection_info( + const Domain* launch_domain) { if (nullptr == launch_domain || store_->has_scalar_storage()) - return std::make_unique(); + return std::make_unique(); // We're about to create a legion partition for this store, so the store should have its region // created. auto legion_partition = storage_partition_->get_legion_partition(); auto proj_id = store_->compute_projection(launch_domain->dim); - return std::make_unique(legion_partition, proj_id); + return std::make_unique(legion_partition, proj_id); } bool LogicalStorePartition::is_disjoint_for(const Domain* launch_domain) const diff --git a/src/core/data/detail/logical_store.h b/src/core/data/detail/logical_store.h index 9ad915a6fe..7758584b71 100644 --- a/src/core/data/detail/logical_store.h +++ b/src/core/data/detail/logical_store.h @@ -23,7 +23,6 @@ #include "core/data/store.h" #include "core/partitioning/partition.h" #include "core/partitioning/restriction.h" -#include "core/runtime/detail/req_analyzer.h" namespace legate::mapping { class MachineDesc; @@ -31,7 +30,7 @@ class MachineDesc; namespace legate::detail { -class Projection; +class ProjectionInfo; class StoragePartition; class LogicalStorePartition; @@ -206,7 +205,6 @@ class LogicalStore : public std::enable_shared_from_this { public: Restrictions compute_restrictions() const; - std::unique_ptr create_projection(const Partition* partition, int32_t launch_ndim); std::shared_ptr find_or_create_key_partition(const mapping::MachineDesc& machine, const Restrictions& restrictions); bool has_key_partition(const mapping::MachineDesc& machine, @@ -239,16 +237,19 @@ class LogicalStore : public std::enable_shared_from_this { class LogicalStorePartition : public std::enable_shared_from_this { public: - LogicalStorePartition(std::shared_ptr storage_partition, + LogicalStorePartition(std::shared_ptr partition, + std::shared_ptr storage_partition, std::shared_ptr store); public: + std::shared_ptr partition() const { return partition_; } std::shared_ptr storage_partition() const { return storage_partition_; } std::shared_ptr store() const { return store_; } - std::unique_ptr create_projection(const Domain* launch_domain); + std::unique_ptr create_projection_info(const Domain* launch_domain); bool is_disjoint_for(const Domain* launch_domain) const; private: + std::shared_ptr partition_; std::shared_ptr storage_partition_; std::shared_ptr store_; }; diff --git a/src/core/data/scalar.h b/src/core/data/scalar.h index 718ffd9296..538663fd42 100644 --- a/src/core/data/scalar.h +++ b/src/core/data/scalar.h @@ -40,7 +40,6 @@ class BufferBuilder; */ class Scalar { public: - Scalar() = default; Scalar(const Scalar& other); Scalar(Scalar&& other); @@ -92,6 +91,22 @@ class Scalar { template Scalar(const std::vector& values); + public: + /** + * @brief Creates a point scalar + * + * @param point A point from which the scalar should be constructed + */ + template + Scalar(const Point& point); + /** + * @brief Creates a rect scalar + * + * @param point A rect from which the scalar should be constructed + */ + template + Scalar(const Rect& rect); + public: Scalar& operator=(const Scalar& other); diff --git a/src/core/data/scalar.inl b/src/core/data/scalar.inl index e4debab7dd..a2c13364cb 100644 --- a/src/core/data/scalar.inl +++ b/src/core/data/scalar.inl @@ -55,6 +55,22 @@ Scalar::Scalar(const std::vector& values) data_ = buffer; } +template +Scalar::Scalar(const Point& point) : own_(true), type_(point_type(DIM)) +{ + auto buffer = malloc(sizeof(Point)); + memcpy(buffer, &point, sizeof(Point)); + data_ = buffer; +} + +template +Scalar::Scalar(const Rect& rect) : own_(true), type_(rect_type(DIM)) +{ + auto buffer = malloc(sizeof(Rect)); + memcpy(buffer, &rect, sizeof(Rect)); + data_ = buffer; +} + template VAL Scalar::value() const { diff --git a/src/core/mapping/machine.cc b/src/core/mapping/machine.cc index 23bc4b154f..85e866a7fc 100644 --- a/src/core/mapping/machine.cc +++ b/src/core/mapping/machine.cc @@ -232,14 +232,22 @@ MachineDesc MachineDesc::only(const std::vector& targets) const return MachineDesc(new_processor_ranges); } -MachineDesc MachineDesc::slice(uint32_t from, uint32_t to, TaskTarget target) const +MachineDesc MachineDesc::slice(uint32_t from, + uint32_t to, + TaskTarget target, + bool keep_others) const { - return MachineDesc({{target, processor_range(target).slice(from, to)}}); + if (keep_others) { + std::map new_ranges(processor_ranges); + new_ranges[target] = processor_range(target).slice(from, to); + return MachineDesc(std::move(new_ranges)); + } else + return MachineDesc({{target, processor_range(target).slice(from, to)}}); } -MachineDesc MachineDesc::slice(uint32_t from, uint32_t to) const +MachineDesc MachineDesc::slice(uint32_t from, uint32_t to, bool keep_others) const { - return slice(from, to, preferred_target); + return slice(from, to, preferred_target, keep_others); } MachineDesc MachineDesc::operator[](TaskTarget target) const { return only(target); } diff --git a/src/core/mapping/machine.h b/src/core/mapping/machine.h index d3465e2ae9..233890c6fc 100644 --- a/src/core/mapping/machine.h +++ b/src/core/mapping/machine.h @@ -234,19 +234,21 @@ struct MachineDesc { * @param from Starting index * @param to End index * @param targets Processor type to slice + * @param keep_others Optional flag to keep unsliced ranges in the returned machine descriptor * * @return Machine descriptor with the chosen procssor range sliced */ - MachineDesc slice(uint32_t from, uint32_t to, TaskTarget target) const; + MachineDesc slice(uint32_t from, uint32_t to, TaskTarget target, bool keep_others = false) const; /** * @brief Slices the processor range for the preferred processor type of this machine descriptor * * @param from Starting index * @param to End index + * @param keep_others Optional flag to keep unsliced ranges in the returned machine descriptor * * @return Machine descriptor with the preferred processor range sliced */ - MachineDesc slice(uint32_t from, uint32_t to) const; + MachineDesc slice(uint32_t from, uint32_t to, bool keep_others = false) const; /** * @brief Selects the processor range for a given processor type and constructs a machine diff --git a/src/core/partitioning/constraint_solver.cc b/src/core/partitioning/constraint_solver.cc index cce76d0efe..96f4d91db9 100644 --- a/src/core/partitioning/constraint_solver.cc +++ b/src/core/partitioning/constraint_solver.cc @@ -80,9 +80,10 @@ ConstraintSolver::ConstraintSolver() {} ConstraintSolver::~ConstraintSolver() {} -void ConstraintSolver::add_partition_symbol(const Variable* partition_symbol) +void ConstraintSolver::add_partition_symbol(const Variable* partition_symbol, bool is_output) { partition_symbols_.insert(partition_symbol); + is_output_.insert({*partition_symbol, is_output}); } void ConstraintSolver::add_constraint(const Constraint* constraint) @@ -188,6 +189,11 @@ const Restrictions& ConstraintSolver::find_restrictions(const Variable* partitio return equiv_class_map_.at(*partition_symbol)->restrictions; } +bool ConstraintSolver::is_output(const Variable& partition_symbol) const +{ + return is_output_.at(partition_symbol); +} + void ConstraintSolver::dump() { log_legate.debug("===== Constraint Graph ====="); @@ -204,6 +210,4 @@ const std::vector& ConstraintSolver::partition_symbols() const return partition_symbols_.elements(); } -const std::vector& ConstraintSolver::constraints() const { return constraints_; } - } // namespace legate::detail diff --git a/src/core/partitioning/constraint_solver.h b/src/core/partitioning/constraint_solver.h index 822b125a69..c1dcc29696 100644 --- a/src/core/partitioning/constraint_solver.h +++ b/src/core/partitioning/constraint_solver.h @@ -32,7 +32,7 @@ struct ConstraintSolver { ~ConstraintSolver(); public: - void add_partition_symbol(const Variable* partition_symbol); + void add_partition_symbol(const Variable* partition_symbol, bool is_output = false); void add_constraint(const Constraint* constraint); public: @@ -40,16 +40,17 @@ struct ConstraintSolver { public: const std::vector& partition_symbols() const; - const std::vector& constraints() const; public: void solve_constraints(); const std::vector& find_equivalence_class( const Variable* partition_symbol) const; const Restrictions& find_restrictions(const Variable* partition_symbol) const; + bool is_output(const Variable& partition_symbol) const; private: ordered_set partition_symbols_{}; + std::map is_output_{}; std::vector constraints_{}; private: diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index 531ea35137..77e9e8b7bb 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -19,7 +19,6 @@ #include "core/data/detail/logical_store.h" #include "core/partitioning/partition.h" #include "core/runtime/detail/partition_manager.h" -#include "core/runtime/detail/req_analyzer.h" #include "core/runtime/detail/runtime.h" namespace legate { diff --git a/src/core/partitioning/partitioner.cc b/src/core/partitioning/partitioner.cc index 7b7bd59c53..e0d6912eb8 100644 --- a/src/core/partitioning/partitioner.cc +++ b/src/core/partitioning/partitioner.cc @@ -21,7 +21,6 @@ #include "core/partitioning/constraint.h" #include "core/partitioning/constraint_solver.h" #include "core/partitioning/partition.h" -#include "core/runtime/detail/launcher.h" #include "core/runtime/detail/runtime.h" #include "core/runtime/operation.h" @@ -38,6 +37,7 @@ class LaunchDomainResolver { public: void record_launch_domain(const Domain& launch_domain); void record_unbound_store(int32_t unbound_dim); + void set_must_be_sequential(bool must_be_sequential) { must_be_sequential_ = must_be_sequential; } public: std::unique_ptr resolve_launch_domain() const; @@ -142,7 +142,7 @@ bool Strategy::has_assignment(const Variable* partition_symbol) const return assignments_.find(*partition_symbol) != assignments_.end(); } -const std::shared_ptr& Strategy::operator[](const Variable* partition_symbol) const +std::shared_ptr Strategy::operator[](const Variable* partition_symbol) const { auto finder = assignments_.find(*partition_symbol); #ifdef DEBUG_LEGATE @@ -176,27 +176,28 @@ void Strategy::dump() const log_legate.debug("===================="); } -void Strategy::compute_launch_domains() +void Strategy::compute_launch_domains(const ConstraintSolver& solver) { std::map domain_resolvers; - for (auto& assignment : assignments_) { - auto& part_symb = assignment.first; - auto& partition = assignment.second; - auto* op = part_symb.operation(); + for (auto& [part_symb, partition] : assignments_) { + auto* op = part_symb.operation(); + auto& domain_resolver = domain_resolvers[op]; if (partition->has_launch_domain()) { - domain_resolvers[op].record_launch_domain(partition->launch_domain()); + domain_resolver.record_launch_domain(partition->launch_domain()); continue; } auto store = op->find_store(&part_symb); - - if (store->unbound()) domain_resolvers[op].record_unbound_store(store->dim()); + if (store->unbound()) + domain_resolver.record_unbound_store(store->dim()); + else if (solver.is_output(part_symb)) + domain_resolver.set_must_be_sequential(true); } - for (auto& pair : domain_resolvers) - launch_domains_[pair.first] = pair.second.resolve_launch_domain(); + for (auto& [op, domain_resolver] : domain_resolvers) + launch_domains_[op] = domain_resolver.resolve_launch_domain(); } //////////////////////////////////////////////////// @@ -259,7 +260,7 @@ std::unique_ptr Partitioner::partition_stores() for (auto symb : equiv_class) strategy->insert(symb, partition); } - strategy->compute_launch_domains(); + strategy->compute_launch_domains(solver); #ifdef DEBUG_LEGATE strategy->dump(); diff --git a/src/core/partitioning/partitioner.h b/src/core/partitioning/partitioner.h index 5d70a1bd6d..387076738d 100644 --- a/src/core/partitioning/partitioner.h +++ b/src/core/partitioning/partitioner.h @@ -34,7 +34,7 @@ namespace legate::detail { class ConstraintSolver; class LogicalStore; class Partitioner; -class Projection; +class ProjectionInfo; class Strategy { friend class Partitioner; @@ -53,14 +53,14 @@ class Strategy { std::shared_ptr partition, Legion::FieldSpace field_space); bool has_assignment(const Variable* partition_symbol) const; - const std::shared_ptr& operator[](const Variable* partition_symbol) const; + std::shared_ptr operator[](const Variable* partition_symbol) const; const Legion::FieldSpace& find_field_space(const Variable* partition_symbol) const; public: void dump() const; private: - void compute_launch_domains(); + void compute_launch_domains(const ConstraintSolver& solver); private: std::map> assignments_{}; diff --git a/src/core/runtime/copy.cc b/src/core/runtime/copy.cc new file mode 100644 index 0000000000..6619f04555 --- /dev/null +++ b/src/core/runtime/copy.cc @@ -0,0 +1,212 @@ +/* Copyright 2021 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/runtime/operation.h" + +#include "core/data/detail/logical_store.h" +#include "core/data/logical_store.h" +#include "core/partitioning/constraint.h" +#include "core/partitioning/constraint_solver.h" +#include "core/partitioning/partition.h" +#include "core/partitioning/partitioner.h" +#include "core/runtime/context.h" +#include "core/runtime/detail/copy_launcher.h" +#include "core/runtime/detail/projection.h" + +namespace legate { + +Copy::Copy(LibraryContext* library, int64_t unique_id, mapping::MachineDesc&& machine) + : Operation(library, unique_id, std::move(machine)) +{ +} + +void Copy::add_store(std::vector& store_args, + LogicalStore& store, + const Variable* partition_symbol) +{ + auto store_impl = store.impl(); + store_args.push_back(StoreArg(store_impl.get(), partition_symbol)); + record_partition(partition_symbol, std::move(store_impl)); +} + +void Copy::add_store(std::optional& store_arg, + LogicalStore& store, + const Variable* partition_symbol) +{ + auto store_impl = store.impl(); + store_arg = StoreArg(store_impl.get(), partition_symbol); + record_partition(partition_symbol, std::move(store_impl)); +} + +void check_store(LogicalStore store) +{ + if (store.unbound() || store.impl()->has_scalar_storage() || store.transformed()) { + std::string msg = "Copy accepts only normal, not transformed, region-backed store"; + throw std::runtime_error(msg); + } +} + +void Copy::add_input(LogicalStore store) +{ + check_store(store); + add_store(inputs_, store, declare_partition()); +} + +void Copy::add_output(LogicalStore store) +{ + check_store(store); + if (reductions_.size() > 0) + throw std::runtime_error("Copy targets must be either all normal outputs or reductions"); + add_store(outputs_, store, declare_partition()); +} + +void Copy::add_reduction(LogicalStore store, Legion::ReductionOpID redop) +{ + check_store(store); + if (outputs_.size() > 0) + throw std::runtime_error("Copy targets must be either all normal outputs or reductions"); + add_store(reductions_, store, declare_partition()); + reduction_ops_.push_back(redop); +} + +void Copy::add_source_indirect(LogicalStore store) +{ + check_store(store); + add_store(source_indirect_, store, declare_partition()); +} + +void Copy::add_target_indirect(LogicalStore store) +{ + check_store(store); + add_store(target_indirect_, store, declare_partition()); +} + +void Copy::set_source_indirect_out_of_range(bool flag) { source_indirect_out_of_range_ = flag; } + +void Copy::set_target_indirect_out_of_range(bool flag) { target_indirect_out_of_range_ = flag; } + +void Copy::launch(detail::Strategy* p_strategy) +{ + auto& strategy = *p_strategy; + detail::CopyLauncher launcher( + library_, machine_, source_indirect_out_of_range_, target_indirect_out_of_range_); + auto launch_domain = strategy.launch_domain(this); + + auto create_projection_info = [&strategy, &launch_domain](auto& store, auto& var) { + auto store_partition = store->create_partition(strategy[var]); + return store_partition->create_projection_info(launch_domain); + }; + + for (auto& [store, var] : inputs_) launcher.add_input(store, create_projection_info(store, var)); + // FIXME: today a copy is a scatter copy only when a target indirection + // is given. In the future, we may pass store transforms directly to + // Legion and some transforms turn some copies into scatter copies even + // when no target indirection is provided. So, this scatter copy check + // will need to be extended accordingly. + bool scatter = target_indirect_.has_value(); + for (auto& [store, var] : outputs_) { + if (scatter) + launcher.add_inout(store, create_projection_info(store, var)); + else + launcher.add_output(store, create_projection_info(store, var)); + store->set_key_partition(machine(), strategy[var].get()); + } + uint32_t idx = 0; + for (auto& [store, var] : reductions_) { + auto store_partition = store->create_partition(strategy[var]); + auto proj = store_partition->create_projection_info(launch_domain); + bool read_write = store_partition->is_disjoint_for(launch_domain); + auto redop = reduction_ops_[idx++]; + proj->set_reduction_op(redop); + launcher.add_reduction(store, std::move(proj), read_write); + } + + if (source_indirect_.has_value()) { + auto& [store, var] = source_indirect_.value(); + launcher.add_source_indirect(store, create_projection_info(store, var)); + } + + if (target_indirect_.has_value()) { + auto& [store, var] = target_indirect_.value(); + launcher.add_target_indirect(store, create_projection_info(store, var)); + } + + if (launch_domain != nullptr) { + return launcher.execute(*launch_domain); + } else { + return launcher.execute_single(); + } +} + +void Copy::add_to_solver(detail::ConstraintSolver& solver) +{ + bool gather = source_indirect_.has_value(); + bool scatter = target_indirect_.has_value(); + + if (inputs_.size() != outputs_.size()) + throw std::runtime_error( + "Number of inputs and outputs should be the same in the Copy operation"); + + if (gather && inputs_.size() != 1) + throw std::runtime_error("when source indirect is specified, there could be only one input"); + + if (scatter && outputs_.size() != 1) + throw std::runtime_error("when target indirect is specified, there could be only one output"); + + // fill constraints + if (!gather && !scatter) { + for (size_t i = 0; i < inputs_.size(); i++) { + const auto& [in, in_part] = inputs_[i]; + const auto& [out, out_part] = outputs_[i]; + if (in->extents() != out->extents()) + throw std::runtime_error( + "Each output must have the same shape as the corresponding input in a copy"); + constraints_.push_back(align(in_part, out_part)); + } + } else if (gather && scatter) { + const auto& [src_indirect, src_indirect_part] = source_indirect_.value(); + const auto& [tgt_indirect, tgt_indirect_part] = target_indirect_.value(); + if (src_indirect->extents() != tgt_indirect->extents()) + throw std::runtime_error( + "Source and target indirect must have the same shape in a gather-scatter copy"); + constraints_.push_back(align(src_indirect_part, tgt_indirect_part)); + } else if (gather) { + const auto& [out, out_part] = reductions_.empty() ? outputs_.front() : reductions_.front(); + const auto& [indirect, indirect_part] = source_indirect_.value(); + if (out->extents() != indirect->extents()) + throw std::runtime_error( + "Source indirect must have the same shape as the output in a gather copy"); + constraints_.push_back(align(out_part, indirect_part)); + } else if (scatter) { + const auto& [in, in_part] = inputs_.front(); + const auto& [indirect, indirect_part] = target_indirect_.value(); + if (in->extents() != indirect->extents()) + throw std::runtime_error( + "Target indirect must have the same shape as the input in a scatter copy"); + constraints_.push_back(align(in_part, indirect_part)); + } + + for (auto& constraint : constraints_) solver.add_constraint(constraint.get()); + for (auto& [_, symb] : inputs_) solver.add_partition_symbol(symb); + for (auto& [_, symb] : outputs_) solver.add_partition_symbol(symb, true); + for (auto& [_, symb] : reductions_) solver.add_partition_symbol(symb); + if (source_indirect_.has_value()) solver.add_partition_symbol(source_indirect_.value().second); + if (target_indirect_.has_value()) solver.add_partition_symbol(target_indirect_.value().second); +} + +std::string Copy::to_string() const { return "Copy:" + std::to_string(unique_id_); } + +} // namespace legate diff --git a/src/core/runtime/detail/copy_launcher.cc b/src/core/runtime/detail/copy_launcher.cc new file mode 100644 index 0000000000..2354e48a75 --- /dev/null +++ b/src/core/runtime/detail/copy_launcher.cc @@ -0,0 +1,256 @@ +/* Copyright 2021 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/runtime/detail/copy_launcher.h" +#include "core/data/detail/logical_store.h" +#include "core/mapping/machine.h" +#include "core/runtime/context.h" +#include "core/runtime/detail/launcher_arg.h" +#include "core/runtime/detail/projection.h" +#include "core/runtime/detail/runtime.h" +#include "core/utilities/buffer_builder.h" + +namespace legate::detail { + +struct CopyArg : public ArgWrapper { + public: + CopyArg(uint32_t req_idx, + LogicalStore* store, + Legion::FieldID field_id, + Legion::PrivilegeMode privilege, + std::unique_ptr proj_info); + + public: + void pack(BufferBuilder& buffer) const override; + + public: + template + void populate_requirement(Legion::RegionRequirement& requirement) + { + proj_info_->template populate_requirement( + requirement, region_, {field_id_}, privilege_); + } + + public: + ~CopyArg() {} + + private: + uint32_t req_idx_; + LogicalStore* store_; + Legion::LogicalRegion region_; + Legion::FieldID field_id_; + Legion::PrivilegeMode privilege_; + std::unique_ptr proj_info_; +}; + +CopyArg::CopyArg(uint32_t req_idx, + LogicalStore* store, + Legion::FieldID field_id, + Legion::PrivilegeMode privilege, + std::unique_ptr proj_info) + : req_idx_(req_idx), + store_(store), + region_(store_->get_region_field()->region()), + field_id_(field_id), + privilege_(privilege), + proj_info_(std::move(proj_info)) +{ +} + +void CopyArg::pack(BufferBuilder& buffer) const +{ + store_->pack(buffer); + + buffer.pack(proj_info_->redop); + buffer.pack(region_.get_dim()); + buffer.pack(req_idx_); + buffer.pack(field_id_); +} + +CopyLauncher::CopyLauncher(LibraryContext* library, + const mapping::MachineDesc& machine, + bool source_indirect_out_of_range, + bool target_indirect_out_of_range, + int64_t tag) + : library_(library), + source_indirect_out_of_range_(source_indirect_out_of_range), + target_indirect_out_of_range_(target_indirect_out_of_range), + tag_(tag) +{ + mapper_arg_ = new BufferBuilder(); + machine.pack(*mapper_arg_); +} + +CopyLauncher::~CopyLauncher() +{ + delete mapper_arg_; + for (auto& arg : inputs_) delete arg; + for (auto& arg : outputs_) delete arg; + for (auto& arg : source_indirect_) delete arg; + for (auto& arg : target_indirect_) delete arg; +} + +int64_t CopyLauncher::legion_mapper_id() const { return library_->get_mapper_id(); } + +void CopyLauncher::add_store(std::vector& args, + detail::LogicalStore* store, + std::unique_ptr proj_info, + Legion::PrivilegeMode privilege) +{ + uint32_t req_idx = args.size(); + auto region_field = store->get_region_field(); + auto region = region_field->region(); + auto field_id = region_field->field_id(); + args.push_back(new CopyArg(req_idx, store, field_id, privilege, std::move(proj_info))); +} + +void CopyLauncher::add_input(detail::LogicalStore* store, std::unique_ptr proj_info) +{ + add_store(inputs_, store, std::move(proj_info), LEGION_READ_ONLY); +} + +void CopyLauncher::add_output(detail::LogicalStore* store, + std::unique_ptr proj_info) +{ + add_store(outputs_, store, std::move(proj_info), LEGION_WRITE_ONLY); +} + +void CopyLauncher::add_inout(detail::LogicalStore* store, std::unique_ptr proj_info) +{ + add_store(outputs_, store, std::move(proj_info), LEGION_READ_WRITE); +} + +void CopyLauncher::add_reduction(detail::LogicalStore* store, + std::unique_ptr proj_info, + bool read_write) +{ + if (read_write) + add_store(outputs_, store, std::move(proj_info), LEGION_READ_WRITE); + else + add_store(outputs_, store, std::move(proj_info), LEGION_REDUCE); +} +void CopyLauncher::add_source_indirect(detail::LogicalStore* store, + std::unique_ptr proj_info) +{ + add_store(source_indirect_, store, std::move(proj_info), LEGION_READ_ONLY); +} + +void CopyLauncher::add_target_indirect(detail::LogicalStore* store, + std::unique_ptr proj_info) +{ + add_store(target_indirect_, store, std::move(proj_info), LEGION_READ_ONLY); +} + +void CopyLauncher::execute(const Legion::Domain& launch_domain) +{ + auto legion_copy_launcher = build_index_copy(launch_domain); + return Runtime::get_runtime()->dispatch(legion_copy_launcher.get()); +} + +void CopyLauncher::execute_single() +{ + auto legion_copy_launcher = build_single_copy(); + return Runtime::get_runtime()->dispatch(legion_copy_launcher.get()); +} + +void CopyLauncher::pack_sharding_functor_id() +{ + // TODO: Generate the right sharding functor id + mapper_arg_->pack(0); +} + +void CopyLauncher::pack_args() +{ + pack_sharding_functor_id(); + + auto pack_args = [&](const std::vector& args) { + mapper_arg_->pack(args.size()); + for (auto& arg : args) arg->pack(*mapper_arg_); + }; + pack_args(inputs_); + pack_args(outputs_); + pack_args(source_indirect_); + pack_args(target_indirect_); +} + +namespace { + +template +constexpr bool is_single; +template <> +constexpr bool is_single = true; +template <> +constexpr bool is_single = false; + +} // namespace + +template +void CopyLauncher::populate_copy(Launcher* launcher) +{ + auto populate_requirements = [&](auto& args, auto& requirements) { + requirements.resize(args.size()); + for (uint32_t idx = 0; idx < args.size(); ++idx) { + auto& req = requirements[idx]; + auto& arg = args[idx]; + arg->template populate_requirement>(req); + } + }; + + populate_requirements(inputs_, launcher->src_requirements); + populate_requirements(outputs_, launcher->dst_requirements); + populate_requirements(source_indirect_, launcher->src_indirect_requirements); + populate_requirements(target_indirect_, launcher->dst_indirect_requirements); + + launcher->src_indirect_is_range.resize(source_indirect_.size(), false); + launcher->dst_indirect_is_range.resize(target_indirect_.size(), false); + + launcher->possible_src_indirect_out_of_range = source_indirect_out_of_range_; + launcher->possible_dst_indirect_out_of_range = target_indirect_out_of_range_; +} + +std::unique_ptr CopyLauncher::build_index_copy( + const Legion::Domain& launch_domain) +{ + pack_args(); + auto* runtime = Runtime::get_runtime(); + auto& provenance = runtime->provenance_manager()->get_provenance(); + auto index_copy = std::make_unique(launch_domain, + Legion::Predicate::TRUE_PRED, + legion_mapper_id(), + tag_, + mapper_arg_->to_legion_buffer(), + provenance.c_str()); + + populate_copy(index_copy.get()); + return std::move(index_copy); +} + +std::unique_ptr CopyLauncher::build_single_copy() +{ + pack_args(); + auto* runtime = Runtime::get_runtime(); + auto& provenance = runtime->provenance_manager()->get_provenance(); + auto single_copy = std::make_unique(Legion::Predicate::TRUE_PRED, + legion_mapper_id(), + tag_, + mapper_arg_->to_legion_buffer(), + provenance.c_str()); + + populate_copy(single_copy.get()); + return std::move(single_copy); +} + +} // namespace legate::detail diff --git a/src/core/runtime/detail/copy_launcher.h b/src/core/runtime/detail/copy_launcher.h new file mode 100644 index 0000000000..f5f1c57252 --- /dev/null +++ b/src/core/runtime/detail/copy_launcher.h @@ -0,0 +1,98 @@ +/* Copyright 2021 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include +#include + +#include "legion.h" + +namespace legate { +class BufferBuilder; +class LibraryContext; +class Scalar; +} // namespace legate + +namespace legate::mapping { +class MachineDesc; +} // namespace legate::mapping + +namespace legate::detail { + +class LogicalStore; +class ProjectionInfo; +class OutputRequirementAnalyzer; +class CopyArg; +class RequirementAnalyzer; + +class CopyLauncher { + public: + CopyLauncher(LibraryContext* library, + const mapping::MachineDesc& machine, + bool source_indirect_out_of_range, + bool target_indirect_out_of_range, + int64_t tag = 0); + ~CopyLauncher(); + + public: + int64_t legion_mapper_id() const; + + public: + void add_input(detail::LogicalStore* store, std::unique_ptr proj_info); + void add_output(detail::LogicalStore* store, std::unique_ptr proj_info); + void add_inout(detail::LogicalStore* store, std::unique_ptr proj_info); + void add_reduction(detail::LogicalStore* store, + std::unique_ptr proj_info, + bool read_write); + void add_source_indirect(detail::LogicalStore* store, std::unique_ptr proj_info); + void add_target_indirect(detail::LogicalStore* store, std::unique_ptr proj_info); + + private: + void add_store(std::vector& args, + detail::LogicalStore* store, + std::unique_ptr proj_info, + Legion::PrivilegeMode privilege); + + public: + void execute(const Legion::Domain& launch_domain); + void execute_single(); + + private: + void pack_args(); + void pack_sharding_functor_id(); + std::unique_ptr build_index_copy(const Legion::Domain& launch_domain); + std::unique_ptr build_single_copy(); + template + void populate_copy(Launcher* launcher); + + private: + LibraryContext* library_; + int64_t tag_; + + private: + BufferBuilder* mapper_arg_; + std::vector inputs_{}; + std::vector outputs_{}; + std::vector source_indirect_{}; + std::vector target_indirect_{}; + + private: + bool source_indirect_out_of_range_; + bool target_indirect_out_of_range_; +}; + +} // namespace legate::detail diff --git a/src/core/runtime/detail/fill.cc b/src/core/runtime/detail/fill.cc new file mode 100644 index 0000000000..48b7220734 --- /dev/null +++ b/src/core/runtime/detail/fill.cc @@ -0,0 +1,67 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/runtime/detail/fill.h" + +#include "core/data/detail/logical_store.h" +#include "core/partitioning/constraint_solver.h" +#include "core/partitioning/partitioner.h" +#include "core/runtime/context.h" +#include "core/runtime/detail/fill_launcher.h" +#include "core/runtime/detail/projection.h" + +namespace legate { + +Fill::Fill(LibraryContext* library, + LogicalStore lhs, + LogicalStore value, + int64_t unique_id, + mapping::MachineDesc&& machine) + : Operation(library, unique_id, std::move(machine)), + lhs_var_(declare_partition()), + lhs_(lhs.impl()), + value_(value.impl()) +{ + store_mappings_[*lhs_var_] = lhs_.get(); + if (lhs_->unbound() || lhs_->has_scalar_storage()) + throw std::runtime_error("Fill lhs must be a normal, region-backed store"); + + if (!value_->has_scalar_storage()) + throw std::runtime_error("Fill value should be a Future-back store"); +} + +void Fill::launch(detail::Strategy* strategy) +{ + detail::FillLauncher launcher(library_, machine_); + auto launch_domain = strategy->launch_domain(this); + auto part = (*strategy)[lhs_var_]; + auto lhs_proj = lhs_->create_partition(part)->create_projection_info(launch_domain); + lhs_->set_key_partition(machine(), part.get()); + + if (nullptr == launch_domain) + launcher.launch_single(lhs_.get(), *lhs_proj, value_.get()); + else + launcher.launch(*launch_domain, lhs_.get(), *lhs_proj, value_.get()); +} + +std::string Fill::to_string() const { return "Fill:" + std::to_string(unique_id_); } + +void Fill::add_to_solver(detail::ConstraintSolver& solver) +{ + solver.add_partition_symbol(lhs_var_); +} + +} // namespace legate diff --git a/src/core/runtime/detail/fill.h b/src/core/runtime/detail/fill.h new file mode 100644 index 0000000000..1a40db578a --- /dev/null +++ b/src/core/runtime/detail/fill.h @@ -0,0 +1,47 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include "core/runtime/operation.h" + +namespace legate { + +class Fill : public Operation { + private: + friend class detail::Runtime; + Fill(LibraryContext* library, + LogicalStore lhs, + LogicalStore value, + int64_t unique_id, + mapping::MachineDesc&& machine); + + public: + void launch(detail::Strategy* strategy) override; + + public: + std::string to_string() const override; + + public: + void add_to_solver(detail::ConstraintSolver& solver) override; + + private: + const Variable* lhs_var_; + std::shared_ptr lhs_; + std::shared_ptr value_; +}; + +} // namespace legate diff --git a/src/core/runtime/detail/fill_launcher.cc b/src/core/runtime/detail/fill_launcher.cc new file mode 100644 index 0000000000..83c9b79402 --- /dev/null +++ b/src/core/runtime/detail/fill_launcher.cc @@ -0,0 +1,110 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/runtime/detail/fill_launcher.h" +#include "core/data/detail/logical_store.h" +#include "core/mapping/machine.h" +#include "core/runtime/context.h" +#include "core/runtime/detail/projection.h" +#include "core/runtime/detail/runtime.h" +#include "core/utilities/buffer_builder.h" + +namespace legate::detail { + +FillLauncher::FillLauncher(LibraryContext* library, + const mapping::MachineDesc& machine, + int64_t tag) + : library_(library), tag_(tag) +{ + mapper_arg_ = new BufferBuilder(); + machine.pack(*mapper_arg_); +} + +FillLauncher::~FillLauncher() { delete mapper_arg_; } + +void FillLauncher::launch(const Legion::Domain& launch_domain, + LogicalStore* lhs, + const ProjectionInfo& lhs_proj, + LogicalStore* value) +{ + auto legion_fill_launcher = build_index_fill(launch_domain, lhs, lhs_proj, value); + return Runtime::get_runtime()->dispatch(legion_fill_launcher.get()); +} + +void FillLauncher::launch_single(LogicalStore* lhs, + const ProjectionInfo& lhs_proj, + LogicalStore* value) +{ + auto legion_fill_launcher = build_single_fill(lhs, lhs_proj, value); + return Runtime::get_runtime()->dispatch(legion_fill_launcher.get()); +} + +void FillLauncher::pack_args() { mapper_arg_->pack(0); } + +std::unique_ptr FillLauncher::build_index_fill( + const Legion::Domain& launch_domain, + LogicalStore* lhs, + const ProjectionInfo& lhs_proj, + LogicalStore* value) +{ + pack_args(); + auto* runtime = Runtime::get_runtime(); + auto& provenance = runtime->provenance_manager()->get_provenance(); + auto lhs_region_field = lhs->get_region_field(); + auto lhs_region = lhs_region_field->region(); + auto field_id = lhs_region_field->field_id(); + auto future_value = value->get_future(); + auto lhs_parent = Runtime::get_runtime()->find_parent_region(lhs_region); + auto index_fill = std::make_unique(launch_domain, + lhs_proj.partition, + lhs_parent, + future_value, + lhs_proj.proj_id, + Legion::Predicate::TRUE_PRED, + library_->get_mapper_id(), + lhs_proj.tag, + mapper_arg_->to_legion_buffer(), + provenance.c_str()); + + index_fill->add_field(field_id); + return std::move(index_fill); +} + +std::unique_ptr FillLauncher::build_single_fill( + LogicalStore* lhs, const ProjectionInfo& lhs_proj, LogicalStore* value) +{ + pack_args(); + auto* runtime = Runtime::get_runtime(); + auto& provenance = runtime->provenance_manager()->get_provenance(); + auto lhs_region_field = lhs->get_region_field(); + auto lhs_region = lhs_region_field->region(); + auto field_id = lhs_region_field->field_id(); + auto future_value = value->get_future(); + auto lhs_parent = Runtime::get_runtime()->find_parent_region(lhs_region); + auto single_fill = std::make_unique(lhs_region, + lhs_parent, + future_value, + Legion::Predicate::TRUE_PRED, + library_->get_mapper_id(), + lhs_proj.tag, + mapper_arg_->to_legion_buffer(), + provenance.c_str()); + + single_fill->add_field(field_id); + return std::move(single_fill); +} + +} // namespace legate::detail diff --git a/src/core/runtime/detail/fill_launcher.h b/src/core/runtime/detail/fill_launcher.h new file mode 100644 index 0000000000..071241ad24 --- /dev/null +++ b/src/core/runtime/detail/fill_launcher.h @@ -0,0 +1,67 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include + +#include "legion.h" + +namespace legate { +class BufferBuilder; +class LibraryContext; +} // namespace legate + +namespace legate::mapping { +class MachineDesc; +} // namespace legate::mapping + +namespace legate::detail { + +class LogicalStore; +class ProjectionInfo; + +class FillLauncher { + public: + FillLauncher(LibraryContext* library, const mapping::MachineDesc& machine, int64_t tag = 0); + ~FillLauncher(); + + public: + void launch(const Legion::Domain& launch_domain, + LogicalStore* lhs, + const ProjectionInfo& lhs_proj, + LogicalStore* value); + void launch_single(LogicalStore* lhs, const ProjectionInfo& lhs_proj, LogicalStore* value); + + private: + void pack_args(); + std::unique_ptr build_index_fill(const Legion::Domain& launch_domain, + LogicalStore* lhs, + const ProjectionInfo& lhs_proj, + LogicalStore* value); + std::unique_ptr build_single_fill(LogicalStore* lhs, + const ProjectionInfo& lhs_proj, + LogicalStore* value); + + private: + LibraryContext* library_; + int64_t tag_; + + private: + BufferBuilder* mapper_arg_; +}; + +} // namespace legate::detail diff --git a/src/core/runtime/detail/launcher_arg.cc b/src/core/runtime/detail/launcher_arg.cc index 18b067e663..946d9336b7 100644 --- a/src/core/runtime/detail/launcher_arg.cc +++ b/src/core/runtime/detail/launcher_arg.cc @@ -16,7 +16,6 @@ #include "core/runtime/detail/launcher_arg.h" #include "core/data/detail/logical_region_field.h" -#include "core/runtime/detail/launcher.h" #include "core/runtime/detail/req_analyzer.h" namespace legate::detail { @@ -27,13 +26,13 @@ RegionFieldArg::RegionFieldArg(RequirementAnalyzer* analyzer, LogicalStore* store, Legion::FieldID field_id, Legion::PrivilegeMode privilege, - const ProjectionInfo* proj_info) + std::unique_ptr proj_info) : analyzer_(analyzer), store_(store), region_(store_->get_region_field()->region()), field_id_(field_id), privilege_(privilege), - proj_info_(proj_info) + proj_info_(std::move(proj_info)) { } @@ -43,7 +42,7 @@ void RegionFieldArg::pack(BufferBuilder& buffer) const buffer.pack(proj_info_->redop); buffer.pack(region_.get_dim()); - buffer.pack(analyzer_->get_requirement_index(region_, privilege_, proj_info_)); + buffer.pack(analyzer_->get_requirement_index(region_, privilege_, *proj_info_)); buffer.pack(field_id_); } diff --git a/src/core/runtime/detail/launcher_arg.h b/src/core/runtime/detail/launcher_arg.h index 2d1ba8c895..9a8287a1fa 100644 --- a/src/core/runtime/detail/launcher_arg.h +++ b/src/core/runtime/detail/launcher_arg.h @@ -37,10 +37,10 @@ struct ScalarArg : public ArgWrapper { ScalarArg(const T& value) : value_(value) {} public: - virtual ~ScalarArg() {} + ~ScalarArg() {} public: - virtual void pack(BufferBuilder& buffer) const override { buffer.pack(value_); } + void pack(BufferBuilder& buffer) const override { buffer.pack(value_); } private: T value_; @@ -51,10 +51,10 @@ struct UntypedScalarArg : public ArgWrapper { UntypedScalarArg(const Scalar& scalar) : scalar_(scalar) {} public: - virtual ~UntypedScalarArg() {} + ~UntypedScalarArg() {} public: - virtual void pack(BufferBuilder& buffer) const override; + void pack(BufferBuilder& buffer) const override; private: Scalar scalar_; @@ -66,13 +66,13 @@ struct RegionFieldArg : public ArgWrapper { LogicalStore* store, Legion::FieldID field_id, Legion::PrivilegeMode privilege, - const ProjectionInfo* proj_info); + std::unique_ptr proj_info); public: - virtual void pack(BufferBuilder& buffer) const override; + void pack(BufferBuilder& buffer) const override; public: - virtual ~RegionFieldArg() {} + ~RegionFieldArg() {} private: RequirementAnalyzer* analyzer_; @@ -80,7 +80,7 @@ struct RegionFieldArg : public ArgWrapper { Legion::LogicalRegion region_; Legion::FieldID field_id_; Legion::PrivilegeMode privilege_; - const ProjectionInfo* proj_info_; + std::unique_ptr proj_info_; }; struct OutputRegionArg : public ArgWrapper { @@ -91,10 +91,10 @@ struct OutputRegionArg : public ArgWrapper { Legion::FieldID field_id); public: - virtual void pack(BufferBuilder& buffer) const override; + void pack(BufferBuilder& buffer) const override; public: - virtual ~OutputRegionArg() {} + ~OutputRegionArg() {} public: LogicalStore* store() const { return store_; } @@ -118,10 +118,10 @@ struct FutureStoreArg : public ArgWrapper { Legion::ReductionOpID redop); public: - virtual ~FutureStoreArg() {} + ~FutureStoreArg() {} public: - virtual void pack(BufferBuilder& buffer) const override; + void pack(BufferBuilder& buffer) const override; private: LogicalStore* store_; diff --git a/src/core/runtime/detail/projection.cc b/src/core/runtime/detail/projection.cc new file mode 100644 index 0000000000..3a7b8fbf2f --- /dev/null +++ b/src/core/runtime/detail/projection.cc @@ -0,0 +1,92 @@ +/* Copyright 2022 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/runtime/detail/projection.h" +#include "core/runtime/detail/runtime.h" + +namespace legate::detail { + +template <> +void ProjectionInfo::populate_requirement(Legion::RegionRequirement& requirement, + const Legion::LogicalRegion& region, + const std::vector& fields, + Legion::PrivilegeMode privilege) const +{ + auto parent = Runtime::get_runtime()->find_parent_region(region); + + if (LEGION_REDUCE == privilege) { + new (&requirement) Legion::RegionRequirement(region, redop, LEGION_EXCLUSIVE, parent, tag); + } else { + new (&requirement) Legion::RegionRequirement(region, privilege, LEGION_EXCLUSIVE, parent, tag); + } + requirement.add_fields(fields).add_flags(flags); +} + +template <> +void ProjectionInfo::populate_requirement(Legion::RegionRequirement& requirement, + const Legion::LogicalRegion& region, + const std::vector& fields, + Legion::PrivilegeMode privilege) const +{ + if (Legion::LogicalPartition::NO_PART == partition) { + populate_requirement(requirement, region, fields, privilege); + return; + } + + auto parent = Runtime::get_runtime()->find_parent_region(region); + + if (LEGION_REDUCE == privilege) { + new (&requirement) + Legion::RegionRequirement(partition, proj_id, redop, LEGION_EXCLUSIVE, parent, tag); + } else { + new (&requirement) + Legion::RegionRequirement(partition, proj_id, privilege, LEGION_EXCLUSIVE, parent, tag); + } + requirement.add_fields(fields).add_flags(flags); +} + +ProjectionInfo::ProjectionInfo(Legion::LogicalPartition _partition, Legion::ProjectionID _proj_id) + : partition(_partition), proj_id(_proj_id) +{ +} + +bool ProjectionInfo::operator<(const ProjectionInfo& other) const +{ + if (partition < other.partition) + return true; + else if (other.partition < partition) + return false; + if (proj_id < other.proj_id) + return true; + else if (proj_id > other.proj_id) + return false; + if (redop < other.redop) + return true; + else if (redop > other.redop) + return false; + if (tag < other.tag) + return true; + else + return false; +} + +bool ProjectionInfo::operator==(const ProjectionInfo& other) const +{ + return partition == other.partition && proj_id == other.proj_id && redop == other.redop && + tag == other.tag && flags == other.flags; +} + +} // namespace legate::detail diff --git a/src/core/runtime/detail/projection.h b/src/core/runtime/detail/projection.h new file mode 100644 index 0000000000..c4dab4828b --- /dev/null +++ b/src/core/runtime/detail/projection.h @@ -0,0 +1,52 @@ +/* Copyright 2022 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include +#include "core/utilities/typedefs.h" + +namespace legate::detail { + +struct ProjectionInfo { + ProjectionInfo() {} + ProjectionInfo(Legion::LogicalPartition partition, Legion::ProjectionID proj_id); + + ProjectionInfo(const ProjectionInfo&) = default; + ProjectionInfo& operator=(const ProjectionInfo&) = default; + + bool operator<(const ProjectionInfo& other) const; + bool operator==(const ProjectionInfo& other) const; + + // TODO: Ideally we want this method to return a requirement, instead of taking an inout argument. + // We go with an inout parameter for now, as RegionRequirement doesn't have a move + // constructor/assignment. + template + void populate_requirement(Legion::RegionRequirement& requirement, + const Legion::LogicalRegion& region, + const std::vector& fields, + Legion::PrivilegeMode privilege) const; + + void set_reduction_op(Legion::ReductionOpID _redop) { redop = _redop; } + + Legion::LogicalPartition partition{Legion::LogicalPartition::NO_PART}; + Legion::ProjectionID proj_id{0}; + Legion::ReductionOpID redop{-1}; + Legion::MappingTagID tag{0}; + Legion::RegionFlags flags{LEGION_NO_FLAG}; +}; + +} // namespace legate::detail diff --git a/src/core/runtime/detail/req_analyzer.cc b/src/core/runtime/detail/req_analyzer.cc index a1d97a08e3..c38e88abc4 100644 --- a/src/core/runtime/detail/req_analyzer.cc +++ b/src/core/runtime/detail/req_analyzer.cc @@ -19,123 +19,17 @@ namespace legate::detail { -///////////// -// Projection -///////////// - -Projection::Projection(Legion::LogicalPartition p, Legion::ProjectionID pr) - : partition(p), proj_id(pr) -{ -} - -void Projection::set_reduction_op(Legion::ReductionOpID r) { redop = r; } - -///////////////// -// ProjectionInfo -///////////////// - -ProjectionInfo::ProjectionInfo(const Projection* proj, - Legion::MappingTagID _tag, - Legion::RegionFlags _flags) - : partition(proj->partition), proj_id(proj->proj_id), redop(proj->redop), tag(_tag), flags(_flags) -{ -} - -bool ProjectionInfo::operator<(const ProjectionInfo& other) const -{ - if (partition < other.partition) - return true; - else if (other.partition < partition) - return false; - if (proj_id < other.proj_id) - return true; - else if (proj_id > other.proj_id) - return false; - if (redop < other.redop) - return true; - else if (redop > other.redop) - return false; - if (tag < other.tag) - return true; - else - return false; -} - -bool ProjectionInfo::operator==(const ProjectionInfo& other) const -{ - return partition == other.partition && proj_id == other.proj_id && redop == other.redop && - tag == other.tag && flags == other.flags; -} - -void ProjectionInfo::populate_launcher(Legion::TaskLauncher* task, - const Legion::LogicalRegion& region, - const std::vector& fields, - Legion::PrivilegeMode privilege) const -{ - Legion::RegionRequirement legion_req; - - auto parent = Runtime::get_runtime()->find_parent_region(region); - - if (LEGION_REDUCE == privilege) { -#ifdef DEBUG_LEGATE - assert(redop != -1); -#endif - new (&legion_req) Legion::RegionRequirement(region, redop, LEGION_EXCLUSIVE, parent, tag); - } else { - new (&legion_req) Legion::RegionRequirement(region, privilege, LEGION_EXCLUSIVE, parent, tag); - } - - legion_req.add_fields(fields).add_flags(flags); - task->add_region_requirement(legion_req); -} - -void ProjectionInfo::populate_launcher(Legion::IndexTaskLauncher* task, - const Legion::LogicalRegion& region, - const std::vector& fields, - Legion::PrivilegeMode privilege) const -{ - Legion::RegionRequirement legion_req; - - auto parent = Runtime::get_runtime()->find_parent_region(region); - - // Broadcast - if (Legion::LogicalPartition::NO_PART == partition) { - if (LEGION_REDUCE == privilege) { -#ifdef DEBUG_LEGATE - assert(redop != -1); -#endif - new (&legion_req) Legion::RegionRequirement(region, redop, LEGION_EXCLUSIVE, parent, tag); - } else { - new (&legion_req) Legion::RegionRequirement(region, privilege, LEGION_EXCLUSIVE, parent, tag); - } - } else { - if (LEGION_REDUCE == privilege) { -#ifdef DEBUG_LEGATE - assert(redop != -1); -#endif - new (&legion_req) - Legion::RegionRequirement(partition, proj_id, redop, LEGION_EXCLUSIVE, parent, tag); - } else { - new (&legion_req) - Legion::RegionRequirement(partition, proj_id, privilege, LEGION_EXCLUSIVE, parent, tag); - } - } - - legion_req.add_fields(fields).add_flags(flags); - task->add_region_requirement(legion_req); -} - //////////////// // ProjectionSet //////////////// -void ProjectionSet::insert(Legion::PrivilegeMode new_privilege, const ProjectionInfo* proj_info) +void ProjectionSet::insert(Legion::PrivilegeMode new_privilege, const ProjectionInfo& proj_info) { if (proj_infos.empty()) privilege = new_privilege; // conflicting privileges are promoted to a single read-write privilege else if (privilege != new_privilege) privilege = LEGION_READ_WRITE; - proj_infos.emplace(*proj_info); + proj_infos.emplace(proj_info); if (privilege != LEGION_READ_ONLY && proj_infos.size() > 1) { log_legate.error("Interfering requirements are found"); @@ -149,7 +43,7 @@ void ProjectionSet::insert(Legion::PrivilegeMode new_privilege, const Projection void FieldSet::insert(Legion::FieldID field_id, Legion::PrivilegeMode privilege, - const ProjectionInfo* proj_info) + const ProjectionInfo& proj_info) { field_projs_[field_id].insert(privilege, proj_info); } @@ -157,10 +51,10 @@ void FieldSet::insert(Legion::FieldID field_id, uint32_t FieldSet::num_requirements() const { return static_cast(coalesced_.size()); } uint32_t FieldSet::get_requirement_index(Legion::PrivilegeMode privilege, - const ProjectionInfo* proj_info) const + const ProjectionInfo& proj_info) const { - auto finder = req_indices_.find(Key(privilege, *proj_info)); - if (req_indices_.end() == finder) finder = req_indices_.find(Key(LEGION_READ_WRITE, *proj_info)); + auto finder = req_indices_.find(Key(privilege, proj_info)); + if (req_indices_.end() == finder) finder = req_indices_.find(Key(LEGION_READ_WRITE, proj_info)); #ifdef DEBUG_LEGATE assert(finder != req_indices_.end()); #endif @@ -178,25 +72,29 @@ void FieldSet::coalesce() for (const auto& entry : coalesced_) req_indices_[entry.first] = idx++; } -void FieldSet::populate_launcher(Legion::IndexTaskLauncher* task, - const Legion::LogicalRegion& region) const -{ - for (auto& entry : coalesced_) { - auto privilege = entry.first.first; - const auto& proj_info = entry.first.second; - const auto& fields = entry.second; - proj_info.populate_launcher(task, region, fields, privilege); - } -} +namespace { + +template +constexpr bool is_single; +template <> +constexpr bool is_single = true; +template <> +constexpr bool is_single = false; -void FieldSet::populate_launcher(Legion::TaskLauncher* task, - const Legion::LogicalRegion& region) const +} // namespace + +template +void FieldSet::populate_launcher(Launcher* task, const Legion::LogicalRegion& region) const { for (auto& entry : coalesced_) { auto privilege = entry.first.first; const auto& proj_info = entry.first.second; const auto& fields = entry.second; - proj_info.populate_launcher(task, region, fields, privilege); + + task->region_requirements.push_back(Legion::RegionRequirement()); + auto& requirement = task->region_requirements.back(); + proj_info.template populate_requirement>( + requirement, region, fields, privilege); } } @@ -209,14 +107,14 @@ RequirementAnalyzer::~RequirementAnalyzer() {} void RequirementAnalyzer::insert(const Legion::LogicalRegion& region, Legion::FieldID field_id, Legion::PrivilegeMode privilege, - const ProjectionInfo* proj_info) + const ProjectionInfo& proj_info) { field_sets_[region].first.insert(field_id, privilege, proj_info); } uint32_t RequirementAnalyzer::get_requirement_index(const Legion::LogicalRegion& region, Legion::PrivilegeMode privilege, - const ProjectionInfo* proj_info) const + const ProjectionInfo& proj_info) const { auto finder = field_sets_.find(region); #ifdef DEBUG_LEGATE @@ -242,13 +140,16 @@ void RequirementAnalyzer::analyze_requirements() void RequirementAnalyzer::populate_launcher(Legion::IndexTaskLauncher* task) const { - for (auto& entry : field_sets_) { - const auto& field_set = entry.second.first; - field_set.populate_launcher(task, entry.first); - } + _populate_launcher(task); } void RequirementAnalyzer::populate_launcher(Legion::TaskLauncher* task) const +{ + _populate_launcher(task); +} + +template +void RequirementAnalyzer::_populate_launcher(Launcher* task) const { for (auto& entry : field_sets_) { const auto& field_set = entry.second.first; diff --git a/src/core/runtime/detail/req_analyzer.h b/src/core/runtime/detail/req_analyzer.h index 8e152b89d8..5a433dd6b7 100644 --- a/src/core/runtime/detail/req_analyzer.h +++ b/src/core/runtime/detail/req_analyzer.h @@ -16,59 +16,14 @@ #pragma once -#include -#include "legate.h" +#include "core/runtime/detail/projection.h" +#include "core/runtime/detail/runtime.h" namespace legate::detail { -class Projection { - public: - Projection() {} - Projection(Legion::LogicalPartition partition, Legion::ProjectionID proj_id); - - public: - void set_reduction_op(Legion::ReductionOpID redop); - - public: - const Legion::LogicalPartition partition{Legion::LogicalPartition::NO_PART}; - const Legion::ProjectionID proj_id{0}; - // TODO: Make this const as well - Legion::ReductionOpID redop{-1}; -}; - -class ProjectionInfo { - public: - ProjectionInfo(const Projection* proj, Legion::MappingTagID tag, Legion::RegionFlags flags); - - public: - ProjectionInfo(const ProjectionInfo&) = default; - ProjectionInfo& operator=(const ProjectionInfo&) = default; - - public: - bool operator<(const ProjectionInfo& other) const; - bool operator==(const ProjectionInfo& other) const; - - public: - void populate_launcher(Legion::TaskLauncher* task, - const Legion::LogicalRegion& region, - const std::vector& fields, - Legion::PrivilegeMode privilege) const; - void populate_launcher(Legion::IndexTaskLauncher* task, - const Legion::LogicalRegion& region, - const std::vector& fields, - Legion::PrivilegeMode privilege) const; - - public: - Legion::LogicalPartition partition; - Legion::ProjectionID proj_id; - Legion::ReductionOpID redop; - Legion::MappingTagID tag; - Legion::RegionFlags flags; -}; - class ProjectionSet { public: - void insert(Legion::PrivilegeMode new_privilege, const ProjectionInfo* proj_info); + void insert(Legion::PrivilegeMode new_privilege, const ProjectionInfo& proj_info); public: Legion::PrivilegeMode privilege; @@ -82,16 +37,15 @@ class FieldSet { public: void insert(Legion::FieldID field_id, Legion::PrivilegeMode privilege, - const ProjectionInfo* proj_info); + const ProjectionInfo& proj_info); uint32_t num_requirements() const; uint32_t get_requirement_index(Legion::PrivilegeMode privilege, - const ProjectionInfo* proj_info) const; + const ProjectionInfo& proj_info) const; public: void coalesce(); - void populate_launcher(Legion::IndexTaskLauncher* task, - const Legion::LogicalRegion& region) const; - void populate_launcher(Legion::TaskLauncher* task, const Legion::LogicalRegion& region) const; + template + void populate_launcher(Launcher* task, const Legion::LogicalRegion& region) const; private: std::map> coalesced_; @@ -109,10 +63,10 @@ class RequirementAnalyzer { void insert(const Legion::LogicalRegion& region, Legion::FieldID field_id, Legion::PrivilegeMode privilege, - const ProjectionInfo* proj_info); + const ProjectionInfo& proj_info); uint32_t get_requirement_index(const Legion::LogicalRegion& region, Legion::PrivilegeMode privilege, - const ProjectionInfo* proj_info) const; + const ProjectionInfo& proj_info) const; bool empty() const { return field_sets_.empty(); } public: @@ -120,6 +74,10 @@ class RequirementAnalyzer { void populate_launcher(Legion::IndexTaskLauncher* task) const; void populate_launcher(Legion::TaskLauncher* task) const; + private: + template + void _populate_launcher(Launcher* task) const; + private: std::map> field_sets_; }; diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index 32e430572e..a182e464c7 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -22,7 +22,8 @@ #include "core/mapping/core_mapper.h" #include "core/mapping/default_mapper.h" #include "core/partitioning/partitioner.h" -#include "core/runtime/detail/launcher.h" +#include "core/runtime/detail/fill.h" +#include "core/runtime/detail/task_launcher.h" #include "core/runtime/projection.h" #include "core/runtime/shard.h" #include "env_defaults.h" @@ -118,12 +119,6 @@ int32_t Runtime::find_reduction_operator(int32_t type_uid, int32_t op_kind) cons return finder->second; } -void Runtime::enter_callback() { in_callback_ = true; } - -void Runtime::exit_callback() { in_callback_ = false; } - -bool Runtime::is_in_callback() const { return in_callback_; } - void Runtime::initialize(Legion::Context legion_context) { if (initialized_) throw std::runtime_error("Legate runtime has already been initialized"); @@ -182,6 +177,22 @@ std::unique_ptr Runtime::create_task(LibraryContext* library, return std::unique_ptr(task); } +std::unique_ptr Runtime::create_copy(LibraryContext* library) +{ + auto machine = machine_manager_->get_machine(); + auto copy = new Copy(library, next_unique_id_++, std::move(machine)); + return std::unique_ptr(copy); +} + +void Runtime::issue_fill(LibraryContext* library, + legate::LogicalStore lhs, + legate::LogicalStore value) +{ + auto machine = machine_manager_->get_machine(); + submit( + std::unique_ptr(new Fill(library, lhs, value, next_unique_id_++, std::move(machine)))); +} + void Runtime::flush_scheduling_window() { if (operations_.size() == 0) return; @@ -534,6 +545,30 @@ Legion::FutureMap Runtime::dispatch(Legion::IndexTaskLauncher* launcher, return legion_runtime_->execute_index_space(legion_context_, *launcher, output_requirements); } +void Runtime::dispatch(Legion::CopyLauncher* launcher) +{ + assert(nullptr != legion_context_); + return legion_runtime_->issue_copy_operation(legion_context_, *launcher); +} + +void Runtime::dispatch(Legion::IndexCopyLauncher* launcher) +{ + assert(nullptr != legion_context_); + return legion_runtime_->issue_copy_operation(legion_context_, *launcher); +} + +void Runtime::dispatch(Legion::FillLauncher* launcher) +{ + assert(nullptr != legion_context_); + return legion_runtime_->fill_fields(legion_context_, *launcher); +} + +void Runtime::dispatch(Legion::IndexFillLauncher* launcher) +{ + assert(nullptr != legion_context_); + return legion_runtime_->fill_fields(legion_context_, *launcher); +} + Legion::Future Runtime::extract_scalar(const Legion::Future& result, uint32_t idx) const { auto& machine = get_machine(); diff --git a/src/core/runtime/detail/runtime.h b/src/core/runtime/detail/runtime.h index 9592a593d1..a2487719b9 100644 --- a/src/core/runtime/detail/runtime.h +++ b/src/core/runtime/detail/runtime.h @@ -19,6 +19,7 @@ #include #include +#include "core/data/logical_store.h" #include "core/data/shape.h" #include "core/data/store.h" #include "core/mapping/machine.h" @@ -35,6 +36,7 @@ namespace legate { class AutoTask; +class Copy; class LibraryContext; class ManualTask; class Operation; @@ -61,11 +63,6 @@ class Runtime { void record_reduction_operator(int32_t type_uid, int32_t op_kind, int32_t legion_op_id); int32_t find_reduction_operator(int32_t type_uid, int32_t op_kind) const; - public: - void enter_callback(); - void exit_callback(); - bool is_in_callback() const; - public: void initialize(Legion::Context legion_context); @@ -75,6 +72,8 @@ class Runtime { std::unique_ptr create_task(LibraryContext* library, int64_t task_id, const Shape& launch_shape); + std::unique_ptr create_copy(LibraryContext* library); + void issue_fill(LibraryContext* library, legate::LogicalStore lhs, legate::LogicalStore value); void flush_scheduling_window(); void submit(std::unique_ptr op); @@ -155,6 +154,10 @@ class Runtime { std::vector* output_requirements = nullptr); Legion::FutureMap dispatch(Legion::IndexTaskLauncher* launcher, std::vector* output_requirements = nullptr); + void dispatch(Legion::CopyLauncher* launcher); + void dispatch(Legion::IndexCopyLauncher* launcher); + void dispatch(Legion::FillLauncher* launcher); + void dispatch(Legion::IndexFillLauncher* launcher); public: Legion::Future extract_scalar(const Legion::Future& result, uint32_t idx) const; @@ -220,9 +223,6 @@ class Runtime { uint64_t next_store_id_{1}; uint64_t next_storage_id_{1}; - private: - bool in_callback_{false}; - private: std::map libraries_{}; diff --git a/src/core/runtime/detail/launcher.cc b/src/core/runtime/detail/task_launcher.cc similarity index 88% rename from src/core/runtime/detail/launcher.cc rename to src/core/runtime/detail/task_launcher.cc index d20c6f0d3f..6ed99361ce 100644 --- a/src/core/runtime/detail/launcher.cc +++ b/src/core/runtime/detail/task_launcher.cc @@ -14,7 +14,7 @@ * */ -#include "core/runtime/detail/launcher.h" +#include "core/runtime/detail/task_launcher.h" #include "core/data/detail/logical_region_field.h" #include "core/data/detail/logical_store.h" #include "core/data/scalar.h" @@ -77,32 +77,24 @@ void TaskLauncher::add_scalar(const Scalar& scalar) scalars_.push_back(new UntypedScalarArg(scalar)); } -void TaskLauncher::add_input(LogicalStore* store, - std::unique_ptr proj, - Legion::MappingTagID tag, - Legion::RegionFlags flags) +void TaskLauncher::add_input(LogicalStore* store, std::unique_ptr proj_info) { - add_store(inputs_, store, std::move(proj), READ_ONLY, tag, flags); + add_store(inputs_, store, std::move(proj_info), READ_ONLY); } -void TaskLauncher::add_output(LogicalStore* store, - std::unique_ptr proj, - Legion::MappingTagID tag, - Legion::RegionFlags flags) +void TaskLauncher::add_output(LogicalStore* store, std::unique_ptr proj_info) { - add_store(outputs_, store, std::move(proj), WRITE_ONLY, tag, flags); + add_store(outputs_, store, std::move(proj_info), WRITE_ONLY); } void TaskLauncher::add_reduction(LogicalStore* store, - std::unique_ptr proj, - bool read_write, - Legion::MappingTagID tag, - Legion::RegionFlags flags) + std::unique_ptr proj_info, + bool read_write) { if (read_write) - add_store(reductions_, store, std::move(proj), READ_WRITE, tag, flags); + add_store(reductions_, store, std::move(proj_info), READ_WRITE); else - add_store(reductions_, store, std::move(proj), REDUCE, tag, flags); + add_store(reductions_, store, std::move(proj_info), REDUCE); } void TaskLauncher::add_unbound_output(LogicalStore* store, @@ -155,27 +147,22 @@ Legion::Future TaskLauncher::execute_single() void TaskLauncher::add_store(std::vector& args, LogicalStore* store, - std::unique_ptr proj, - Legion::PrivilegeMode privilege, - Legion::MappingTagID tag, - Legion::RegionFlags flags) + std::unique_ptr proj_info, + Legion::PrivilegeMode privilege) { - auto redop = proj->redop; - if (store->has_scalar_storage()) { auto has_storage = privilege != WRITE_ONLY; auto read_only = privilege == READ_ONLY; if (has_storage) futures_.push_back(store->get_future()); - args.push_back(new FutureStoreArg(store, read_only, has_storage, redop)); + args.push_back(new FutureStoreArg(store, read_only, has_storage, proj_info->redop)); } else { auto region_field = store->get_region_field(); auto region = region_field->region(); auto field_id = region_field->field_id(); - auto proj_info = new ProjectionInfo(proj.get(), tag, flags); - - req_analyzer_->insert(region, field_id, privilege, proj_info); - args.push_back(new RegionFieldArg(req_analyzer_, store, field_id, privilege, proj_info)); + req_analyzer_->insert(region, field_id, privilege, *proj_info); + args.push_back( + new RegionFieldArg(req_analyzer_, store, field_id, privilege, std::move(proj_info))); } } diff --git a/src/core/runtime/detail/launcher.h b/src/core/runtime/detail/task_launcher.h similarity index 82% rename from src/core/runtime/detail/launcher.h rename to src/core/runtime/detail/task_launcher.h index ec21d5304b..032f1a1c91 100644 --- a/src/core/runtime/detail/launcher.h +++ b/src/core/runtime/detail/task_launcher.h @@ -32,7 +32,7 @@ namespace legate::detail { class ArgWrapper; class LogicalStore; -class Projection; +class ProjectionInfo; class OutputRegionArg; class OutputRequirementAnalyzer; class RequirementAnalyzer; @@ -59,19 +59,11 @@ class TaskLauncher { public: void add_scalar(const Scalar& scalar); - void add_input(LogicalStore* store, - std::unique_ptr proj, - Legion::MappingTagID tag = 0, - Legion::RegionFlags flags = LEGION_NO_FLAG); - void add_output(LogicalStore* store, - std::unique_ptr proj, - Legion::MappingTagID tag = 0, - Legion::RegionFlags flags = LEGION_NO_FLAG); + void add_input(LogicalStore* store, std::unique_ptr proj_info); + void add_output(LogicalStore* store, std::unique_ptr proj_info); void add_reduction(LogicalStore* store, - std::unique_ptr proj, - bool read_write = false, - Legion::MappingTagID tag = 0, - Legion::RegionFlags flags = LEGION_NO_FLAG); + std::unique_ptr proj_info, + bool read_write); void add_unbound_output(LogicalStore* store, Legion::FieldSpace field_space, Legion::FieldID field_id); @@ -90,10 +82,8 @@ class TaskLauncher { private: void add_store(std::vector& args, LogicalStore* store, - std::unique_ptr proj, - Legion::PrivilegeMode privilege, - Legion::MappingTagID tag, - Legion::RegionFlags flags); + std::unique_ptr proj_info, + Legion::PrivilegeMode privilege); public: Legion::FutureMap execute(const Legion::Domain& launch_domain); diff --git a/src/core/runtime/operation.h b/src/core/runtime/operation.h index 124d29048c..eb6daf52a1 100644 --- a/src/core/runtime/operation.h +++ b/src/core/runtime/operation.h @@ -59,9 +59,9 @@ class Operation { virtual ~Operation() {} public: - virtual void add_to_solver(detail::ConstraintSolver& constraint_graph) const = 0; - virtual void launch(detail::Strategy* strategy) = 0; - virtual std::string to_string() const = 0; + virtual void add_to_solver(detail::ConstraintSolver& solver) = 0; + virtual void launch(detail::Strategy* strategy) = 0; + virtual std::string to_string() const = 0; public: /** @@ -87,6 +87,9 @@ class Operation { */ const std::string& provenance() const { return provenance_; } + protected: + void record_partition(const Variable* variable, std::shared_ptr store); + protected: LibraryContext* library_; uint64_t unique_id_; @@ -128,6 +131,12 @@ class Task : public Operation { * @param scalar A scalar to add to the task */ void add_scalar_arg(const Scalar& scalar); + /** + * @brief Adds a by-value scalar argument to the task + * + * @param scalar A scalar to add to the task + */ + void add_scalar_arg(Scalar&& scalar); /** * @brief Sets whether the task needs a concurrent task launch. * @@ -243,7 +252,7 @@ class AutoTask : public Task { * @param constraint A partitioning constraint */ void add_constraint(std::unique_ptr constraint); - void add_to_solver(detail::ConstraintSolver& constraint_graph) const override; + void add_to_solver(detail::ConstraintSolver& solver) override; private: std::vector> constraints_{}; @@ -322,10 +331,51 @@ class ManualTask : public Task { void launch(detail::Strategy* strategy) override; public: - void add_to_solver(detail::ConstraintSolver& constraint_graph) const override; + void add_to_solver(detail::ConstraintSolver& solver) override; private: std::unique_ptr strategy_; }; +class Copy : public Operation { + private: + friend class detail::Runtime; + Copy(LibraryContext* library, int64_t unique_id, mapping::MachineDesc&& machine); + + public: + void add_input(LogicalStore store); + void add_output(LogicalStore store); + void add_reduction(LogicalStore store, Legion::ReductionOpID redop); + void add_source_indirect(LogicalStore store); + void add_target_indirect(LogicalStore store); + + private: + void add_store(std::vector& store_args, + LogicalStore& store, + const Variable* partition_symbol); + void add_store(std::optional& store_arg, + LogicalStore& store, + const Variable* partition_symbol); + + public: + void set_source_indirect_out_of_range(bool flag); + void set_target_indirect_out_of_range(bool flag); + + public: + void launch(detail::Strategy* strategy) override; + + public: + void add_to_solver(detail::ConstraintSolver& solver) override; + + public: + std::string to_string() const override; + + private: + std::vector> constraints_{}; + std::optional source_indirect_{}; + std::optional target_indirect_{}; + bool source_indirect_out_of_range_{true}; + bool target_indirect_out_of_range_{true}; +}; + } // namespace legate diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 110f910410..0b59ed0e0a 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -163,6 +163,21 @@ std::unique_ptr Runtime::create_task(LibraryContext* library, return impl_->create_task(library, task_id, launch_shape); } +std::unique_ptr Runtime::create_copy(LibraryContext* library) +{ + return impl_->create_copy(library); +} + +void Runtime::issue_fill(LibraryContext* library, LogicalStore lhs, LogicalStore value) +{ + impl_->issue_fill(library, lhs, value); +} + +void Runtime::issue_fill(LibraryContext* library, LogicalStore lhs, const Scalar& value) +{ + issue_fill(library, lhs, create_store(value)); +} + void Runtime::submit(std::unique_ptr op) { impl_->submit(std::move(op)); } LogicalStore Runtime::create_store(std::unique_ptr type, int32_t dim) @@ -234,6 +249,8 @@ int32_t start(int32_t argc, char** argv) { return detail::Runtime::start(argc, a int32_t finish() { return detail::Runtime::get_runtime()->finish(); } +const mapping::MachineDesc& get_machine() { return Runtime::get_runtime()->get_machine(); } + } // namespace legate extern "C" { diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 271ab246dd..76aede68d5 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -87,6 +87,7 @@ struct Core { }; class AutoTask; +class Copy; class ManualTask; class Operation; @@ -107,7 +108,7 @@ class Runtime; class Runtime { public: /** - * @brief Find a library + * @brief Finds a library * * @param library_name Library name * @param can_fail Optional flag indicating that the query can fail. When it's true and no @@ -119,7 +120,7 @@ class Runtime { */ LibraryContext* find_library(const std::string& library_name, bool can_fail = false) const; /** - * @brief Create a library + * @brief Creates a library * * A library is a collection of tasks and custom reduction operators. The maximum number of * tasks and reduction operators can be optionally specified with a `ResourceConfig` object. @@ -141,7 +142,7 @@ class Runtime { public: /** - * @brief Create an AutoTask + * @brief Creates an AutoTask * * @param library Library to query the task * @param task_id Library-local Task ID @@ -150,7 +151,7 @@ class Runtime { */ std::unique_ptr create_task(LibraryContext* library, int64_t task_id); /** - * @brief Create a ManualTask + * @brief Creates a ManualTask * * @param library Library to query the task * @param task_id Library-local Task ID @@ -161,6 +162,30 @@ class Runtime { std::unique_ptr create_task(LibraryContext* library, int64_t task_id, const Shape& launch_shape); + /** + * @brief Creates a Copy + * + * @param library Library in which the copy is created + * + * @return Copy object + */ + std::unique_ptr create_copy(LibraryContext* library); + /** + * @brief Fills a given store with a constant + * + * @param library Library in which the fill is performed + * @param lhs Logical store to fill + * @param value Logical store that contains the constant value to fill the store with + */ + void issue_fill(LibraryContext* library, LogicalStore lhs, LogicalStore value); + /** + * @brief Fills a given store with a constant + * + * @param library Library in which the fill is performed + * @param lhs Logical store to fill + * @param value Value to fill the store with + */ + void issue_fill(LibraryContext* library, LogicalStore lhs, const Scalar& value); /** * @brief Submits an operation for execution * @@ -304,6 +329,13 @@ int32_t start(int32_t argc, char** argv); */ int32_t finish(); +/** + * @brief Returns the machine for the current scope + * + * @return Machine object + */ +const mapping::MachineDesc& get_machine(); + } // namespace legate #include "core/runtime/runtime.inl" diff --git a/src/core/runtime/operation.cc b/src/core/runtime/task.cc similarity index 89% rename from src/core/runtime/operation.cc rename to src/core/runtime/task.cc index 06df29427e..6abd35b290 100644 --- a/src/core/runtime/operation.cc +++ b/src/core/runtime/task.cc @@ -27,10 +27,10 @@ #include "core/partitioning/partitioner.h" #include "core/runtime/context.h" #include "core/runtime/detail/communicator_manager.h" -#include "core/runtime/detail/launcher.h" +#include "core/runtime/detail/projection.h" #include "core/runtime/detail/provenance_manager.h" -#include "core/runtime/detail/req_analyzer.h" #include "core/runtime/detail/runtime.h" +#include "core/runtime/detail/task_launcher.h" namespace legate { @@ -54,11 +54,18 @@ const Variable* Operation::declare_partition() detail::LogicalStore* Operation::find_store(const Variable* part_symb) const { - auto finder = store_mappings_.find(*part_symb); -#ifdef DEBUG_LEGATE - assert(store_mappings_.end() != finder); -#endif - return finder->second; + return store_mappings_.at(*part_symb); +} + +void Operation::record_partition(const Variable* variable, + std::shared_ptr store) +{ + if (store_mappings_.find(*variable) != store_mappings_.end()) { + throw std::invalid_argument("Variable " + variable->to_string() + + " is already assigned to another store"); + } + store_mappings_[*variable] = store.get(); + all_stores_.insert(std::move(store)); } //////////////////////////////////////////////////// @@ -75,6 +82,8 @@ Task::Task(LibraryContext* library, void Task::add_scalar_arg(const Scalar& scalar) { scalars_.push_back(scalar); } +void Task::add_scalar_arg(Scalar&& scalar) { scalars_.emplace_back(std::move(scalar)); } + void Task::set_concurrent(bool concurrent) { concurrent_ = concurrent; } void Task::set_side_effect(bool has_side_effect) { has_side_effect_ = has_side_effect; } @@ -96,30 +105,26 @@ void Task::launch(detail::Strategy* p_strategy) detail::TaskLauncher launcher(library_, machine_, provenance_, task_id_); const auto* launch_domain = strategy.launch_domain(this); + auto create_projection_info = [&strategy, &launch_domain](auto& store, auto& var) { + auto store_partition = store->create_partition(strategy[var]); + return store_partition->create_projection_info(launch_domain); + }; + // Add input stores - for (auto& [store, var] : inputs_) { - const auto& part = strategy[var]; - auto store_partition = store->create_partition(part); - auto proj = store_partition->create_projection(launch_domain); - launcher.add_input(store, std::move(proj)); - } + for (auto& [store, var] : inputs_) launcher.add_input(store, create_projection_info(store, var)); // Add normal output stores for (auto& [store, var] : outputs_) { if (store->unbound()) continue; - const auto& part = strategy[var]; - auto store_partition = store->create_partition(part); - auto proj = store_partition->create_projection(launch_domain); - launcher.add_output(store, std::move(proj)); - store->set_key_partition(machine(), part.get()); + launcher.add_output(store, create_projection_info(store, var)); + store->set_key_partition(machine(), strategy[var].get()); } // Add reduction stores uint32_t idx = 0; for (auto& [store, var] : reductions_) { - const auto& part = strategy[var]; - auto store_partition = store->create_partition(part); - auto proj = store_partition->create_projection(launch_domain); + auto store_partition = store->create_partition(strategy[var]); + auto proj = store_partition->create_projection_info(launch_domain); bool read_write = store_partition->is_disjoint_for(launch_domain); auto redop = reduction_ops_[idx++]; proj->set_reduction_op(redop); @@ -294,8 +299,7 @@ void AutoTask::add_store(std::vector& store_args, { auto store_impl = store.impl(); store_args.push_back(StoreArg(store_impl.get(), partition_symbol)); - store_mappings_[*partition_symbol] = store_impl.get(); - all_stores_.insert(std::move(store_impl)); + record_partition(partition_symbol, store_impl); } void AutoTask::add_constraint(std::unique_ptr constraint) @@ -303,10 +307,10 @@ void AutoTask::add_constraint(std::unique_ptr constraint) constraints_.push_back(std::move(constraint)); } -void AutoTask::add_to_solver(detail::ConstraintSolver& solver) const +void AutoTask::add_to_solver(detail::ConstraintSolver& solver) { for (auto& constraint : constraints_) solver.add_constraint(constraint.get()); - for (auto& [_, symb] : outputs_) solver.add_partition_symbol(symb); + for (auto& [_, symb] : outputs_) solver.add_partition_symbol(symb, true); for (auto& [_, symb] : reductions_) solver.add_partition_symbol(symb); for (auto& [_, symb] : inputs_) solver.add_partition_symbol(symb); } @@ -386,6 +390,6 @@ void ManualTask::add_store(std::vector& store_args, void ManualTask::launch(detail::Strategy*) { Task::launch(strategy_.get()); } -void ManualTask::add_to_solver(detail::ConstraintSolver& constraint_graph) const {} +void ManualTask::add_to_solver(detail::ConstraintSolver& solver) {} } // namespace legate diff --git a/src/core/type/type_info.cc b/src/core/type/type_info.cc index bb3fa4b76c..bb87b25a84 100644 --- a/src/core/type/type_info.cc +++ b/src/core/type/type_info.cc @@ -307,4 +307,25 @@ std::unique_ptr complex128() { return primitive_type(Type::Code::COMPLEX12 std::unique_ptr string() { return string_type(); } +namespace { + +constexpr int32_t POINT_UID_BASE = static_cast(Type::Code::INVALID); +constexpr int32_t RECT_UID_BASE = POINT_UID_BASE + LEGATE_MAX_DIM + 1; + +} // namespace + +std::unique_ptr point_type(int32_t dim) +{ + if (dim == 1) return int64(); + return std::make_unique(POINT_UID_BASE + dim, int64(), dim); +} + +std::unique_ptr rect_type(int32_t dim) +{ + std::vector> field_types; + field_types.push_back(point_type(dim)); + field_types.push_back(point_type(dim)); + return std::make_unique(RECT_UID_BASE + dim, std::move(field_types), true /*align*/); +} + } // namespace legate diff --git a/src/core/type/type_info.h b/src/core/type/type_info.h index a200cb6096..30abacc7b4 100644 --- a/src/core/type/type_info.h +++ b/src/core/type/type_info.h @@ -529,4 +529,20 @@ std::unique_ptr complex128(); */ std::unique_ptr string(); +/** + * @ingroup types + * @brief Creates a point type + * + * @return Type object + */ +std::unique_ptr point_type(int32_t dim); + +/** + * @ingroup types + * @brief Creates a rect type + * + * @return Type object + */ +std::unique_ptr rect_type(int32_t dim); + } // namespace legate diff --git a/src/core/utilities/deserializer.h b/src/core/utilities/deserializer.h index 4b1b577c66..f00fb9652b 100644 --- a/src/core/utilities/deserializer.h +++ b/src/core/utilities/deserializer.h @@ -62,6 +62,12 @@ class BaseDeserializer { values.reserve(size); for (uint32_t idx = 0; idx < size; ++idx) values.emplace_back(unpack()); } + void _unpack(std::vector& values) + { + auto size = unpack(); + values.reserve(size); + for (uint32_t idx = 0; idx < size; ++idx) values.emplace_back(_unpack_scalar()); + } template void _unpack(std::pair& values) { @@ -70,7 +76,7 @@ class BaseDeserializer { } public: - void _unpack(Scalar& value); + Scalar _unpack_scalar(); void _unpack(mapping::TaskTarget& value); void _unpack(mapping::ProcessorRange& value); void _unpack(mapping::MachineDesc& value); diff --git a/src/core/utilities/deserializer.inl b/src/core/utilities/deserializer.inl index c35a529c6f..e6e9930382 100644 --- a/src/core/utilities/deserializer.inl +++ b/src/core/utilities/deserializer.inl @@ -28,11 +28,12 @@ BaseDeserializer::BaseDeserializer(const void* args, size_t arglen } template -void BaseDeserializer::_unpack(Scalar& value) +Scalar BaseDeserializer::_unpack_scalar() { auto type = unpack_type(); - value = Scalar(std::move(type), args_.ptr()); - args_ = args_.subspan(value.size()); + Scalar result(std::move(type), args_.ptr()); + args_ = args_.subspan(result.size()); + return result; } template diff --git a/src/core/utilities/tuple.h b/src/core/utilities/tuple.h index 3014b3e12b..2f4d837e22 100644 --- a/src/core/utilities/tuple.h +++ b/src/core/utilities/tuple.h @@ -48,6 +48,7 @@ class tuple { public: bool operator==(const tuple& other) const; + bool operator!=(const tuple& other) const; bool operator<(const tuple& other) const; tuple operator+(const tuple& other) const; tuple operator+(const T& other) const; diff --git a/src/core/utilities/tuple.inl b/src/core/utilities/tuple.inl index 77fc1af436..6535f74cdd 100644 --- a/src/core/utilities/tuple.inl +++ b/src/core/utilities/tuple.inl @@ -68,6 +68,12 @@ bool tuple::operator==(const tuple& other) const return data_ == other.data_; } +template +bool tuple::operator!=(const tuple& other) const +{ + return data_ != other.data_; +} + template bool tuple::operator<(const tuple& other) const { diff --git a/tests/cpp/integration/copy_failure.cc b/tests/cpp/integration/copy_failure.cc new file mode 100644 index 0000000000..3ca2e4b33d --- /dev/null +++ b/tests/cpp/integration/copy_failure.cc @@ -0,0 +1,112 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "legate.h" + +namespace copy_failure { + +void test_input_output_failure() +{ + // inconsistent number of inputs and outputs + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library("legate.core"); + + std::vector extents = {100}; + auto in_store1 = runtime->create_store(extents, legate::int64()); + auto in_store2 = runtime->create_store(extents, legate::int64()); + auto out_store = runtime->create_store(extents, legate::int64()); + // fill input store with some values + auto copy = runtime->create_copy(context); + copy->add_input(in_store1); + copy->add_input(in_store2); + copy->add_output(out_store); + EXPECT_THROW(runtime->submit(std::move(copy)), std::runtime_error); +} + +void test_indirect_failure() +{ + // using indirect with several inputs/outputs + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library("legate.core"); + + std::vector extents = {100}; + auto in_store1 = runtime->create_store(extents, legate::int64()); + auto in_store2 = runtime->create_store(extents, legate::int64()); + auto out_store1 = runtime->create_store(extents, legate::int64()); + auto out_store2 = runtime->create_store(extents, legate::int64()); + auto indirect_store = runtime->create_store(extents, legate::int64()); + + auto copy = runtime->create_copy(context); + copy->add_input(in_store1); + copy->add_input(in_store2); + copy->add_output(out_store1); + copy->add_output(out_store2); + copy->add_target_indirect(indirect_store); + + EXPECT_THROW(runtime->submit(std::move(copy)), std::runtime_error); +} + +void test_shape_check_failure() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library("legate.core"); + + auto store1 = runtime->create_store({10, 10}, legate::int64()); + auto store2 = runtime->create_store({5, 20}, legate::int64()); + auto store3 = runtime->create_store({20, 5}, legate::int64()); + + { + auto copy = runtime->create_copy(context); + copy->add_input(store1); + copy->add_output(store2); + EXPECT_THROW(runtime->submit(std::move(copy)), std::runtime_error); + } + + { + auto copy = runtime->create_copy(context); + copy->add_input(store1); + copy->add_source_indirect(store2); + copy->add_output(store3); + EXPECT_THROW(runtime->submit(std::move(copy)), std::runtime_error); + } + + { + auto copy = runtime->create_copy(context); + copy->add_input(store1); + copy->add_target_indirect(store2); + copy->add_output(store3); + EXPECT_THROW(runtime->submit(std::move(copy)), std::runtime_error); + } + + { + auto copy = runtime->create_copy(context); + copy->add_input(store1); + copy->add_source_indirect(store2); + copy->add_target_indirect(store3); + copy->add_output(store1); + EXPECT_THROW(runtime->submit(std::move(copy)), std::runtime_error); + } +} + +TEST(Copy, FailureInputOutputMismatch) { test_input_output_failure(); } + +TEST(Copy, FailureMultipleIndirectCopies) { test_indirect_failure(); } + +TEST(Copy, FailureDifferentShapes) { test_shape_check_failure(); } + +} // namespace copy_failure diff --git a/tests/cpp/integration/copy_gather.cc b/tests/cpp/integration/copy_gather.cc new file mode 100644 index 0000000000..4d51e2d853 --- /dev/null +++ b/tests/cpp/integration/copy_gather.cc @@ -0,0 +1,188 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "copy_util.inl" +#include "legate.h" + +namespace copy_gather { + +static const char* library_name = "test_copy_gather"; +static legate::Logger logger(library_name); + +constexpr int32_t CHECK_GATHER_TASK = FILL_INDIRECT_TASK + TEST_MAX_DIM * TEST_MAX_DIM; + +template +struct CheckGatherTask : public legate::LegateTask> { + struct CheckGatherTaskBody { + template + void operator()(legate::TaskContext& context) + { + using VAL = legate::legate_type_of; + + auto& src_store = context.inputs().at(0); + auto& tgt_store = context.inputs().at(1); + auto& ind_store = context.inputs().at(2); + + auto ind_shape = ind_store.shape(); + if (ind_shape.empty()) return; + + auto src_acc = src_store.read_accessor(); + auto tgt_acc = tgt_store.read_accessor(); + auto ind_acc = ind_store.read_accessor, IND_DIM>(); + + for (legate::PointInRectIterator it(ind_shape); it.valid(); ++it) { + auto copy = tgt_acc[*it]; + auto source = src_acc[ind_acc[*it]]; + EXPECT_EQ(copy, source); + } + } + }; + + static const int32_t TASK_ID = CHECK_GATHER_TASK + IND_DIM * TEST_MAX_DIM + SRC_DIM; + static void cpu_variant(legate::TaskContext& context) + { + auto type_code = context.inputs().at(0).type().code; + type_dispatch_for_test(type_code, CheckGatherTaskBody{}, context); + } +}; + +void register_tasks() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + FillTask<1>::register_variants(context); + FillTask<2>::register_variants(context); + FillTask<3>::register_variants(context); + + // XXX: Tasks unused by the test cases are commented out + // FillIndirectTask<1, 1>::register_variants(context); + FillIndirectTask<1, 2>::register_variants(context); + // FillIndirectTask<1, 3>::register_variants(context); + // FillIndirectTask<2, 1>::register_variants(context); + FillIndirectTask<2, 2>::register_variants(context); + FillIndirectTask<2, 3>::register_variants(context); + FillIndirectTask<3, 1>::register_variants(context); + FillIndirectTask<3, 2>::register_variants(context); + // FillIndirectTask<3, 3>::register_variants(context); + + // CheckGatherTask<1, 1>::register_variants(context); + CheckGatherTask<1, 2>::register_variants(context); + // CheckGatherTask<1, 3>::register_variants(context); + // CheckGatherTask<2, 1>::register_variants(context); + CheckGatherTask<2, 2>::register_variants(context); + CheckGatherTask<2, 3>::register_variants(context); + CheckGatherTask<3, 1>::register_variants(context); + CheckGatherTask<3, 2>::register_variants(context); + // CheckGatherTask<3, 3>::register_variants(context); +} + +void check_gather_output(legate::LibraryContext* context, + const legate::LogicalStore& src, + const legate::LogicalStore& tgt, + const legate::LogicalStore& ind) +{ + auto runtime = legate::Runtime::get_runtime(); + auto machine = runtime->get_machine(); + int32_t task_id = CHECK_GATHER_TASK + ind.dim() * TEST_MAX_DIM + src.dim(); + auto task = runtime->create_task(context, task_id); + auto src_part = task->declare_partition(); + auto tgt_part = task->declare_partition(); + auto ind_part = task->declare_partition(); + task->add_input(src, src_part); + task->add_input(tgt, tgt_part); + task->add_input(ind, ind_part); + + task->add_constraint(legate::broadcast(src_part, legate::from_range(src.dim()))); + task->add_constraint(legate::broadcast(tgt_part, legate::from_range(tgt.dim()))); + task->add_constraint(legate::broadcast(ind_part, legate::from_range(ind.dim()))); + + runtime->submit(std::move(task)); +} + +struct GatherSpec { + std::vector ind_shape; + std::vector data_shape; + legate::Scalar seed; + + std::string to_string() const + { + std::stringstream ss; + ss << "source shape: " << ::to_string(data_shape) + << ", indirection/target shape: " << ::to_string(ind_shape) + << ", data type: " << seed.type().to_string(); + return std::move(ss).str(); + } +}; + +void test_gather(const GatherSpec& spec) +{ + logger.print() << "Gather Copy: " << spec.to_string(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + auto& type = spec.seed.type(); + auto src = runtime->create_store(spec.data_shape, type.clone()); + auto tgt = runtime->create_store(spec.ind_shape, type.clone()); + auto ind = runtime->create_store(spec.ind_shape, legate::point_type(spec.data_shape.size())); + + fill_input(context, src, spec.seed); + fill_indirect(context, ind, src); + + auto copy = runtime->create_copy(context); + copy->add_input(src); + copy->add_output(tgt); + copy->add_source_indirect(ind); + runtime->submit(std::move(copy)); + + check_gather_output(context, src, tgt, ind); +} + +TEST(Copy, Gather2Dto1D) +{ + legate::Core::perform_registration(); + std::vector shape1d{5}; + test_gather(GatherSpec{shape1d, {7, 11}, legate::Scalar(int64_t(123))}); +} + +TEST(Copy, Gather3Dto2D) +{ + legate::Core::perform_registration(); + test_gather(GatherSpec{{3, 7}, {3, 2, 5}, legate::Scalar(uint32_t(456))}); +} + +TEST(Copy, Gather1Dto3D) +{ + legate::Core::perform_registration(); + std::vector shape1d{5}; + test_gather(GatherSpec{{2, 5, 4}, shape1d, legate::Scalar(789.0)}); +} + +TEST(Copy, Gather2Dto2D) +{ + legate::Core::perform_registration(); + test_gather(GatherSpec{{4, 5}, {10, 11}, legate::Scalar(int64_t(12))}); +} + +TEST(Copy, Gather2Dto3D) +{ + legate::Core::perform_registration(); + test_gather(GatherSpec{{100, 100, 100}, {10, 10}, legate::Scalar(7.0)}); +} + +} // namespace copy_gather diff --git a/tests/cpp/integration/copy_gather_scatter.cc b/tests/cpp/integration/copy_gather_scatter.cc new file mode 100644 index 0000000000..db59fbcfe4 --- /dev/null +++ b/tests/cpp/integration/copy_gather_scatter.cc @@ -0,0 +1,228 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "copy_util.inl" +#include "legate.h" + +namespace copy_gather_scatter { + +static const char* library_name = "test_copy_gather_scatter"; +static legate::Logger logger(library_name); + +constexpr int32_t CHECK_GATHER_SCATTER_TASK = FILL_INDIRECT_TASK + TEST_MAX_DIM * TEST_MAX_DIM; + +template +struct CheckGatherScatterTask + : public legate::LegateTask> { + struct CheckGatherScatterTaskBody { + template + void operator()(legate::TaskContext& context) + { + using VAL = legate::legate_type_of; + + auto& src_store = context.inputs().at(0); + auto& tgt_store = context.inputs().at(1); + auto& src_ind_store = context.inputs().at(2); + auto& tgt_ind_store = context.inputs().at(3); + auto init = context.scalars().at(0).value(); + + auto src_shape = src_store.shape(); + auto tgt_shape = tgt_store.shape(); + auto ind_shape = src_ind_store.shape(); + + legate::Buffer mask(tgt_shape, legate::Memory::Kind::SYSTEM_MEM); + for (legate::PointInRectIterator it(tgt_shape); it.valid(); ++it) mask[*it] = false; + + auto src_acc = src_store.read_accessor(); + auto tgt_acc = tgt_store.read_accessor(); + auto src_ind_acc = src_ind_store.read_accessor, IND_DIM>(); + auto tgt_ind_acc = tgt_ind_store.read_accessor, IND_DIM>(); + + for (legate::PointInRectIterator it(ind_shape); it.valid(); ++it) { + auto src_point = src_ind_acc[*it]; + auto tgt_point = tgt_ind_acc[*it]; + auto source = src_acc[src_point]; + auto copy = tgt_acc[tgt_point]; + EXPECT_EQ(copy, source); + mask[tgt_point] = true; + } + + for (legate::PointInRectIterator it(tgt_shape); it.valid(); ++it) { + auto p = *it; + if (mask[p]) continue; + EXPECT_EQ(tgt_acc[p], init); + } + } + }; + + static const int32_t TASK_ID = CHECK_GATHER_SCATTER_TASK + SRC_DIM * TEST_MAX_DIM * TEST_MAX_DIM + + IND_DIM * TEST_MAX_DIM + TGT_DIM; + static void cpu_variant(legate::TaskContext& context) + { + auto type_code = context.inputs().at(0).type().code; + type_dispatch_for_test(type_code, CheckGatherScatterTaskBody{}, context); + } +}; + +struct GatherScatterSpec { + std::vector src_shape; + std::vector ind_shape; + std::vector tgt_shape; + legate::Scalar seed; + legate::Scalar init; + + std::string to_string() const + { + std::stringstream ss; + ss << "source shape: " << ::to_string(src_shape) + << ", indirection shape: " << ::to_string(ind_shape) + << ", target shape: " << ::to_string(tgt_shape) + << ", data type: " << seed.type().to_string(); + return std::move(ss).str(); + } +}; + +void register_tasks() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + FillTask<1>::register_variants(context); + FillTask<2>::register_variants(context); + FillTask<3>::register_variants(context); + + FillIndirectTask<1, 1>::register_variants(context); + FillIndirectTask<1, 2>::register_variants(context); + FillIndirectTask<1, 3>::register_variants(context); + FillIndirectTask<2, 1>::register_variants(context); + FillIndirectTask<2, 2>::register_variants(context); + FillIndirectTask<2, 3>::register_variants(context); + FillIndirectTask<3, 1>::register_variants(context); + FillIndirectTask<3, 2>::register_variants(context); + FillIndirectTask<3, 3>::register_variants(context); + + CheckGatherScatterTask<1, 2, 3>::register_variants(context); + CheckGatherScatterTask<2, 3, 1>::register_variants(context); + CheckGatherScatterTask<3, 1, 2>::register_variants(context); + CheckGatherScatterTask<3, 3, 3>::register_variants(context); + CheckGatherScatterTask<2, 2, 3>::register_variants(context); +} + +void check_gather_scatter_output(legate::LibraryContext* context, + const legate::LogicalStore& src, + const legate::LogicalStore& tgt, + const legate::LogicalStore& src_ind, + const legate::LogicalStore& tgt_ind, + const legate::Scalar& init) +{ + auto runtime = legate::Runtime::get_runtime(); + auto machine = runtime->get_machine(); + + int32_t task_id = CHECK_GATHER_SCATTER_TASK + src.dim() * TEST_MAX_DIM * TEST_MAX_DIM + + src_ind.dim() * TEST_MAX_DIM + tgt.dim(); + + auto task = runtime->create_task(context, task_id); + + auto src_part = task->declare_partition(); + auto tgt_part = task->declare_partition(); + auto src_ind_part = task->declare_partition(); + auto tgt_ind_part = task->declare_partition(); + task->add_input(src, src_part); + task->add_input(tgt, tgt_part); + task->add_input(src_ind, src_ind_part); + task->add_input(tgt_ind, tgt_ind_part); + task->add_scalar_arg(init); + + task->add_constraint(legate::broadcast(src_part, legate::from_range(src.dim()))); + task->add_constraint(legate::broadcast(tgt_part, legate::from_range(tgt.dim()))); + task->add_constraint(legate::broadcast(src_ind_part, legate::from_range(src_ind.dim()))); + task->add_constraint(legate::broadcast(tgt_ind_part, legate::from_range(tgt_ind.dim()))); + + runtime->submit(std::move(task)); +} + +void test_gather_scatter(const GatherScatterSpec& spec) +{ + assert(spec.seed.type() == spec.init.type()); + logger.print() << "Gather-scatter Copy: " << spec.to_string(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + auto& type = spec.seed.type(); + auto src = runtime->create_store(spec.src_shape, type.clone()); + auto tgt = runtime->create_store(spec.tgt_shape, type.clone()); + auto src_ind = runtime->create_store(spec.ind_shape, legate::point_type(spec.src_shape.size())); + auto tgt_ind = runtime->create_store(spec.ind_shape, legate::point_type(spec.tgt_shape.size())); + + fill_input(context, src, spec.seed); + fill_indirect(context, src_ind, src); + fill_indirect(context, tgt_ind, tgt); + runtime->issue_fill(context, tgt, spec.init); + + auto copy = runtime->create_copy(context); + copy->add_input(src); + copy->add_output(tgt); + copy->add_source_indirect(src_ind); + copy->add_target_indirect(tgt_ind); + runtime->submit(std::move(copy)); + + check_gather_scatter_output(context, src, tgt, src_ind, tgt_ind, spec.init); +} + +TEST(Copy, GatherScatter1Dto3Dvia2D) +{ + legate::Core::perform_registration(); + std::vector shape1d{5}; + test_gather_scatter(GatherScatterSpec{ + shape1d, {7, 11}, {10, 10, 10}, legate::Scalar(int64_t(123)), legate::Scalar(int64_t(42))}); +} + +TEST(Copy, GatherScatter2Dto1Dvia3D) +{ + legate::Core::perform_registration(); + std::vector shape1d{1000}; + test_gather_scatter(GatherScatterSpec{ + {3, 7}, {3, 6, 5}, shape1d, legate::Scalar(uint32_t(456)), legate::Scalar(uint32_t(42))}); +} + +TEST(Copy, GatherScatter3Dto2Dvia1D) +{ + legate::Core::perform_registration(); + std::vector shape1d{100}; + test_gather_scatter(GatherScatterSpec{ + {4, 5, 2}, shape1d, {50, 50}, legate::Scalar(int64_t(12)), legate::Scalar(int64_t(42))}); +} + +TEST(Copy, GatherScatter3Dto3Dvia3D) +{ + legate::Core::perform_registration(); + test_gather_scatter(GatherScatterSpec{{10, 10, 10}, + {5, 4, 2}, + {10, 10, 10}, + legate::Scalar(int64_t(1)), + legate::Scalar(int64_t(42))}); +} + +TEST(Copy, GatherScatter2Dto3Dvia2D) +{ + legate::Core::perform_registration(); + test_gather_scatter(GatherScatterSpec{ + {27, 33}, {11, 7}, {132, 121, 3}, legate::Scalar(int64_t(2)), legate::Scalar(int64_t(84))}); +} + +} // namespace copy_gather_scatter diff --git a/tests/cpp/integration/copy_normal.cc b/tests/cpp/integration/copy_normal.cc new file mode 100644 index 0000000000..9c0d0e3f84 --- /dev/null +++ b/tests/cpp/integration/copy_normal.cc @@ -0,0 +1,163 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "legate.h" + +#include "copy_util.inl" + +namespace copy_normal { + +static const char* library_name = "test_copy_normal"; + +static constexpr int32_t TEST_MAX_DIM = 3; + +constexpr int32_t CHECK_TASK = FILL_TASK + TEST_MAX_DIM; + +template +struct CheckTask : public legate::LegateTask> { + struct CheckTaskBody { + template + void operator()(legate::Store& source, legate::Store& target, legate::Rect& shape) + { + using VAL = legate::legate_type_of; + auto src = source.read_accessor(shape); + auto tgt = target.read_accessor(shape); + for (legate::PointInRectIterator it(shape); it.valid(); ++it) { + EXPECT_EQ(src[*it], tgt[*it]); + } + } + }; + + static const int32_t TASK_ID = CHECK_TASK + DIM; + static void cpu_variant(legate::TaskContext& context) + { + auto& source = context.inputs().at(0); + auto& target = context.inputs().at(1); + auto shape = source.shape(); + + if (shape.empty()) return; + + type_dispatch(source.type().code, CheckTaskBody{}, source, target, shape); + } +}; + +void register_tasks() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + FillTask<1>::register_variants(context); + FillTask<2>::register_variants(context); + FillTask<3>::register_variants(context); + CheckTask<1>::register_variants(context); + CheckTask<2>::register_variants(context); + CheckTask<3>::register_variants(context); +} + +void check_output(legate::LibraryContext* context, + const legate::LogicalStore& src, + const legate::LogicalStore& tgt) +{ + auto runtime = legate::Runtime::get_runtime(); + auto machine = runtime->get_machine(); + auto task = runtime->create_task(context, CHECK_TASK + tgt.dim()); + + auto src_part = task->declare_partition(); + auto tgt_part = task->declare_partition(); + + task->add_input(src, src_part); + task->add_input(tgt, tgt_part); + task->add_constraint(legate::align(src_part, tgt_part)); + + runtime->submit(std::move(task)); +} + +const auto uint32 = legate::uint32(); +const auto int64 = legate::int64(); +const auto float64 = legate::float64(); + +struct NormalCopySpec { + std::vector shape; + const legate::Type& type; + legate::Scalar seed; +}; + +void test_normal_copies(const std::vector specs) +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + std::vector inputs; + std::vector outputs; + + for (auto& [shape, type, seed] : specs) { + auto input = runtime->create_store(shape, type.clone()); + fill_input(context, input, seed); + inputs.push_back(std::move(input)); + outputs.push_back(runtime->create_store(shape, type.clone())); + } + + auto copy = runtime->create_copy(context); + for (auto& input : inputs) copy->add_input(input); + for (auto& output : outputs) copy->add_output(output); + runtime->submit(std::move(copy)); + + // check the result of copy + for (uint32_t idx = 0; idx < outputs.size(); ++idx) { + auto& output = outputs[idx]; + auto& input = inputs.at(idx); + check_output(context, input, output); + } +} + +void test_all_normal_copies() +{ + test_normal_copies({{{1000, 100}, *uint32, legate::Scalar(uint32_t(3))}}); +} + +TEST(Copy, Single) +{ + legate::Core::perform_registration(); + // For some reason, clang-format gets tripped over by singleton initialization lists, + // so factor out the definition here + std::vector shape1d{9}; + test_normal_copies({{{4, 7}, *int64, legate::Scalar(int64_t(12))}}); + test_normal_copies({{{1000, 100}, *uint32, legate::Scalar(uint32_t(3))}}); +} + +TEST(Copy, Multi2) +{ + legate::Core::perform_registration(); + std::vector shape1d{13}; + test_normal_copies({ + {shape1d, *float64, legate::Scalar(double(5.0))}, + {{3, 7, 5}, *uint32, legate::Scalar(uint32_t(456))}, + }); +} + +TEST(Copy, Multi3) +{ + legate::Core::perform_registration(); + std::vector shape1d{11}; + test_normal_copies({ + {shape1d, *int64, legate::Scalar(int64_t(789))}, + {{5, 5}, *float64, legate::Scalar(double(10.0))}, + {{2, 4, 4}, *uint32, legate::Scalar(uint32_t(7))}, + }); +} + +} // namespace copy_normal diff --git a/tests/cpp/integration/copy_scatter.cc b/tests/cpp/integration/copy_scatter.cc new file mode 100644 index 0000000000..2a74731e6d --- /dev/null +++ b/tests/cpp/integration/copy_scatter.cc @@ -0,0 +1,214 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "copy_util.inl" +#include "legate.h" + +namespace copy_scatter { + +static const char* library_name = "test_copy_scatter"; +static legate::Logger logger(library_name); + +constexpr int32_t CHECK_SCATTER_TASK = FILL_INDIRECT_TASK + TEST_MAX_DIM * TEST_MAX_DIM; + +template +struct CheckScatterTask : public legate::LegateTask> { + struct CheckScatterTaskBody { + template + void operator()(legate::TaskContext& context) + { + using VAL = legate::legate_type_of; + + auto& src_store = context.inputs().at(0); + auto& tgt_store = context.inputs().at(1); + auto& ind_store = context.inputs().at(2); + auto init = context.scalars().at(0).value(); + + auto ind_shape = ind_store.shape(); + if (ind_shape.empty()) return; + + auto tgt_shape = tgt_store.shape(); + + legate::Buffer mask(tgt_shape, legate::Memory::Kind::SYSTEM_MEM); + for (legate::PointInRectIterator it(tgt_shape); it.valid(); ++it) mask[*it] = false; + + auto src_acc = src_store.read_accessor(); + auto tgt_acc = tgt_store.read_accessor(); + auto ind_acc = ind_store.read_accessor, IND_DIM>(); + + for (legate::PointInRectIterator it(ind_shape); it.valid(); ++it) { + auto p = ind_acc[*it]; + auto copy = tgt_acc[p]; + auto source = src_acc[*it]; + EXPECT_EQ(copy, source); + mask[p] = true; + } + + for (legate::PointInRectIterator it(tgt_shape); it.valid(); ++it) { + auto p = *it; + if (mask[p]) continue; + EXPECT_EQ(tgt_acc[p], init); + } + } + }; + + static const int32_t TASK_ID = CHECK_SCATTER_TASK + IND_DIM * TEST_MAX_DIM + TGT_DIM; + static void cpu_variant(legate::TaskContext& context) + { + auto type_code = context.inputs().at(0).type().code; + type_dispatch_for_test(type_code, CheckScatterTaskBody{}, context); + } +}; + +void register_tasks() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + FillTask<1>::register_variants(context); + FillTask<2>::register_variants(context); + FillTask<3>::register_variants(context); + + // XXX: Tasks unused by the test cases are commented out + // FillIndirectTask<1, 1>::register_variants(context); + FillIndirectTask<1, 2>::register_variants(context); + // FillIndirectTask<1, 3>::register_variants(context); + // FillIndirectTask<2, 1>::register_variants(context); + FillIndirectTask<2, 2>::register_variants(context); + FillIndirectTask<2, 3>::register_variants(context); + FillIndirectTask<3, 1>::register_variants(context); + FillIndirectTask<3, 2>::register_variants(context); + // FillIndirectTask<3, 3>::register_variants(context); + + // CheckScatterTask<1, 1>::register_variants(context); + CheckScatterTask<1, 2>::register_variants(context); + // CheckScatterTask<1, 3>::register_variants(context); + // CheckScatterTask<2, 1>::register_variants(context); + CheckScatterTask<2, 2>::register_variants(context); + CheckScatterTask<2, 3>::register_variants(context); + CheckScatterTask<3, 1>::register_variants(context); + CheckScatterTask<3, 2>::register_variants(context); + // CheckScatterTask<3, 3>::register_variants(context); +} + +struct ScatterSpec { + std::vector ind_shape; + std::vector data_shape; + legate::Scalar seed; + legate::Scalar init; + + std::string to_string() const + { + std::stringstream ss; + auto print_shape = [&](auto& shape) { + ss << "("; + for (auto& ext : shape) ss << ext << ","; + ss << ")"; + }; + ss << "indirection/source shape: " << ::to_string(ind_shape) + << ", target shape: " << ::to_string(data_shape); + ss << ", data type: " << seed.type().to_string(); + return std::move(ss).str(); + } +}; + +void check_scatter_output(legate::LibraryContext* context, + const legate::LogicalStore& src, + const legate::LogicalStore& tgt, + const legate::LogicalStore& ind, + const legate::Scalar& init) +{ + auto runtime = legate::Runtime::get_runtime(); + auto machine = runtime->get_machine(); + + int32_t task_id = CHECK_SCATTER_TASK + ind.dim() * TEST_MAX_DIM + tgt.dim(); + + auto task = runtime->create_task(context, task_id); + + auto src_part = task->declare_partition(); + auto tgt_part = task->declare_partition(); + auto ind_part = task->declare_partition(); + task->add_input(src, src_part); + task->add_input(tgt, tgt_part); + task->add_input(ind, ind_part); + task->add_scalar_arg(init); + + task->add_constraint(legate::broadcast(src_part, legate::from_range(src.dim()))); + task->add_constraint(legate::broadcast(tgt_part, legate::from_range(tgt.dim()))); + task->add_constraint(legate::broadcast(ind_part, legate::from_range(ind.dim()))); + + runtime->submit(std::move(task)); +} + +void test_scatter(const ScatterSpec& spec) +{ + assert(spec.seed.type() == spec.init.type()); + logger.print() << "Scatter Copy: " << spec.to_string(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + auto& type = spec.seed.type(); + auto src = runtime->create_store(spec.ind_shape, type.clone()); + auto tgt = runtime->create_store(spec.data_shape, type.clone()); + auto ind = runtime->create_store(spec.ind_shape, legate::point_type(spec.data_shape.size())); + + fill_input(context, src, spec.seed); + fill_indirect(context, ind, tgt); + runtime->issue_fill(context, tgt, spec.init); + + auto copy = runtime->create_copy(context); + copy->add_input(src); + copy->add_output(tgt); + copy->add_target_indirect(ind); + runtime->submit(std::move(copy)); + + check_scatter_output(context, src, tgt, ind, spec.init); +} + +// Note that the volume of indirection field should be smaller than that of the target to avoid +// duplicate updates on the same element, whose semantics is undefined. +TEST(Copy, Scatter1Dto2D) +{ + legate::Core::perform_registration(); + std::vector shape1d{5}; + test_scatter( + ScatterSpec{shape1d, {7, 11}, legate::Scalar(int64_t(123)), legate::Scalar(int64_t(42))}); +} + +TEST(Copy, Scatter2Dto3D) +{ + legate::Core::perform_registration(); + test_scatter( + ScatterSpec{{3, 7}, {3, 6, 5}, legate::Scalar(uint32_t(456)), legate::Scalar(uint32_t(42))}); +} + +TEST(Copy, Scatter2Dto2D) +{ + legate::Core::perform_registration(); + test_scatter( + ScatterSpec{{4, 5}, {10, 11}, legate::Scalar(int64_t(12)), legate::Scalar(int64_t(42))}); +} + +TEST(Copy, Scatter3Dto2D) +{ + legate::Core::perform_registration(); + test_scatter( + ScatterSpec{{10, 10, 10}, {200, 200}, legate::Scalar(int64_t(1)), legate::Scalar(int64_t(42))}); +} + +} // namespace copy_scatter diff --git a/tests/cpp/integration/copy_util.inl b/tests/cpp/integration/copy_util.inl new file mode 100644 index 0000000000..e7183221e7 --- /dev/null +++ b/tests/cpp/integration/copy_util.inl @@ -0,0 +1,179 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/data/detail/logical_store.h" +#include "legate.h" + +namespace { + +constexpr int32_t TEST_MAX_DIM = 3; + +// Instantiate only the cases exercised in copy tests +template +constexpr decltype(auto) type_dispatch_for_test(legate::Type::Code code, + Functor f, + Fnargs&&... args) +{ + switch (code) { + case legate::Type::Code::INT64: { + return f.template operator()(std::forward(args)...); + } + case legate::Type::Code::UINT32: { + return f.template operator()(std::forward(args)...); + } + case legate::Type::Code::FLOAT64: { + return f.template operator()(std::forward(args)...); + } + default: break; + } + assert(false); + return f.template operator()(std::forward(args)...); +} + +enum TaskIDs { + FILL_TASK = 0, + FILL_INDIRECT_TASK = FILL_TASK + TEST_MAX_DIM, +}; + +template +struct FillTask : public legate::LegateTask> { + struct FillTaskBody { + template + void operator()(legate::Store& output, legate::Rect& shape, legate::Scalar& seed) + { + using VAL = legate::legate_type_of; + auto acc = output.write_accessor(shape); + auto to_fill = seed.value(); + size_t i = 1; + for (legate::PointInRectIterator it(shape); it.valid(); ++it, ++i) + acc[*it] = i * to_fill; + } + }; + + static const int32_t TASK_ID = FILL_TASK + DIM; + static void cpu_variant(legate::TaskContext& context) + { + auto& output = context.outputs().at(0); + auto shape = output.shape(); + auto& seed = context.scalars().at(0); + + if (shape.empty()) return; + + type_dispatch_for_test(output.type().code, FillTaskBody{}, output, shape, seed); + } +}; + +template +legate::Point delinearize(size_t index, + const legate::Point& extents, + const legate::Point& lo) +{ + legate::Point result; + for (int32_t dim = 0; dim < DIM; ++dim) { + result[dim] = index % extents[dim]; + index = index / extents[dim]; + } + return result + lo; +} + +template +struct FillIndirectTask : public legate::LegateTask> { + static const int32_t TASK_ID = FILL_INDIRECT_TASK + IND_DIM * TEST_MAX_DIM + DATA_DIM; + static void cpu_variant(legate::TaskContext& context) + { + auto& output = context.outputs().at(0); + auto ind_shape = output.shape(); + auto data_shape = context.scalars().at(0).value>(); + + size_t data_vol = data_shape.volume(); + size_t ind_vol = ind_shape.volume(); + + if (0 == ind_vol) return; + + auto data_extents = data_shape.hi - data_shape.lo + legate::Point::ONES(); + auto acc = output.write_accessor, IND_DIM>(ind_shape); + auto idx = 0; + for (legate::PointInRectIterator it(ind_shape); it.valid(); ++it) { + auto p = delinearize((data_vol * idx++ / ind_vol) % data_vol, data_extents, data_shape.lo); + acc[*it] = p; + } + } +}; + +void fill_input(legate::LibraryContext* context, + const legate::LogicalStore& src, + const legate::Scalar& value) +{ + auto runtime = legate::Runtime::get_runtime(); + auto machine = runtime->get_machine(); + auto task = runtime->create_task(context, FILL_TASK + src.dim()); + task->add_output(src, task->declare_partition()); + task->add_scalar_arg(value); + runtime->submit(std::move(task)); +} + +void fill_indirect(legate::LibraryContext* context, + const legate::LogicalStore& ind, + const legate::LogicalStore& data) +{ + auto runtime = legate::Runtime::get_runtime(); + auto machine = runtime->get_machine(); + int32_t task_id = FILL_INDIRECT_TASK + ind.dim() * TEST_MAX_DIM + data.dim(); + auto task = runtime->create_task(context, task_id); + auto part = task->declare_partition(); + task->add_output(ind, part); + // Technically indirection fields for gather coipes can have repeated points + // and thus be initialized in parallel, but we always serialize the + // initialization to simplify the logic + task->add_constraint(legate::broadcast(part, legate::from_range(ind.dim()))); + auto domain = legate::to_domain(data.extents()); + switch (data.dim()) { + case 1: { + task->add_scalar_arg(legate::Rect<1>(domain)); + break; + } + case 2: { + task->add_scalar_arg(legate::Rect<2>(domain)); + break; + } + case 3: { + task->add_scalar_arg(legate::Rect<3>(domain)); + break; + } + default: { + assert(false); + break; + } + } + runtime->submit(std::move(task)); + // Important to reset the indirection field's key partition. Here's why: multi-dimensional point + // types have bigger sizes than the value types used in copy tests, the partitioner will favor the + // indirection field's key partition over others. Since indirection fields are initialized by + // sequential tasks in this function, they would end up always serializing downstream indirect + // copies, which is undesirable. + ind.impl()->reset_key_partition(); +} + +std::string to_string(const std::vector& shape) +{ + std::stringstream ss; + ss << "("; + for (auto& ext : shape) ss << ext << ","; + ss << ")"; + return std::move(ss).str(); +} + +} // namespace diff --git a/tests/cpp/integration/fill.cc b/tests/cpp/integration/fill.cc new file mode 100644 index 0000000000..7cbf957b32 --- /dev/null +++ b/tests/cpp/integration/fill.cc @@ -0,0 +1,194 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "legate.h" + +namespace fill { + +static const char* library_name = "test_fill"; + +constexpr size_t SIZE = 10; + +enum TaskIDs { + CHECK_TASK = 0, + CHECK_SLICE_TASK = 3, +}; + +template +struct CheckTask : public legate::LegateTask> { + static const int32_t TASK_ID = CHECK_TASK + DIM; + static void cpu_variant(legate::TaskContext& context); +}; + +template +struct CheckSliceTask : public legate::LegateTask> { + static const int32_t TASK_ID = CHECK_SLICE_TASK + DIM; + static void cpu_variant(legate::TaskContext& context); +}; + +void register_tasks() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + CheckTask<1>::register_variants(context); + CheckTask<2>::register_variants(context); + CheckTask<3>::register_variants(context); + CheckSliceTask<1>::register_variants(context); + CheckSliceTask<2>::register_variants(context); + CheckSliceTask<3>::register_variants(context); +} + +template +/*static*/ void CheckTask::cpu_variant(legate::TaskContext& context) +{ + auto& input = context.inputs().at(0); + auto shape = input.shape(); + int64_t value = context.scalars().at(0).value(); + + if (shape.empty()) return; + + auto acc = input.write_accessor(shape); + for (legate::PointInRectIterator it(shape); it.valid(); ++it) { EXPECT_EQ(acc[*it], value); } +} + +template +/*static*/ void CheckSliceTask::cpu_variant(legate::TaskContext& context) +{ + auto& input = context.inputs().at(0); + auto shape = input.shape(); + int64_t value_in_slice = context.scalars().at(0).value(); + int64_t value_outside_slice = context.scalars().at(1).value(); + int64_t offset = context.scalars().at(2).value(); + + if (shape.empty()) return; + + auto acc = input.write_accessor(shape); + auto in_slice = [&offset](const auto& p) { + for (int32_t dim = 0; dim < DIM; ++dim) + if (p[dim] < offset) return false; + return true; + }; + for (legate::PointInRectIterator it(shape); it.valid(); ++it) { + EXPECT_EQ(acc[*it], in_slice(*it) ? value_in_slice : value_outside_slice); + } +} + +void check_output(legate::LogicalStore store, int64_t value) +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + auto task = runtime->create_task(context, CHECK_TASK + store.dim()); + auto part = task->declare_partition(); + task->add_input(store, part); + task->add_scalar_arg(value); + runtime->submit(std::move(task)); +} + +void check_output_slice(legate::LogicalStore store, + int64_t value_in_slice, + int64_t value_outside_slice, + int64_t offset) +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + auto task = runtime->create_task(context, CHECK_SLICE_TASK + store.dim()); + auto part = task->declare_partition(); + task->add_input(store, part); + task->add_scalar_arg(value_in_slice); + task->add_scalar_arg(value_outside_slice); + task->add_scalar_arg(offset); + runtime->submit(std::move(task)); +} + +void test_fill_index(int32_t dim) +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + int64_t v = 10; + + auto lhs = runtime->create_store(legate::Shape(dim, SIZE), legate::int64()); + auto value = runtime->create_store(v); + + // fill input store with some values + runtime->issue_fill(context, lhs, value); + + // check the result of fill + check_output(lhs, v); +} + +void test_fill_single(int32_t dim) +{ + auto runtime = legate::Runtime::get_runtime(); + auto machine = runtime->get_machine(); + legate::MachineTracker tracker(machine.slice(0, 1, legate::mapping::TaskTarget::CPU)); + test_fill_index(dim); +} + +void test_fill_slice(int32_t dim) +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + int64_t v1 = 100; + int64_t v2 = 200; + int64_t offset = 3; + + auto lhs = runtime->create_store(legate::Shape(dim, SIZE), legate::int64()); + auto value_in_slice = runtime->create_store(v1); + auto value_outside_slice = runtime->create_store(v2); + + // First fill the entire store with v1 + runtime->issue_fill(context, lhs, value_outside_slice); + + // Then fill a slice with v2 + auto sliced = lhs; + for (int32_t idx = 0; idx < dim; ++idx) sliced = sliced.slice(idx, legate::Slice(offset)); + runtime->issue_fill(context, sliced, value_in_slice); + + // check if the slice is filled correctly + check_output_slice(lhs, v1, v2, offset); +} + +TEST(Integration, FillIndex) +{ + legate::Core::perform_registration(); + test_fill_index(1); + test_fill_index(2); + test_fill_index(3); +} + +TEST(Integration, FillSingle) +{ + legate::Core::perform_registration(); + test_fill_single(1); + test_fill_single(2); + test_fill_single(3); +} + +TEST(Integration, FillSlice) +{ + legate::Core::perform_registration(); + test_fill_slice(1); + test_fill_slice(2); + test_fill_slice(3); +} + +} // namespace fill diff --git a/tests/cpp/integration/machine_scope.cc b/tests/cpp/integration/machine_scope.cc index 7bdad1e8c8..3ab058d990 100644 --- a/tests/cpp/integration/machine_scope.cc +++ b/tests/cpp/integration/machine_scope.cc @@ -16,7 +16,6 @@ #include -#include "core/mapping/mapping.h" #include "legate.h" namespace machine_scope { diff --git a/tests/cpp/integration/manual_simple.cc b/tests/cpp/integration/manual_simple.cc index e765aa417b..6f77389139 100644 --- a/tests/cpp/integration/manual_simple.cc +++ b/tests/cpp/integration/manual_simple.cc @@ -16,7 +16,6 @@ #include -#include "core/mapping/mapping.h" #include "legate.h" #include "tasks/task_simple.h" diff --git a/tests/cpp/integration/multi_scalar_out.cc b/tests/cpp/integration/multi_scalar_out.cc index f12aa8c7a1..4d5e435b71 100644 --- a/tests/cpp/integration/multi_scalar_out.cc +++ b/tests/cpp/integration/multi_scalar_out.cc @@ -16,7 +16,6 @@ #include -#include "core/mapping/mapping.h" #include "legate.h" #include "tasks/task_simple.h" From 2a9853bde9c6a78f112e470c81fa4c4b74adad89 Mon Sep 17 00:00:00 2001 From: Parag Kulkarni Date: Fri, 9 Jun 2023 22:08:37 -0700 Subject: [PATCH 0144/1425] Add rule to trigger the pipeline. * Add rule to trigger the pipeline when: - Merge request is made. - When run on legate/legate.core.internal See merge request legate/legate.core.internal!51 --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e1c62e0396..4dbbbc9580 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,6 +7,9 @@ stages: Build:LegateCore: stage: build + rules: + - if: ($CI_PIPELINE_SOURCE == "merge_request_event" && $CI_PROJECT_PATH == "legate/legate.core.internal") + tags: - cpu_only - gpu/disabled From 58ea73d6a854c775271f311a53807213b6a7caf4 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 14 Jun 2023 22:18:58 -0700 Subject: [PATCH 0145/1425] Dynamic sharding functor generation * Missing include * Make sure we convert storage partitions only when we can * Modernize code with structured binding * Don't generate sharding functors on a single node * Start generating sharding functors dynamically * Fix incorrect local node id initialization * String serialization for debugging See merge request legate/legate.core.internal!52 --- src/core/data/detail/logical_store.cc | 2 +- src/core/mapping/machine.cc | 27 ++++++++++++-- src/core/mapping/machine.h | 9 +++++ src/core/partitioning/partition.h | 4 +++ src/core/partitioning/partitioner.cc | 11 ++++++ src/core/partitioning/partitioner.h | 4 +++ src/core/runtime/copy.cc | 4 ++- src/core/runtime/detail/copy_launcher.cc | 7 ++-- src/core/runtime/detail/copy_launcher.h | 4 ++- src/core/runtime/detail/fill_launcher.cc | 14 ++++---- src/core/runtime/detail/fill_launcher.h | 5 +-- src/core/runtime/detail/req_analyzer.cc | 39 +++++++------------- src/core/runtime/detail/runtime.cc | 45 +++++++++++++++++++++++- src/core/runtime/detail/runtime.h | 9 ++++- src/core/runtime/detail/task_launcher.cc | 6 ++-- src/core/runtime/detail/task_launcher.h | 1 + src/core/runtime/task.cc | 4 ++- 17 files changed, 148 insertions(+), 47 deletions(-) diff --git a/src/core/data/detail/logical_store.cc b/src/core/data/detail/logical_store.cc index 9e0f8bebe5..3a2dbc1fb9 100644 --- a/src/core/data/detail/logical_store.cc +++ b/src/core/data/detail/logical_store.cc @@ -586,7 +586,7 @@ std::shared_ptr LogicalStore::find_or_create_key_partition( storage_part = storage_->find_key_partition(machine, transform_->invert(restrictions)); std::unique_ptr store_part = nullptr; - if (nullptr == storage_part) { + if (nullptr == storage_part || (!transform_->identity() && !storage_part->is_convertible())) { auto part_mgr = Runtime::get_runtime()->partition_manager(); auto launch_shape = part_mgr->compute_launch_shape(machine, restrictions, extents_); if (launch_shape.empty()) diff --git a/src/core/mapping/machine.cc b/src/core/mapping/machine.cc index 85e866a7fc..184408ecb7 100644 --- a/src/core/mapping/machine.cc +++ b/src/core/mapping/machine.cc @@ -17,6 +17,8 @@ #include "core/mapping/machine.h" #include "core/utilities/buffer_builder.h" +#include "realm/network.h" + namespace legate::mapping { TaskTarget to_target(Processor::Kind kind) @@ -146,6 +148,12 @@ void ProcessorRange::pack(BufferBuilder& buffer) const buffer.pack(per_node_count); } +std::ostream& operator<<(std::ostream& stream, const ProcessorRange& range) +{ + stream << range.to_string(); + return stream; +} + /////////////////////////////////////////// // legate::mapping::MachineDesc ////////////////////////////////////////// @@ -317,12 +325,27 @@ const Processor& LocalProcessorRange::operator[](uint32_t idx) const return procs_[local_idx]; } +std::string LocalProcessorRange::to_string() const +{ + std::stringstream ss; + ss << "{offset: " << offset_ << ", total processor count: " << total_proc_count_ + << ", processors: "; + for (uint32_t idx = 0; idx < procs_.size(); ++idx) ss << procs_[idx] << ","; + ss << "}"; + return std::move(ss).str(); +} + +std::ostream& operator<<(std::ostream& stream, const LocalProcessorRange& range) +{ + stream << range.to_string(); + return stream; +} + ////////////////////////////// // legate::mapping::Machine ////////////////////////////// Machine::Machine(Legion::Machine legion_machine) - : local_node(Processor::get_executing_processor().address_space()), - total_nodes(legion_machine.get_address_space_count()) + : local_node(Realm::Network::my_node_id), total_nodes(legion_machine.get_address_space_count()) { Legion::Machine::ProcessorQuery procs(legion_machine); // Query to find all our local processors diff --git a/src/core/mapping/machine.h b/src/core/mapping/machine.h index 233890c6fc..f6be311740 100644 --- a/src/core/mapping/machine.h +++ b/src/core/mapping/machine.h @@ -36,6 +36,8 @@ Processor::Kind to_kind(TaskTarget target); LegateVariantCode to_variant_code(TaskTarget target); +std::ostream& operator<<(std::ostream& stream, const TaskTarget& target); + /** * @ingroup mapping * @brief A class to represent a range of processors. @@ -114,6 +116,8 @@ struct ProcessorRange { uint32_t per_node_count{1}; }; +std::ostream& operator<<(std::ostream& stream, const ProcessorRange& range); + /** * @ingroup mapping * @brief Machine descriptor class @@ -324,12 +328,17 @@ class LocalProcessorRange { public: bool empty() const { return procs_.size() == 0; } + public: + std::string to_string() const; + private: uint32_t offset_; uint32_t total_proc_count_; Span procs_; }; +std::ostream& operator<<(std::ostream& stream, const LocalProcessorRange& info); + class Machine { public: Machine(Legion::Machine legion_machine); diff --git a/src/core/partitioning/partition.h b/src/core/partitioning/partition.h index 6ce5e1e72c..939232ce86 100644 --- a/src/core/partitioning/partition.h +++ b/src/core/partitioning/partition.h @@ -50,6 +50,7 @@ struct Partition { virtual bool is_complete_for(const detail::Storage* storage) const = 0; virtual bool is_disjoint_for(const Domain* launch_domain) const = 0; virtual bool satisfies_restrictions(const Restrictions& restrictions) const = 0; + virtual bool is_convertible() const = 0; public: virtual Legion::LogicalPartition construct(Legion::LogicalRegion region, @@ -80,6 +81,7 @@ class NoPartition : public Partition { bool is_complete_for(const detail::Storage* storage) const override; bool is_disjoint_for(const Domain* launch_domain) const override; bool satisfies_restrictions(const Restrictions& restrictions) const override; + bool is_convertible() const override { return true; } public: Legion::LogicalPartition construct(Legion::LogicalRegion region, bool complete) const override; @@ -120,6 +122,7 @@ class Tiling : public Partition { bool is_complete_for(const detail::Storage* storage) const override; bool is_disjoint_for(const Domain* launch_domain) const override; bool satisfies_restrictions(const Restrictions& restrictions) const override; + bool is_convertible() const override { return true; } public: Legion::LogicalPartition construct(Legion::LogicalRegion region, bool complete) const override; @@ -167,6 +170,7 @@ class Weighted : public Partition { bool is_complete_for(const detail::Storage* storage) const override; bool is_disjoint_for(const Domain* launch_domain) const override; bool satisfies_restrictions(const Restrictions& restrictions) const override; + bool is_convertible() const override { return false; } public: Legion::LogicalPartition construct(Legion::LogicalRegion region, bool complete) const override; diff --git a/src/core/partitioning/partitioner.cc b/src/core/partitioning/partitioner.cc index e0d6912eb8..a4544aef43 100644 --- a/src/core/partitioning/partitioner.cc +++ b/src/core/partitioning/partitioner.cc @@ -160,6 +160,11 @@ const Legion::FieldSpace& Strategy::find_field_space(const Variable* partition_s return finder->second; } +bool Strategy::is_key_partition(const Variable* partition_symbol) const +{ + return key_partition_.has_value() && key_partition_.value() == partition_symbol; +} + void Strategy::dump() const { log_legate.debug("===== Solution ====="); @@ -200,6 +205,11 @@ void Strategy::compute_launch_domains(const ConstraintSolver& solver) launch_domains_[op] = domain_resolver.resolve_launch_domain(); } +void Strategy::record_key_partition(const Variable* partition_symbol) +{ + if (!key_partition_) key_partition_ = partition_symbol; +} + //////////////////////////////////////////////////// // legate::detail::Partitioner //////////////////////////////////////////////////// @@ -253,6 +263,7 @@ std::unique_ptr Partitioner::partition_stores() auto* op = part_symb->operation(); auto store = op->find_store(part_symb); auto partition = store->find_or_create_key_partition(op->machine(), restrictions); + strategy->record_key_partition(part_symb); #ifdef DEBUG_LEGATE assert(partition != nullptr); #endif diff --git a/src/core/partitioning/partitioner.h b/src/core/partitioning/partitioner.h index 387076738d..1f9c9873e1 100644 --- a/src/core/partitioning/partitioner.h +++ b/src/core/partitioning/partitioner.h @@ -17,6 +17,7 @@ #pragma once #include +#include #include #include "core/data/shape.h" @@ -55,17 +56,20 @@ class Strategy { bool has_assignment(const Variable* partition_symbol) const; std::shared_ptr operator[](const Variable* partition_symbol) const; const Legion::FieldSpace& find_field_space(const Variable* partition_symbol) const; + bool is_key_partition(const Variable* partition_symbol) const; public: void dump() const; private: void compute_launch_domains(const ConstraintSolver& solver); + void record_key_partition(const Variable* partition_symbol); private: std::map> assignments_{}; std::map field_spaces_{}; std::map> launch_domains_{}; + std::optional key_partition_{}; }; class Partitioner { diff --git a/src/core/runtime/copy.cc b/src/core/runtime/copy.cc index 6619f04555..28664b7fe9 100644 --- a/src/core/runtime/copy.cc +++ b/src/core/runtime/copy.cc @@ -107,7 +107,9 @@ void Copy::launch(detail::Strategy* p_strategy) auto create_projection_info = [&strategy, &launch_domain](auto& store, auto& var) { auto store_partition = store->create_partition(strategy[var]); - return store_partition->create_projection_info(launch_domain); + auto proj_info = store_partition->create_projection_info(launch_domain); + proj_info->tag = strategy.is_key_partition(var) ? LEGATE_CORE_KEY_STORE_TAG : 0; + return std::move(proj_info); }; for (auto& [store, var] : inputs_) launcher.add_input(store, create_projection_info(store, var)); diff --git a/src/core/runtime/detail/copy_launcher.cc b/src/core/runtime/detail/copy_launcher.cc index 2354e48a75..e552dc2d93 100644 --- a/src/core/runtime/detail/copy_launcher.cc +++ b/src/core/runtime/detail/copy_launcher.cc @@ -86,12 +86,13 @@ CopyLauncher::CopyLauncher(LibraryContext* library, bool target_indirect_out_of_range, int64_t tag) : library_(library), + machine_(machine), source_indirect_out_of_range_(source_indirect_out_of_range), target_indirect_out_of_range_(target_indirect_out_of_range), tag_(tag) { mapper_arg_ = new BufferBuilder(); - machine.pack(*mapper_arg_); + machine_.pack(*mapper_arg_); } CopyLauncher::~CopyLauncher() @@ -114,6 +115,7 @@ void CopyLauncher::add_store(std::vector& args, auto region_field = store->get_region_field(); auto region = region_field->region(); auto field_id = region_field->field_id(); + if (LEGATE_CORE_KEY_STORE_TAG == proj_info->tag) key_proj_id_ = proj_info->proj_id; args.push_back(new CopyArg(req_idx, store, field_id, privilege, std::move(proj_info))); } @@ -168,8 +170,7 @@ void CopyLauncher::execute_single() void CopyLauncher::pack_sharding_functor_id() { - // TODO: Generate the right sharding functor id - mapper_arg_->pack(0); + mapper_arg_->pack(Runtime::get_runtime()->get_sharding(machine_, key_proj_id_)); } void CopyLauncher::pack_args() diff --git a/src/core/runtime/detail/copy_launcher.h b/src/core/runtime/detail/copy_launcher.h index f5f1c57252..cc4adb14de 100644 --- a/src/core/runtime/detail/copy_launcher.h +++ b/src/core/runtime/detail/copy_launcher.h @@ -19,7 +19,7 @@ #include #include -#include "legion.h" +#include "core/mapping/machine.h" namespace legate { class BufferBuilder; @@ -82,6 +82,8 @@ class CopyLauncher { private: LibraryContext* library_; int64_t tag_; + mapping::MachineDesc machine_; + Legion::ProjectionID key_proj_id_{0}; private: BufferBuilder* mapper_arg_; diff --git a/src/core/runtime/detail/fill_launcher.cc b/src/core/runtime/detail/fill_launcher.cc index 83c9b79402..9ead9a4338 100644 --- a/src/core/runtime/detail/fill_launcher.cc +++ b/src/core/runtime/detail/fill_launcher.cc @@ -27,10 +27,8 @@ namespace legate::detail { FillLauncher::FillLauncher(LibraryContext* library, const mapping::MachineDesc& machine, int64_t tag) - : library_(library), tag_(tag) + : library_(library), tag_(tag), machine_(machine), mapper_arg_(new BufferBuilder()) { - mapper_arg_ = new BufferBuilder(); - machine.pack(*mapper_arg_); } FillLauncher::~FillLauncher() { delete mapper_arg_; } @@ -52,7 +50,11 @@ void FillLauncher::launch_single(LogicalStore* lhs, return Runtime::get_runtime()->dispatch(legion_fill_launcher.get()); } -void FillLauncher::pack_args() { mapper_arg_->pack(0); } +void FillLauncher::pack_mapper_arg(Legion::ProjectionID proj_id) +{ + machine_.pack(*mapper_arg_); + mapper_arg_->pack(Runtime::get_runtime()->get_sharding(machine_, proj_id)); +} std::unique_ptr FillLauncher::build_index_fill( const Legion::Domain& launch_domain, @@ -60,7 +62,7 @@ std::unique_ptr FillLauncher::build_index_fill( const ProjectionInfo& lhs_proj, LogicalStore* value) { - pack_args(); + pack_mapper_arg(lhs_proj.proj_id); auto* runtime = Runtime::get_runtime(); auto& provenance = runtime->provenance_manager()->get_provenance(); auto lhs_region_field = lhs->get_region_field(); @@ -86,7 +88,7 @@ std::unique_ptr FillLauncher::build_index_fill( std::unique_ptr FillLauncher::build_single_fill( LogicalStore* lhs, const ProjectionInfo& lhs_proj, LogicalStore* value) { - pack_args(); + pack_mapper_arg(lhs_proj.proj_id); auto* runtime = Runtime::get_runtime(); auto& provenance = runtime->provenance_manager()->get_provenance(); auto lhs_region_field = lhs->get_region_field(); diff --git a/src/core/runtime/detail/fill_launcher.h b/src/core/runtime/detail/fill_launcher.h index 071241ad24..6ec79df7f3 100644 --- a/src/core/runtime/detail/fill_launcher.h +++ b/src/core/runtime/detail/fill_launcher.h @@ -18,7 +18,7 @@ #include -#include "legion.h" +#include "core/mapping/machine.h" namespace legate { class BufferBuilder; @@ -47,7 +47,7 @@ class FillLauncher { void launch_single(LogicalStore* lhs, const ProjectionInfo& lhs_proj, LogicalStore* value); private: - void pack_args(); + void pack_mapper_arg(Legion::ProjectionID proj_id); std::unique_ptr build_index_fill(const Legion::Domain& launch_domain, LogicalStore* lhs, const ProjectionInfo& lhs_proj, @@ -59,6 +59,7 @@ class FillLauncher { private: LibraryContext* library_; int64_t tag_; + mapping::MachineDesc machine_; private: BufferBuilder* mapper_arg_; diff --git a/src/core/runtime/detail/req_analyzer.cc b/src/core/runtime/detail/req_analyzer.cc index c38e88abc4..314eee17e3 100644 --- a/src/core/runtime/detail/req_analyzer.cc +++ b/src/core/runtime/detail/req_analyzer.cc @@ -63,13 +63,12 @@ uint32_t FieldSet::get_requirement_index(Legion::PrivilegeMode privilege, void FieldSet::coalesce() { - for (const auto& entry : field_projs_) { - const auto& proj_set = entry.second; + for (const auto& [field_id, proj_set] : field_projs_) { for (const auto& proj_info : proj_set.proj_infos) - coalesced_[Key(proj_set.privilege, proj_info)].push_back(entry.first); + coalesced_[Key(proj_set.privilege, proj_info)].push_back(field_id); } uint32_t idx = 0; - for (const auto& entry : coalesced_) req_indices_[entry.first] = idx++; + for (const auto& [key, _] : coalesced_) req_indices_[key] = idx++; } namespace { @@ -86,11 +85,8 @@ constexpr bool is_single = false; template void FieldSet::populate_launcher(Launcher* task, const Legion::LogicalRegion& region) const { - for (auto& entry : coalesced_) { - auto privilege = entry.first.first; - const auto& proj_info = entry.first.second; - const auto& fields = entry.second; - + for (auto& [key, fields] : coalesced_) { + auto& [privilege, proj_info] = key; task->region_requirements.push_back(Legion::RegionRequirement()); auto& requirement = task->region_requirements.back(); proj_info.template populate_requirement>( @@ -120,18 +116,15 @@ uint32_t RequirementAnalyzer::get_requirement_index(const Legion::LogicalRegion& #ifdef DEBUG_LEGATE assert(finder != field_sets_.end()); #endif - auto& field_set = finder->second.first; - auto& req_offset = finder->second.second; + auto& [field_set, req_offset] = finder->second; return req_offset + field_set.get_requirement_index(privilege, proj_info); } void RequirementAnalyzer::analyze_requirements() { uint32_t num_reqs = 0; - for (auto& entry : field_sets_) { - auto& field_set = entry.second.first; - auto& req_offset = entry.second.second; - + for (auto& [_, entry] : field_sets_) { + auto& [field_set, req_offset] = entry; field_set.coalesce(); req_offset = num_reqs; num_reqs += field_set.num_requirements(); @@ -151,10 +144,7 @@ void RequirementAnalyzer::populate_launcher(Legion::TaskLauncher* task) const template void RequirementAnalyzer::_populate_launcher(Launcher* task) const { - for (auto& entry : field_sets_) { - const auto& field_set = entry.second.first; - field_set.populate_launcher(task, entry.first); - } + for (auto& [region, entry] : field_sets_) entry.first.populate_launcher(task, region); } //////////////////////////// @@ -189,18 +179,15 @@ uint32_t OutputRequirementAnalyzer::get_requirement_index(const Legion::FieldSpa void OutputRequirementAnalyzer::analyze_requirements() { uint32_t idx = 0; - for (auto& entry : field_groups_) req_infos_[entry.first].req_idx = idx++; + for (const auto& [field_space, _] : field_groups_) req_infos_[field_space].req_idx = idx++; } void OutputRequirementAnalyzer::populate_output_requirements( std::vector& out_reqs) const { - for (auto& entry : field_groups_) { - auto finder = req_infos_.find(entry.first); -#ifdef DEBUG_LEGATE - assert(finder != req_infos_.end()); -#endif - out_reqs.emplace_back(entry.first, entry.second, finder->second.dim, true); + for (auto& [field_space, fields] : field_groups_) { + auto& [dim, _] = req_infos_.at(field_space); + out_reqs.emplace_back(field_space, fields, dim, true /*global_indexing*/); } } diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index a182e464c7..8476576f07 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -28,6 +28,8 @@ #include "core/runtime/shard.h" #include "env_defaults.h" +#include "realm/network.h" + namespace legate { Logger log_legate("legate"); @@ -702,6 +704,44 @@ Legion::ProjectionID Runtime::get_delinearizing_projection() return core_context_->get_projection_id(LEGATE_CORE_DELINEARIZE_PROJ_ID); } +Legion::ShardingID Runtime::get_sharding(const mapping::MachineDesc& machine, + Legion::ProjectionID proj_id) +{ + // If we're running on a single node, we don't need to generate sharding functors + if (Realm::Network::max_node_id == 0) return 0; + + auto& proc_range = machine.processor_range(); + auto [low, high] = proc_range.get_node_range(); + auto offset = proc_range.low % proc_range.per_node_count; + ShardingDesc key{proj_id, low, high, offset, proc_range.per_node_count}; + +#ifdef DEBUG_LEGATE + log_legate.debug() << "Query sharding {proj_id: " << proj_id + << ", processor range: " << proc_range + << ", processor type: " << machine.preferred_target << "}"; +#endif + + auto finder = registered_shardings_.find(key); + if (finder != registered_shardings_.end()) { +#ifdef DEBUG_LEGATE + log_legate.debug() << "Found sharding " << finder->second; +#endif + return finder->second; + } + + auto sharding_id = core_context_->get_sharding_id(next_sharding_id_++); + registered_shardings_.insert({key, sharding_id}); + +#ifdef DEBUG_LEGATE + log_legate.debug() << "Create sharding " << sharding_id; +#endif + + legate_create_sharding_functor_using_projection( + sharding_id, proj_id, low, high, offset, proc_range.per_node_count); + + return sharding_id; +} + CommunicatorManager* Runtime::communicator_manager() const { return communicator_manager_; } MachineManager* Runtime::machine_manager() const { return machine_manager_; } @@ -755,7 +795,10 @@ int32_t Runtime::finish() void Runtime::destroy() { - // Make sure all Legate ops finish + // Flush any outstanding operations before we tear down the runtime + flush_scheduling_window(); + + // Then also make sure all the operations finish issue_execution_fence(true); // Clean up resources used by Legate diff --git a/src/core/runtime/detail/runtime.h b/src/core/runtime/detail/runtime.h index a2487719b9..444daf4785 100644 --- a/src/core/runtime/detail/runtime.h +++ b/src/core/runtime/detail/runtime.h @@ -177,6 +177,8 @@ class Runtime { public: Legion::ProjectionID get_projection(int32_t src_ndim, const proj::SymbolicPoint& point); Legion::ProjectionID get_delinearizing_projection(); + Legion::ShardingID get_sharding(const mapping::MachineDesc& machine, + Legion::ProjectionID proj_id); private: void schedule(std::vector> operations); @@ -209,7 +211,12 @@ class Runtime { private: using ProjectionDesc = std::pair; int64_t next_projection_id_{LEGATE_CORE_FIRST_DYNAMIC_FUNCTOR_ID}; - std::map registered_projections_; + std::map registered_projections_{}; + + private: + using ShardingDesc = std::tuple; + int64_t next_sharding_id_{LEGATE_CORE_FIRST_DYNAMIC_FUNCTOR_ID}; + std::map registered_shardings_{}; private: std::vector> operations_; diff --git a/src/core/runtime/detail/task_launcher.cc b/src/core/runtime/detail/task_launcher.cc index 6ed99361ce..d238dbb778 100644 --- a/src/core/runtime/detail/task_launcher.cc +++ b/src/core/runtime/detail/task_launcher.cc @@ -24,6 +24,7 @@ #include "core/runtime/detail/partition_manager.h" #include "core/runtime/detail/req_analyzer.h" #include "core/runtime/detail/runtime.h" +#include "core/runtime/shard.h" #include "core/utilities/buffer_builder.h" namespace legate::detail { @@ -161,6 +162,8 @@ void TaskLauncher::add_store(std::vector& args, auto field_id = region_field->field_id(); req_analyzer_->insert(region, field_id, privilege, *proj_info); + // Keep the projection functor id of the key store + if (LEGATE_CORE_KEY_STORE_TAG == proj_info->tag) key_proj_id_ = proj_info->proj_id; args.push_back( new RegionFieldArg(req_analyzer_, store, field_id, privilege, std::move(proj_info))); } @@ -174,8 +177,7 @@ void TaskLauncher::pack_args(const std::vector& args) void TaskLauncher::pack_sharding_functor_id() { - // TODO: Generate the right sharding functor id - mapper_arg_->pack(0); + mapper_arg_->pack(Runtime::get_runtime()->get_sharding(machine_, key_proj_id_)); } std::unique_ptr TaskLauncher::build_single_task() diff --git a/src/core/runtime/detail/task_launcher.h b/src/core/runtime/detail/task_launcher.h index 032f1a1c91..bdcd2ff50e 100644 --- a/src/core/runtime/detail/task_launcher.h +++ b/src/core/runtime/detail/task_launcher.h @@ -105,6 +105,7 @@ class TaskLauncher { int64_t tag_; mapping::MachineDesc machine_; std::string provenance_; + Legion::ProjectionID key_proj_id_{0}; private: bool has_side_effect_{true}; diff --git a/src/core/runtime/task.cc b/src/core/runtime/task.cc index 6abd35b290..a97e4558a3 100644 --- a/src/core/runtime/task.cc +++ b/src/core/runtime/task.cc @@ -107,7 +107,9 @@ void Task::launch(detail::Strategy* p_strategy) auto create_projection_info = [&strategy, &launch_domain](auto& store, auto& var) { auto store_partition = store->create_partition(strategy[var]); - return store_partition->create_projection_info(launch_domain); + auto proj_info = store_partition->create_projection_info(launch_domain); + proj_info->tag = strategy.is_key_partition(var) ? LEGATE_CORE_KEY_STORE_TAG : 0; + return std::move(proj_info); }; // Add input stores From b6b485f19fe084f3481b7ed61c9af872bb4dd881 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Thu, 22 Jun 2023 10:41:00 -0700 Subject: [PATCH 0146/1425] Fix the shutdown procedure * Fix the shutdown procedure See merge request legate/legate.core.internal!60 --- src/core/runtime/detail/runtime.cc | 36 +++++++++++++++--------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index 8476576f07..551de895fe 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -798,33 +798,33 @@ void Runtime::destroy() // Flush any outstanding operations before we tear down the runtime flush_scheduling_window(); - // Then also make sure all the operations finish + // Need a fence to make sure all client operations come before the subsequent clean-up tasks + issue_execution_fence(); + + // Destroy all communicators. This will likely launch some tasks for the clean-up. + communicator_manager_->destroy(); + + // Destroy all Legion handles used by Legate + for (auto& [_, region_manager] : region_managers_) region_manager->destroy(true /*unordered*/); + for (auto& [_, index_space] : index_spaces_) + legion_runtime_->destroy_index_space(legion_context_, index_space, true /*unordered*/); + index_spaces_.clear(); + + // We're about to deallocate objects below, so let's block on all outstanding Legion operations issue_execution_fence(true); - // Clean up resources used by Legate + // We finally deallocate managers for (auto& [_, context] : libraries_) delete context; libraries_.clear(); + for (auto& [_, region_manager] : region_managers_) delete region_manager; + region_managers_.clear(); + for (auto& [_, field_manager] : field_managers_) delete field_manager; + field_managers_.clear(); - communicator_manager_->destroy(); delete communicator_manager_; delete machine_manager_; delete partition_manager_; delete provenance_manager_; - - for (auto& [_, region_manager] : region_managers_) { - region_manager->destroy(true /*unordered*/); - delete region_manager; - } - region_managers_.clear(); - - for (auto& [_, field_manager] : field_managers_) delete field_manager; - field_managers_.clear(); - - for (auto& [_, index_space] : index_spaces_) - legion_runtime_->destroy_index_space(legion_context_, index_space, true /*unordered*/); - index_spaces_.clear(); - - issue_execution_fence(); } static void extract_scalar_task( From 5c5c2f2781bb685f6749bfb4e1cbe7e4cd844c37 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Thu, 22 Jun 2023 23:57:22 -0700 Subject: [PATCH 0147/1425] Remove unnecessary context arguments from the API * More clean up for library context arguments * Remove context arguments from the APIs that don't need them See merge request legate/legate.core.internal!62 --- src/core/data/detail/logical_store.cc | 8 ++-- src/core/data/detail/logical_store.h | 4 +- src/core/data/logical_store.cc | 5 +- src/core/data/logical_store.h | 4 +- src/core/runtime/copy.cc | 6 +-- src/core/runtime/detail/copy_launcher.cc | 36 +++++++-------- src/core/runtime/detail/copy_launcher.h | 9 +--- src/core/runtime/detail/fill.cc | 10 ++-- src/core/runtime/detail/fill.h | 6 +-- src/core/runtime/detail/fill_launcher.cc | 48 ++++++++++---------- src/core/runtime/detail/fill_launcher.h | 5 +- src/core/runtime/detail/runtime.cc | 15 +++--- src/core/runtime/detail/runtime.h | 7 +-- src/core/runtime/operation.h | 12 ++--- src/core/runtime/runtime.cc | 14 ++---- src/core/runtime/runtime.h | 10 ++-- src/core/runtime/task.cc | 13 +++--- tests/cpp/integration/copy_failure.cc | 12 ++--- tests/cpp/integration/copy_gather.cc | 2 +- tests/cpp/integration/copy_gather_scatter.cc | 4 +- tests/cpp/integration/copy_normal.cc | 2 +- tests/cpp/integration/copy_scatter.cc | 4 +- tests/cpp/integration/fill.cc | 6 +-- tests/cpp/integration/inline_map.cc | 12 ++--- tests/cpp/integration/manual_simple.cc | 8 ++-- tests/cpp/integration/multi_scalar_out.cc | 14 +++--- 26 files changed, 120 insertions(+), 156 deletions(-) diff --git a/src/core/data/detail/logical_store.cc b/src/core/data/detail/logical_store.cc index 3a2dbc1fb9..0a7116f9fd 100644 --- a/src/core/data/detail/logical_store.cc +++ b/src/core/data/detail/logical_store.cc @@ -194,12 +194,12 @@ void Storage::set_region_field(std::shared_ptr&& region_fiel void Storage::set_future(Legion::Future future) { future_ = future; } -RegionField Storage::map(LibraryContext* context) +RegionField Storage::map() { #ifdef DEBUG_LEGATE assert(Kind::REGION_FIELD == kind_); #endif - return Runtime::get_runtime()->map_region_field(context, get_region_field()); + return Runtime::get_runtime()->map_region_field(get_region_field()); } Restrictions Storage::compute_restrictions() const @@ -523,7 +523,7 @@ std::shared_ptr LogicalStore::delinearize(int32_t idx, std::vector return std::make_shared(std::move(new_extents), storage_, std::move(transform)); } -std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) +std::shared_ptr LogicalStore::get_physical_store() { if (unbound()) { throw std::invalid_argument("Unbound store cannot be inlined mapped"); } if (nullptr != mapped_) return mapped_; @@ -539,7 +539,7 @@ std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) #ifdef DEBUG_LEGATE assert(storage_->kind() == Storage::Kind::REGION_FIELD); #endif - auto region_field = storage_->map(context); + auto region_field = storage_->map(); mapped_ = std::make_shared(dim(), type().clone(), -1, std::move(region_field), transform_); return mapped_; } diff --git a/src/core/data/detail/logical_store.h b/src/core/data/detail/logical_store.h index 7758584b71..f5d3bb23cf 100644 --- a/src/core/data/detail/logical_store.h +++ b/src/core/data/detail/logical_store.h @@ -77,7 +77,7 @@ class Storage : public std::enable_shared_from_this { void set_future(Legion::Future future); public: - RegionField map(LibraryContext* context); + RegionField map(); public: Restrictions compute_restrictions() const; @@ -201,7 +201,7 @@ class LogicalStore : public std::enable_shared_from_this { std::shared_ptr partition_by_tiling(Shape tile_shape); public: - std::shared_ptr get_physical_store(LibraryContext* context); + std::shared_ptr get_physical_store(); public: Restrictions compute_restrictions() const; diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index d06c78a4f0..d72a273d42 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -73,10 +73,7 @@ LogicalStore LogicalStore::delinearize(int32_t dim, std::vector&& sizes return LogicalStore(impl_->delinearize(dim, std::move(sizes))); } -std::shared_ptr LogicalStore::get_physical_store(LibraryContext* context) -{ - return impl_->get_physical_store(context); -} +std::shared_ptr LogicalStore::get_physical_store() { return impl_->get_physical_store(); } void LogicalStore::set_key_partition(const mapping::MachineDesc& machine, const Partition* partition) diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index 663165c173..cfaa087905 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -328,11 +328,9 @@ class LogicalStore { * This call blocks the client's control flow. And it fetches the data for the whole store on * a single node. * - * @param context Library within which the physical store is created - * * @return A physical store of the logical store */ - std::shared_ptr get_physical_store(LibraryContext* context); + std::shared_ptr get_physical_store(); public: void set_key_partition(const mapping::MachineDesc& machine, const Partition* partition); diff --git a/src/core/runtime/copy.cc b/src/core/runtime/copy.cc index 28664b7fe9..3f2818c5e5 100644 --- a/src/core/runtime/copy.cc +++ b/src/core/runtime/copy.cc @@ -28,8 +28,8 @@ namespace legate { -Copy::Copy(LibraryContext* library, int64_t unique_id, mapping::MachineDesc&& machine) - : Operation(library, unique_id, std::move(machine)) +Copy::Copy(int64_t unique_id, mapping::MachineDesc&& machine) + : Operation(unique_id, std::move(machine)) { } @@ -102,7 +102,7 @@ void Copy::launch(detail::Strategy* p_strategy) { auto& strategy = *p_strategy; detail::CopyLauncher launcher( - library_, machine_, source_indirect_out_of_range_, target_indirect_out_of_range_); + machine_, source_indirect_out_of_range_, target_indirect_out_of_range_); auto launch_domain = strategy.launch_domain(this); auto create_projection_info = [&strategy, &launch_domain](auto& store, auto& var) { diff --git a/src/core/runtime/detail/copy_launcher.cc b/src/core/runtime/detail/copy_launcher.cc index e552dc2d93..ca8ccc383d 100644 --- a/src/core/runtime/detail/copy_launcher.cc +++ b/src/core/runtime/detail/copy_launcher.cc @@ -80,16 +80,14 @@ void CopyArg::pack(BufferBuilder& buffer) const buffer.pack(field_id_); } -CopyLauncher::CopyLauncher(LibraryContext* library, - const mapping::MachineDesc& machine, +CopyLauncher::CopyLauncher(const mapping::MachineDesc& machine, bool source_indirect_out_of_range, bool target_indirect_out_of_range, int64_t tag) - : library_(library), - machine_(machine), + : machine_(machine), + tag_(tag), source_indirect_out_of_range_(source_indirect_out_of_range), - target_indirect_out_of_range_(target_indirect_out_of_range), - tag_(tag) + target_indirect_out_of_range_(target_indirect_out_of_range) { mapper_arg_ = new BufferBuilder(); machine_.pack(*mapper_arg_); @@ -104,8 +102,6 @@ CopyLauncher::~CopyLauncher() for (auto& arg : target_indirect_) delete arg; } -int64_t CopyLauncher::legion_mapper_id() const { return library_->get_mapper_id(); } - void CopyLauncher::add_store(std::vector& args, detail::LogicalStore* store, std::unique_ptr proj_info, @@ -228,12 +224,13 @@ std::unique_ptr CopyLauncher::build_index_copy( pack_args(); auto* runtime = Runtime::get_runtime(); auto& provenance = runtime->provenance_manager()->get_provenance(); - auto index_copy = std::make_unique(launch_domain, - Legion::Predicate::TRUE_PRED, - legion_mapper_id(), - tag_, - mapper_arg_->to_legion_buffer(), - provenance.c_str()); + auto index_copy = + std::make_unique(launch_domain, + Legion::Predicate::TRUE_PRED, + runtime->core_context()->get_mapper_id(), + tag_, + mapper_arg_->to_legion_buffer(), + provenance.c_str()); populate_copy(index_copy.get()); return std::move(index_copy); @@ -244,11 +241,12 @@ std::unique_ptr CopyLauncher::build_single_copy() pack_args(); auto* runtime = Runtime::get_runtime(); auto& provenance = runtime->provenance_manager()->get_provenance(); - auto single_copy = std::make_unique(Legion::Predicate::TRUE_PRED, - legion_mapper_id(), - tag_, - mapper_arg_->to_legion_buffer(), - provenance.c_str()); + auto single_copy = + std::make_unique(Legion::Predicate::TRUE_PRED, + runtime->core_context()->get_mapper_id(), + tag_, + mapper_arg_->to_legion_buffer(), + provenance.c_str()); populate_copy(single_copy.get()); return std::move(single_copy); diff --git a/src/core/runtime/detail/copy_launcher.h b/src/core/runtime/detail/copy_launcher.h index cc4adb14de..10b79df385 100644 --- a/src/core/runtime/detail/copy_launcher.h +++ b/src/core/runtime/detail/copy_launcher.h @@ -41,16 +41,12 @@ class RequirementAnalyzer; class CopyLauncher { public: - CopyLauncher(LibraryContext* library, - const mapping::MachineDesc& machine, + CopyLauncher(const mapping::MachineDesc& machine, bool source_indirect_out_of_range, bool target_indirect_out_of_range, int64_t tag = 0); ~CopyLauncher(); - public: - int64_t legion_mapper_id() const; - public: void add_input(detail::LogicalStore* store, std::unique_ptr proj_info); void add_output(detail::LogicalStore* store, std::unique_ptr proj_info); @@ -80,9 +76,8 @@ class CopyLauncher { void populate_copy(Launcher* launcher); private: - LibraryContext* library_; - int64_t tag_; mapping::MachineDesc machine_; + int64_t tag_; Legion::ProjectionID key_proj_id_{0}; private: diff --git a/src/core/runtime/detail/fill.cc b/src/core/runtime/detail/fill.cc index 48b7220734..8edc84cdd2 100644 --- a/src/core/runtime/detail/fill.cc +++ b/src/core/runtime/detail/fill.cc @@ -25,12 +25,8 @@ namespace legate { -Fill::Fill(LibraryContext* library, - LogicalStore lhs, - LogicalStore value, - int64_t unique_id, - mapping::MachineDesc&& machine) - : Operation(library, unique_id, std::move(machine)), +Fill::Fill(LogicalStore lhs, LogicalStore value, int64_t unique_id, mapping::MachineDesc&& machine) + : Operation(unique_id, std::move(machine)), lhs_var_(declare_partition()), lhs_(lhs.impl()), value_(value.impl()) @@ -45,7 +41,7 @@ Fill::Fill(LibraryContext* library, void Fill::launch(detail::Strategy* strategy) { - detail::FillLauncher launcher(library_, machine_); + detail::FillLauncher launcher(machine_); auto launch_domain = strategy->launch_domain(this); auto part = (*strategy)[lhs_var_]; auto lhs_proj = lhs_->create_partition(part)->create_projection_info(launch_domain); diff --git a/src/core/runtime/detail/fill.h b/src/core/runtime/detail/fill.h index 1a40db578a..4d506f88e4 100644 --- a/src/core/runtime/detail/fill.h +++ b/src/core/runtime/detail/fill.h @@ -23,11 +23,7 @@ namespace legate { class Fill : public Operation { private: friend class detail::Runtime; - Fill(LibraryContext* library, - LogicalStore lhs, - LogicalStore value, - int64_t unique_id, - mapping::MachineDesc&& machine); + Fill(LogicalStore lhs, LogicalStore value, int64_t unique_id, mapping::MachineDesc&& machine); public: void launch(detail::Strategy* strategy) override; diff --git a/src/core/runtime/detail/fill_launcher.cc b/src/core/runtime/detail/fill_launcher.cc index 9ead9a4338..9231f67631 100644 --- a/src/core/runtime/detail/fill_launcher.cc +++ b/src/core/runtime/detail/fill_launcher.cc @@ -24,10 +24,8 @@ namespace legate::detail { -FillLauncher::FillLauncher(LibraryContext* library, - const mapping::MachineDesc& machine, - int64_t tag) - : library_(library), tag_(tag), machine_(machine), mapper_arg_(new BufferBuilder()) +FillLauncher::FillLauncher(const mapping::MachineDesc& machine, int64_t tag) + : machine_(machine), tag_(tag), mapper_arg_(new BufferBuilder()) { } @@ -69,17 +67,18 @@ std::unique_ptr FillLauncher::build_index_fill( auto lhs_region = lhs_region_field->region(); auto field_id = lhs_region_field->field_id(); auto future_value = value->get_future(); - auto lhs_parent = Runtime::get_runtime()->find_parent_region(lhs_region); - auto index_fill = std::make_unique(launch_domain, - lhs_proj.partition, - lhs_parent, - future_value, - lhs_proj.proj_id, - Legion::Predicate::TRUE_PRED, - library_->get_mapper_id(), - lhs_proj.tag, - mapper_arg_->to_legion_buffer(), - provenance.c_str()); + auto lhs_parent = runtime->find_parent_region(lhs_region); + auto index_fill = + std::make_unique(launch_domain, + lhs_proj.partition, + lhs_parent, + future_value, + lhs_proj.proj_id, + Legion::Predicate::TRUE_PRED, + runtime->core_context()->get_mapper_id(), + lhs_proj.tag, + mapper_arg_->to_legion_buffer(), + provenance.c_str()); index_fill->add_field(field_id); return std::move(index_fill); @@ -95,15 +94,16 @@ std::unique_ptr FillLauncher::build_single_fill( auto lhs_region = lhs_region_field->region(); auto field_id = lhs_region_field->field_id(); auto future_value = value->get_future(); - auto lhs_parent = Runtime::get_runtime()->find_parent_region(lhs_region); - auto single_fill = std::make_unique(lhs_region, - lhs_parent, - future_value, - Legion::Predicate::TRUE_PRED, - library_->get_mapper_id(), - lhs_proj.tag, - mapper_arg_->to_legion_buffer(), - provenance.c_str()); + auto lhs_parent = runtime->find_parent_region(lhs_region); + auto single_fill = + std::make_unique(lhs_region, + lhs_parent, + future_value, + Legion::Predicate::TRUE_PRED, + runtime->core_context()->get_mapper_id(), + lhs_proj.tag, + mapper_arg_->to_legion_buffer(), + provenance.c_str()); single_fill->add_field(field_id); return std::move(single_fill); diff --git a/src/core/runtime/detail/fill_launcher.h b/src/core/runtime/detail/fill_launcher.h index 6ec79df7f3..585ef698d7 100644 --- a/src/core/runtime/detail/fill_launcher.h +++ b/src/core/runtime/detail/fill_launcher.h @@ -36,7 +36,7 @@ class ProjectionInfo; class FillLauncher { public: - FillLauncher(LibraryContext* library, const mapping::MachineDesc& machine, int64_t tag = 0); + FillLauncher(const mapping::MachineDesc& machine, int64_t tag = 0); ~FillLauncher(); public: @@ -57,9 +57,8 @@ class FillLauncher { LogicalStore* value); private: - LibraryContext* library_; - int64_t tag_; mapping::MachineDesc machine_; + int64_t tag_; private: BufferBuilder* mapper_arg_; diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index 9c7e6682d4..10e0503022 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -179,20 +179,17 @@ std::unique_ptr Runtime::create_task(LibraryContext* library, return std::unique_ptr(task); } -std::unique_ptr Runtime::create_copy(LibraryContext* library) +std::unique_ptr Runtime::create_copy() { auto machine = machine_manager_->get_machine(); - auto copy = new Copy(library, next_unique_id_++, std::move(machine)); + auto copy = new Copy(next_unique_id_++, std::move(machine)); return std::unique_ptr(copy); } -void Runtime::issue_fill(LibraryContext* library, - legate::LogicalStore lhs, - legate::LogicalStore value) +void Runtime::issue_fill(legate::LogicalStore lhs, legate::LogicalStore value) { auto machine = machine_manager_->get_machine(); - submit( - std::unique_ptr(new Fill(library, lhs, value, next_unique_id_++, std::move(machine)))); + submit(std::unique_ptr(new Fill(lhs, value, next_unique_id_++, std::move(machine)))); } void Runtime::flush_scheduling_window() @@ -314,7 +311,7 @@ std::shared_ptr Runtime::import_region_field(Legion::Logical return fld_mgr->import_field(region, field_id); } -RegionField Runtime::map_region_field(LibraryContext* context, const LogicalRegionField* rf) +RegionField Runtime::map_region_field(const LogicalRegionField* rf) { auto root_region = rf->get_root().region(); auto field_id = rf->field_id(); @@ -327,7 +324,7 @@ RegionField Runtime::map_region_field(LibraryContext* context, const LogicalRegi Legion::RegionRequirement req(root_region, READ_WRITE, EXCLUSIVE, root_region); req.add_field(field_id); - auto mapper_id = context->get_mapper_id(); + auto mapper_id = core_context_->get_mapper_id(); // TODO: We need to pass the metadata about logical store Legion::InlineLauncher launcher(req, mapper_id); pr = legion_runtime_->map_region(legion_context_, launcher); diff --git a/src/core/runtime/detail/runtime.h b/src/core/runtime/detail/runtime.h index 444daf4785..22fdc589db 100644 --- a/src/core/runtime/detail/runtime.h +++ b/src/core/runtime/detail/runtime.h @@ -72,8 +72,8 @@ class Runtime { std::unique_ptr create_task(LibraryContext* library, int64_t task_id, const Shape& launch_shape); - std::unique_ptr create_copy(LibraryContext* library); - void issue_fill(LibraryContext* library, legate::LogicalStore lhs, legate::LogicalStore value); + std::unique_ptr create_copy(); + void issue_fill(legate::LogicalStore lhs, legate::LogicalStore value); void flush_scheduling_window(); void submit(std::unique_ptr op); @@ -101,7 +101,7 @@ class Runtime { std::shared_ptr import_region_field(Legion::LogicalRegion region, Legion::FieldID field_id, uint32_t field_size); - RegionField map_region_field(LibraryContext* context, const LogicalRegionField* region_field); + RegionField map_region_field(const LogicalRegionField* region_field); void unmap_physical_region(Legion::PhysicalRegion pr); size_t num_inline_mapped() const; @@ -189,6 +189,7 @@ class Runtime { bool initialized() const { return initialized_; } void destroy(); int32_t finish(); + const LibraryContext* core_context() const { return core_context_; } private: bool initialized_{false}; diff --git a/src/core/runtime/operation.h b/src/core/runtime/operation.h index eb6daf52a1..0f4d5e71ca 100644 --- a/src/core/runtime/operation.h +++ b/src/core/runtime/operation.h @@ -53,7 +53,7 @@ class Scalar; class Operation { protected: using StoreArg = std::pair; - Operation(LibraryContext* library, uint64_t unique_id, mapping::MachineDesc&& machine); + Operation(uint64_t unique_id, mapping::MachineDesc&& machine); public: virtual ~Operation() {} @@ -91,7 +91,6 @@ class Operation { void record_partition(const Variable* variable, std::shared_ptr store); protected: - LibraryContext* library_; uint64_t unique_id_; protected: @@ -116,7 +115,7 @@ class Operation { */ class Task : public Operation { protected: - Task(LibraryContext* library, + Task(const LibraryContext* library, int64_t task_id, uint64_t unique_id, mapping::MachineDesc&& machine); @@ -179,6 +178,7 @@ class Task : public Operation { std::string to_string() const override; protected: + const LibraryContext* library_; int64_t task_id_; bool concurrent_{false}; bool has_side_effect_{false}; @@ -197,7 +197,7 @@ class Task : public Operation { class AutoTask : public Task { public: friend class detail::Runtime; - AutoTask(LibraryContext* library, + AutoTask(const LibraryContext* library, int64_t task_id, uint64_t unique_id, mapping::MachineDesc&& machine); @@ -265,7 +265,7 @@ class AutoTask : public Task { class ManualTask : public Task { private: friend class detail::Runtime; - ManualTask(LibraryContext* library, + ManualTask(const LibraryContext* library, int64_t task_id, const Shape& launch_shape, uint64_t unique_id, @@ -340,7 +340,7 @@ class ManualTask : public Task { class Copy : public Operation { private: friend class detail::Runtime; - Copy(LibraryContext* library, int64_t unique_id, mapping::MachineDesc&& machine); + Copy(int64_t unique_id, mapping::MachineDesc&& machine); public: void add_input(LogicalStore store); diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 0b59ed0e0a..be3955e6f6 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -163,19 +163,13 @@ std::unique_ptr Runtime::create_task(LibraryContext* library, return impl_->create_task(library, task_id, launch_shape); } -std::unique_ptr Runtime::create_copy(LibraryContext* library) -{ - return impl_->create_copy(library); -} +std::unique_ptr Runtime::create_copy() { return impl_->create_copy(); } -void Runtime::issue_fill(LibraryContext* library, LogicalStore lhs, LogicalStore value) -{ - impl_->issue_fill(library, lhs, value); -} +void Runtime::issue_fill(LogicalStore lhs, LogicalStore value) { impl_->issue_fill(lhs, value); } -void Runtime::issue_fill(LibraryContext* library, LogicalStore lhs, const Scalar& value) +void Runtime::issue_fill(LogicalStore lhs, const Scalar& value) { - issue_fill(library, lhs, create_store(value)); + issue_fill(lhs, create_store(value)); } void Runtime::submit(std::unique_ptr op) { impl_->submit(std::move(op)); } diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 76aede68d5..4baf525bcf 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -165,27 +165,23 @@ class Runtime { /** * @brief Creates a Copy * - * @param library Library in which the copy is created - * * @return Copy object */ - std::unique_ptr create_copy(LibraryContext* library); + std::unique_ptr create_copy(); /** * @brief Fills a given store with a constant * - * @param library Library in which the fill is performed * @param lhs Logical store to fill * @param value Logical store that contains the constant value to fill the store with */ - void issue_fill(LibraryContext* library, LogicalStore lhs, LogicalStore value); + void issue_fill(LogicalStore lhs, LogicalStore value); /** * @brief Fills a given store with a constant * - * @param library Library in which the fill is performed * @param lhs Logical store to fill * @param value Value to fill the store with */ - void issue_fill(LibraryContext* library, LogicalStore lhs, const Scalar& value); + void issue_fill(LogicalStore lhs, const Scalar& value); /** * @brief Submits an operation for execution * diff --git a/src/core/runtime/task.cc b/src/core/runtime/task.cc index a97e4558a3..f939210600 100644 --- a/src/core/runtime/task.cc +++ b/src/core/runtime/task.cc @@ -38,9 +38,8 @@ namespace legate { // legate::Operation //////////////////////////////////////////////////// -Operation::Operation(LibraryContext* library, uint64_t unique_id, mapping::MachineDesc&& machine) - : library_(library), - unique_id_(unique_id), +Operation::Operation(uint64_t unique_id, mapping::MachineDesc&& machine) + : unique_id_(unique_id), machine_(std::move(machine)), provenance_(detail::Runtime::get_runtime()->provenance_manager()->get_provenance()) { @@ -72,11 +71,11 @@ void Operation::record_partition(const Variable* variable, // legate::Task //////////////////////////////////////////////////// -Task::Task(LibraryContext* library, +Task::Task(const LibraryContext* library, int64_t task_id, uint64_t unique_id, mapping::MachineDesc&& machine) - : Operation(library, unique_id, std::move(machine)), task_id_(task_id) + : Operation(unique_id, std::move(machine)), library_(library), task_id_(task_id) { } @@ -264,7 +263,7 @@ std::string Task::to_string() const // legate::AutoTask //////////////////////////////////////////////////// -AutoTask::AutoTask(LibraryContext* library, +AutoTask::AutoTask(const LibraryContext* library, int64_t task_id, uint64_t unique_id, mapping::MachineDesc&& machine) @@ -323,7 +322,7 @@ void AutoTask::add_to_solver(detail::ConstraintSolver& solver) ManualTask::~ManualTask() {} -ManualTask::ManualTask(LibraryContext* library, +ManualTask::ManualTask(const LibraryContext* library, int64_t task_id, const Shape& launch_shape, uint64_t unique_id, diff --git a/tests/cpp/integration/copy_failure.cc b/tests/cpp/integration/copy_failure.cc index 3ca2e4b33d..fc63fbf689 100644 --- a/tests/cpp/integration/copy_failure.cc +++ b/tests/cpp/integration/copy_failure.cc @@ -31,7 +31,7 @@ void test_input_output_failure() auto in_store2 = runtime->create_store(extents, legate::int64()); auto out_store = runtime->create_store(extents, legate::int64()); // fill input store with some values - auto copy = runtime->create_copy(context); + auto copy = runtime->create_copy(); copy->add_input(in_store1); copy->add_input(in_store2); copy->add_output(out_store); @@ -51,7 +51,7 @@ void test_indirect_failure() auto out_store2 = runtime->create_store(extents, legate::int64()); auto indirect_store = runtime->create_store(extents, legate::int64()); - auto copy = runtime->create_copy(context); + auto copy = runtime->create_copy(); copy->add_input(in_store1); copy->add_input(in_store2); copy->add_output(out_store1); @@ -71,14 +71,14 @@ void test_shape_check_failure() auto store3 = runtime->create_store({20, 5}, legate::int64()); { - auto copy = runtime->create_copy(context); + auto copy = runtime->create_copy(); copy->add_input(store1); copy->add_output(store2); EXPECT_THROW(runtime->submit(std::move(copy)), std::runtime_error); } { - auto copy = runtime->create_copy(context); + auto copy = runtime->create_copy(); copy->add_input(store1); copy->add_source_indirect(store2); copy->add_output(store3); @@ -86,7 +86,7 @@ void test_shape_check_failure() } { - auto copy = runtime->create_copy(context); + auto copy = runtime->create_copy(); copy->add_input(store1); copy->add_target_indirect(store2); copy->add_output(store3); @@ -94,7 +94,7 @@ void test_shape_check_failure() } { - auto copy = runtime->create_copy(context); + auto copy = runtime->create_copy(); copy->add_input(store1); copy->add_source_indirect(store2); copy->add_target_indirect(store3); diff --git a/tests/cpp/integration/copy_gather.cc b/tests/cpp/integration/copy_gather.cc index 4d51e2d853..c47013a805 100644 --- a/tests/cpp/integration/copy_gather.cc +++ b/tests/cpp/integration/copy_gather.cc @@ -144,7 +144,7 @@ void test_gather(const GatherSpec& spec) fill_input(context, src, spec.seed); fill_indirect(context, ind, src); - auto copy = runtime->create_copy(context); + auto copy = runtime->create_copy(); copy->add_input(src); copy->add_output(tgt); copy->add_source_indirect(ind); diff --git a/tests/cpp/integration/copy_gather_scatter.cc b/tests/cpp/integration/copy_gather_scatter.cc index db59fbcfe4..b878ff4402 100644 --- a/tests/cpp/integration/copy_gather_scatter.cc +++ b/tests/cpp/integration/copy_gather_scatter.cc @@ -172,9 +172,9 @@ void test_gather_scatter(const GatherScatterSpec& spec) fill_input(context, src, spec.seed); fill_indirect(context, src_ind, src); fill_indirect(context, tgt_ind, tgt); - runtime->issue_fill(context, tgt, spec.init); + runtime->issue_fill(tgt, spec.init); - auto copy = runtime->create_copy(context); + auto copy = runtime->create_copy(); copy->add_input(src); copy->add_output(tgt); copy->add_source_indirect(src_ind); diff --git a/tests/cpp/integration/copy_normal.cc b/tests/cpp/integration/copy_normal.cc index 9c0d0e3f84..fb0278f427 100644 --- a/tests/cpp/integration/copy_normal.cc +++ b/tests/cpp/integration/copy_normal.cc @@ -111,7 +111,7 @@ void test_normal_copies(const std::vector specs) outputs.push_back(runtime->create_store(shape, type.clone())); } - auto copy = runtime->create_copy(context); + auto copy = runtime->create_copy(); for (auto& input : inputs) copy->add_input(input); for (auto& output : outputs) copy->add_output(output); runtime->submit(std::move(copy)); diff --git a/tests/cpp/integration/copy_scatter.cc b/tests/cpp/integration/copy_scatter.cc index 2a74731e6d..14bf080819 100644 --- a/tests/cpp/integration/copy_scatter.cc +++ b/tests/cpp/integration/copy_scatter.cc @@ -169,9 +169,9 @@ void test_scatter(const ScatterSpec& spec) fill_input(context, src, spec.seed); fill_indirect(context, ind, tgt); - runtime->issue_fill(context, tgt, spec.init); + runtime->issue_fill(tgt, spec.init); - auto copy = runtime->create_copy(context); + auto copy = runtime->create_copy(); copy->add_input(src); copy->add_output(tgt); copy->add_target_indirect(ind); diff --git a/tests/cpp/integration/fill.cc b/tests/cpp/integration/fill.cc index 7cbf957b32..e0b3a6573b 100644 --- a/tests/cpp/integration/fill.cc +++ b/tests/cpp/integration/fill.cc @@ -128,7 +128,7 @@ void test_fill_index(int32_t dim) auto value = runtime->create_store(v); // fill input store with some values - runtime->issue_fill(context, lhs, value); + runtime->issue_fill(lhs, value); // check the result of fill check_output(lhs, v); @@ -156,12 +156,12 @@ void test_fill_slice(int32_t dim) auto value_outside_slice = runtime->create_store(v2); // First fill the entire store with v1 - runtime->issue_fill(context, lhs, value_outside_slice); + runtime->issue_fill(lhs, value_outside_slice); // Then fill a slice with v2 auto sliced = lhs; for (int32_t idx = 0; idx < dim; ++idx) sliced = sliced.slice(idx, legate::Slice(offset)); - runtime->issue_fill(context, sliced, value_in_slice); + runtime->issue_fill(sliced, value_in_slice); // check if the slice is filled correctly check_output_slice(lhs, v1, v2, offset); diff --git a/tests/cpp/integration/inline_map.cc b/tests/cpp/integration/inline_map.cc index 823af879ae..e248dea8e2 100644 --- a/tests/cpp/integration/inline_map.cc +++ b/tests/cpp/integration/inline_map.cc @@ -49,7 +49,7 @@ void test_mapped_regions_leak(legate::Runtime* runtime, legate::LibraryContext* { { auto l_store = runtime->create_store({5}, legate::int64()); - auto p_store = l_store.get_physical_store(context); + auto p_store = l_store.get_physical_store(); EXPECT_FALSE(p_store->is_future()); EXPECT_EQ(runtime->impl()->num_inline_mapped(), 1); } @@ -59,17 +59,17 @@ void test_mapped_regions_leak(legate::Runtime* runtime, legate::LibraryContext* void test_inline_map_future(legate::Runtime* runtime, legate::LibraryContext* context) { auto l_store = runtime->create_store({1}, legate::int64(), true /*optimize_scalar*/); - auto p_store = l_store.get_physical_store(context); + auto p_store = l_store.get_physical_store(); EXPECT_TRUE(p_store->is_future()); } void test_inline_map_region_and_slice(legate::Runtime* runtime, legate::LibraryContext* context) { auto root_ls = runtime->create_store({5}, legate::int64()); - auto root_ps = root_ls.get_physical_store(context); + auto root_ps = root_ls.get_physical_store(); EXPECT_FALSE(root_ps->is_future()); auto slice_ls = root_ls.slice(0, legate::Slice(1)); - auto slice_ps = slice_ls.get_physical_store(context); + auto slice_ps = slice_ls.get_physical_store(); EXPECT_FALSE(slice_ps->is_future()); auto root_acc = root_ps->write_accessor(); root_acc[2] = 42; @@ -81,7 +81,7 @@ void test_inline_map_and_task(legate::Runtime* runtime, legate::LibraryContext* { auto l_store = runtime->create_store({5}, legate::int64()); { - auto p_store = l_store.get_physical_store(context); + auto p_store = l_store.get_physical_store(); auto acc = p_store->write_accessor(); acc[2] = 42; } @@ -89,7 +89,7 @@ void test_inline_map_and_task(legate::Runtime* runtime, legate::LibraryContext* task->add_input(l_store); task->add_output(l_store); runtime->submit(std::move(task)); - auto p_store = l_store.get_physical_store(context); + auto p_store = l_store.get_physical_store(); auto acc = p_store->read_accessor(); EXPECT_EQ(acc[2], 43); } diff --git a/tests/cpp/integration/manual_simple.cc b/tests/cpp/integration/manual_simple.cc index 6f77389139..a4750c5905 100644 --- a/tests/cpp/integration/manual_simple.cc +++ b/tests/cpp/integration/manual_simple.cc @@ -39,10 +39,10 @@ void test_manual_task(legate::LibraryContext* context, legate::LogicalStore stor runtime->submit(std::move(task)); } -void print_store(legate::LibraryContext* context, legate::LogicalStore store) +void print_store(legate::LogicalStore store) { auto runtime = legate::Runtime::get_runtime(); - auto p_store = store.get_physical_store(context); + auto p_store = store.get_physical_store(); auto acc = p_store->read_accessor(); auto shape = p_store->shape<2>(); std::stringstream ss; @@ -59,9 +59,9 @@ TEST(Integration, ManualSimple) auto store = runtime->create_store({5, 5}, legate::int64()); test_auto_task(context, store); - print_store(context, store); + print_store(store); test_manual_task(context, store); - print_store(context, store); + print_store(store); } } // namespace manualsimple diff --git a/tests/cpp/integration/multi_scalar_out.cc b/tests/cpp/integration/multi_scalar_out.cc index 4d5e435b71..57fe24cc90 100644 --- a/tests/cpp/integration/multi_scalar_out.cc +++ b/tests/cpp/integration/multi_scalar_out.cc @@ -65,13 +65,11 @@ void test_reducer_manual(legate::LibraryContext* context, runtime->submit(std::move(task)); } -void print_stores(legate::LibraryContext* context, - legate::LogicalStore scalar1, - legate::LogicalStore scalar2) +void print_stores(legate::LogicalStore scalar1, legate::LogicalStore scalar2) { auto runtime = legate::Runtime::get_runtime(); - auto p_scalar1 = scalar1.get_physical_store(context); - auto p_scalar2 = scalar2.get_physical_store(context); + auto p_scalar1 = scalar1.get_physical_store(); + auto p_scalar2 = scalar2.get_physical_store(); auto acc1 = p_scalar1->read_accessor(); auto acc2 = p_scalar2->read_accessor(); std::stringstream ss; @@ -90,11 +88,11 @@ TEST(Integration, MultiScalarOut) auto scalar2 = runtime->create_store({1, 1, 1}, legate::int32(), true); auto store = runtime->create_store({10}, legate::int64()); test_writer_auto(context, scalar1, scalar2); - print_stores(context, scalar1, scalar2); + print_stores(scalar1, scalar2); test_reducer_auto(context, scalar1, scalar2, store); - print_stores(context, scalar1, scalar2); + print_stores(scalar1, scalar2); test_reducer_manual(context, scalar1, scalar2); - print_stores(context, scalar1, scalar2); + print_stores(scalar1, scalar2); } } // namespace multiscalarout From 885b1b54e954281bd7cac19c12c0b1640edf7966 Mon Sep 17 00:00:00 2001 From: Parag Kulkarni Date: Fri, 23 Jun 2023 05:38:39 -0700 Subject: [PATCH 0148/1425] Modify rules to trigger pipeline on a push event. * Includes change to make the pipeline rules apply globally across the stages and run the pipeline, when changes are pushed to the legate/legate.core.internal repository. Earlier it was localized to build stage and only triggered with a merge-request event. See merge request legate/legate.core.internal!58 --- .gitlab-ci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4dbbbc9580..78d114f41b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,15 +1,17 @@ image: name: "rapidsai/devcontainers:23.06-cpp-cuda11.8-mambaforge-ubuntu22.04" +workflow: + rules: + - if: ($CI_PIPELINE_SOURCE == "merge_request_event" || $CI_PIPELINE_SOURCE == "push") && + ($CI_PROJECT_PATH == "legate/legate.core.internal") + stages: - build Build:LegateCore: stage: build - rules: - - if: ($CI_PIPELINE_SOURCE == "merge_request_event" && $CI_PROJECT_PATH == "legate/legate.core.internal") - tags: - cpu_only - gpu/disabled From 84619369568d29f98e403c04e5356054a7d3b0c1 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 23 Jun 2023 10:46:05 -0700 Subject: [PATCH 0149/1425] Find-or-declare partition API * Catch up the API change in the test * Find-or-declare partition API See merge request legate/legate.core.internal!63 --- src/core/data/detail/logical_store.cc | 2 ++ src/core/data/detail/logical_store.h | 1 + src/core/runtime/operation.h | 9 ++++++ src/core/runtime/task.cc | 19 ++++++++++-- tests/cpp/integration/inout.cc | 44 +++++++++++++++++++++++++++ 5 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 tests/cpp/integration/inout.cc diff --git a/src/core/data/detail/logical_store.cc b/src/core/data/detail/logical_store.cc index 0a7116f9fd..f420819748 100644 --- a/src/core/data/detail/logical_store.cc +++ b/src/core/data/detail/logical_store.cc @@ -371,6 +371,8 @@ const Type& LogicalStore::type() const { return storage_->type(); } bool LogicalStore::transformed() const { return !transform_->identity(); } +uint64_t LogicalStore::id() const { return store_id_; } + const Storage* LogicalStore::get_storage() const { return storage_.get(); } LogicalRegionField* LogicalStore::get_region_field() { return storage_->get_region_field(); } diff --git a/src/core/data/detail/logical_store.h b/src/core/data/detail/logical_store.h index f5d3bb23cf..7a738d658f 100644 --- a/src/core/data/detail/logical_store.h +++ b/src/core/data/detail/logical_store.h @@ -182,6 +182,7 @@ class LogicalStore : public std::enable_shared_from_this { bool has_scalar_storage() const; const Type& type() const; bool transformed() const; + uint64_t id() const; public: const Storage* get_storage() const; diff --git a/src/core/runtime/operation.h b/src/core/runtime/operation.h index 0f4d5e71ca..646d948f3c 100644 --- a/src/core/runtime/operation.h +++ b/src/core/runtime/operation.h @@ -64,6 +64,14 @@ class Operation { virtual std::string to_string() const = 0; public: + /** + * @brief Finds or creates a partition symbol for the given store + * + * @param Store for which the partition symbol is queried + * + * @return The existing symbol if there is one for the store, a fresh symbol otherwise + */ + const Variable* find_or_declare_partition(LogicalStore store); /** * @brief Declares partition symbol * @@ -104,6 +112,7 @@ class Operation { uint32_t next_part_id_{0}; std::vector> partition_symbols_{}; std::map store_mappings_{}; + std::map part_mappings_{}; std::string provenance_; mapping::MachineDesc machine_; }; diff --git a/src/core/runtime/task.cc b/src/core/runtime/task.cc index f939210600..fba9d786c6 100644 --- a/src/core/runtime/task.cc +++ b/src/core/runtime/task.cc @@ -45,6 +45,16 @@ Operation::Operation(uint64_t unique_id, mapping::MachineDesc&& machine) { } +const Variable* Operation::find_or_declare_partition(LogicalStore store) +{ + auto* impl = store.impl().get(); + auto finder = part_mappings_.find(impl); + if (finder != part_mappings_.end()) return finder->second; + const auto* symb = declare_partition(); + part_mappings_.insert({impl, symb}); + return symb; +} + const Variable* Operation::declare_partition() { partition_symbols_.emplace_back(new Variable(this, next_part_id_++)); @@ -59,9 +69,12 @@ detail::LogicalStore* Operation::find_store(const Variable* part_symb) const void Operation::record_partition(const Variable* variable, std::shared_ptr store) { - if (store_mappings_.find(*variable) != store_mappings_.end()) { - throw std::invalid_argument("Variable " + variable->to_string() + - " is already assigned to another store"); + auto finder = store_mappings_.find(*variable); + if (finder != store_mappings_.end()) { + if (finder->second->id() != store->id()) + throw std::invalid_argument("Variable " + variable->to_string() + + " is already assigned to another store"); + return; } store_mappings_[*variable] = store.get(); all_stores_.insert(std::move(store)); diff --git a/tests/cpp/integration/inout.cc b/tests/cpp/integration/inout.cc new file mode 100644 index 0000000000..06e8429117 --- /dev/null +++ b/tests/cpp/integration/inout.cc @@ -0,0 +1,44 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "legate.h" +#include "tasks/task_simple.h" + +namespace inout { + +void test_inout() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(task::simple::library_name); + + auto store = runtime->create_store({10, 10}, legate::int64()); + runtime->issue_fill(store, legate::Scalar(int64_t(0))); + + auto task = runtime->create_task(context, task::simple::HELLO); + task->add_input(store, task->find_or_declare_partition(store)); + task->add_output(store, task->find_or_declare_partition(store)); + runtime->submit(std::move(task)); +} + +TEST(Integration, InOut) +{ + legate::Core::perform_registration(); + test_inout(); +} + +} // namespace inout From 96f174e4291f0872b6ed193365962b0c1c6551b8 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 23 Jun 2023 12:39:57 -0700 Subject: [PATCH 0150/1425] Capture primary partitions when they are associated with stores for the first time * Make sure we capture the pointer at the right moment * Capture primary partitions when they are associated with stores for the first time See merge request legate/legate.core.internal!64 --- src/core/runtime/task.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/core/runtime/task.cc b/src/core/runtime/task.cc index fba9d786c6..9e7e3e2c37 100644 --- a/src/core/runtime/task.cc +++ b/src/core/runtime/task.cc @@ -76,8 +76,11 @@ void Operation::record_partition(const Variable* variable, " is already assigned to another store"); return; } - store_mappings_[*variable] = store.get(); + auto* p_store = store.get(); + store_mappings_[*variable] = p_store; all_stores_.insert(std::move(store)); + if (part_mappings_.find(p_store) == part_mappings_.end()) + part_mappings_.insert({p_store, variable}); } //////////////////////////////////////////////////// From f06eafe1445431c6ae650e28dd8da68dab95d910 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 23 Jun 2023 22:30:44 -0700 Subject: [PATCH 0151/1425] Make legate::start work with the legate launcher * Make legate::start work with the legate launcher See merge request legate/legate.core.internal!65 --- src/core/runtime/detail/runtime.cc | 42 +++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index 10e0503022..8dbfa52724 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -745,25 +745,41 @@ MachineManager* Runtime::machine_manager() const { return machine_manager_; } /*static*/ int32_t Runtime::start(int32_t argc, char** argv) { - Legion::Runtime::initialize(&argc, &argv, true /*filter legion and realm args*/); + static bool initialized = false; - Legion::Runtime::add_registration_callback(registration_callback); + if (initialized) return 0; + initialized = true; - auto result = Legion::Runtime::start(argc, argv, true); - if (result != 0) { - log_legate.error("Legion Runtime failed to start."); - return result; - } + int32_t result = 0; + if (!Legion::Runtime::has_runtime()) { + Legion::Runtime::initialize(&argc, &argv, true /*filter legion and realm args*/); + + Legion::Runtime::add_registration_callback(registration_callback); + + result = Legion::Runtime::start(argc, argv, true); + if (result != 0) { + log_legate.error("Legion Runtime failed to start."); + return result; + } + } else + Legion::Runtime::perform_registration_callback(registration_callback, true /*global*/); // Get the runtime now that we've started it auto legion_runtime = Legion::Runtime::get_runtime(); - // Then we can make this thread into an implicit top-level task - auto legion_context = legion_runtime->begin_implicit_task(LEGATE_CORE_TOPLEVEL_TASK_ID, - 0 /*mapper id*/, - Processor::LOC_PROC, - TOPLEVEL_NAME, - true /*control replicable*/); + Legion::Context legion_context; + // If the context already exists, that means that some other driver started the top-level task, + // so here we just grab it to initialize the Legate runtime + if (Legion::Runtime::has_context()) + legion_context = Legion::Runtime::get_context(); + else { + // Otherwise we make this thread into an implicit top-level task + legion_context = legion_runtime->begin_implicit_task(LEGATE_CORE_TOPLEVEL_TASK_ID, + 0 /*mapper id*/, + Processor::LOC_PROC, + TOPLEVEL_NAME, + true /*control replicable*/); + } // We can now initialize the Legate runtime with the Legion context Runtime::get_runtime()->initialize(legion_context); From 0c40b698511030953852de8df36327eab80b8a3f Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Sat, 24 Jun 2023 01:12:56 -0700 Subject: [PATCH 0152/1425] Fix compile errors and warnings reported from Clang * Fix compile errors and warnings reported from Clang See merge request legate/legate.core.internal!67 --- src/core/data/transform.cc | 36 ++++++++++++++++++++++ src/core/mapping/machine.cc | 3 ++ src/core/partitioning/constraint_solver.cc | 12 +++++--- src/core/partitioning/constraint_solver.h | 2 +- src/core/runtime/detail/copy_launcher.cc | 2 +- src/core/runtime/detail/req_analyzer.cc | 2 +- 6 files changed, 49 insertions(+), 8 deletions(-) diff --git a/src/core/data/transform.cc b/src/core/data/transform.cc index 1c75149b52..f0e395c878 100644 --- a/src/core/data/transform.cc +++ b/src/core/data/transform.cc @@ -224,6 +224,10 @@ std::unique_ptr Shift::convert(const Partition* partition) const Shape(tiling->color_shape()), tiling->offsets().update(dim_, offset_)); } + default: { + assert(false); + return nullptr; + } } assert(false); return nullptr; @@ -242,6 +246,10 @@ std::unique_ptr Shift::invert(const Partition* partition) const Shape(tiling->color_shape()), tiling->offsets().update(dim_, new_offset)); } + default: { + assert(false); + return nullptr; + } } assert(false); return nullptr; @@ -337,6 +345,10 @@ std::unique_ptr Promote::convert(const Partition* partition) const tiling->color_shape().insert(extra_dim_, 1), tiling->offsets().insert(extra_dim_, 0)); } + default: { + assert(false); + return nullptr; + } } assert(false); return nullptr; @@ -354,6 +366,10 @@ std::unique_ptr Promote::invert(const Partition* partition) const tiling->color_shape().remove(extra_dim_), tiling->offsets().remove(extra_dim_)); } + default: { + assert(false); + return nullptr; + } } assert(false); return nullptr; @@ -451,6 +467,10 @@ std::unique_ptr Project::convert(const Partition* partition) const tiling->color_shape().remove(dim_), tiling->offsets().remove(dim_)); } + default: { + assert(false); + return nullptr; + } } assert(false); return nullptr; @@ -468,6 +488,10 @@ std::unique_ptr Project::invert(const Partition* partition) const tiling->color_shape().insert(dim_, 1), tiling->offsets().insert(dim_, coord_)); } + default: { + assert(false); + return nullptr; + } } assert(false); return nullptr; @@ -561,6 +585,10 @@ std::unique_ptr Transpose::convert(const Partition* partition) const tiling->color_shape().map(axes_), tiling->offsets().map(axes_)); } + default: { + assert(false); + return nullptr; + } } assert(false); return nullptr; @@ -578,6 +606,10 @@ std::unique_ptr Transpose::invert(const Partition* partition) const tiling->color_shape().map(inverse_), tiling->offsets().map(inverse_)); } + default: { + assert(false); + return nullptr; + } } assert(false); return nullptr; @@ -757,6 +789,10 @@ std::unique_ptr Delinearize::invert(const Partition* partition) const return create_tiling( std::move(new_tile_shape), std::move(new_color_shape), std::move(new_offsets)); } + default: { + assert(false); + return nullptr; + } } assert(false); return nullptr; diff --git a/src/core/mapping/machine.cc b/src/core/mapping/machine.cc index 0c0de51186..0bfdec6051 100644 --- a/src/core/mapping/machine.cc +++ b/src/core/mapping/machine.cc @@ -366,6 +366,9 @@ Machine::Machine() omps_.push_back(proc); continue; } + default: { + continue; + } } } diff --git a/src/core/partitioning/constraint_solver.cc b/src/core/partitioning/constraint_solver.cc index 96f4d91db9..ac693110e1 100644 --- a/src/core/partitioning/constraint_solver.cc +++ b/src/core/partitioning/constraint_solver.cc @@ -78,7 +78,10 @@ struct ConstraintSolver::EquivClass { ConstraintSolver::ConstraintSolver() {} -ConstraintSolver::~ConstraintSolver() {} +ConstraintSolver::~ConstraintSolver() +{ + for (auto equiv_class : equiv_classes_) delete equiv_class; +} void ConstraintSolver::add_partition_symbol(const Variable* partition_symbol, bool is_output) { @@ -171,10 +174,9 @@ void ConstraintSolver::solve_constraints() for (auto& [_, entry] : table) distinct_entries.insert(entry); for (auto* entry : distinct_entries) { - auto equiv_class = std::make_unique(entry); - for (auto* symb : equiv_class->partition_symbols) - equiv_class_map_.insert({*symb, equiv_class.get()}); - equiv_classes_.push_back(std::move(equiv_class)); + auto equiv_class = new EquivClass(entry); + for (auto* symb : equiv_class->partition_symbols) equiv_class_map_.insert({*symb, equiv_class}); + equiv_classes_.push_back(equiv_class); } } diff --git a/src/core/partitioning/constraint_solver.h b/src/core/partitioning/constraint_solver.h index c1dcc29696..195410cd35 100644 --- a/src/core/partitioning/constraint_solver.h +++ b/src/core/partitioning/constraint_solver.h @@ -56,7 +56,7 @@ struct ConstraintSolver { private: class EquivClass; std::map equiv_class_map_{}; - std::vector> equiv_classes_{}; + std::vector equiv_classes_{}; }; } // namespace legate::detail diff --git a/src/core/runtime/detail/copy_launcher.cc b/src/core/runtime/detail/copy_launcher.cc index ca8ccc383d..d85946efaf 100644 --- a/src/core/runtime/detail/copy_launcher.cc +++ b/src/core/runtime/detail/copy_launcher.cc @@ -186,7 +186,7 @@ void CopyLauncher::pack_args() namespace { template -constexpr bool is_single; +constexpr bool is_single = false; template <> constexpr bool is_single = true; template <> diff --git a/src/core/runtime/detail/req_analyzer.cc b/src/core/runtime/detail/req_analyzer.cc index 314eee17e3..ad7bb75180 100644 --- a/src/core/runtime/detail/req_analyzer.cc +++ b/src/core/runtime/detail/req_analyzer.cc @@ -74,7 +74,7 @@ void FieldSet::coalesce() namespace { template -constexpr bool is_single; +constexpr bool is_single = false; template <> constexpr bool is_single = true; template <> From 9c3e77e648ab56ce6f482b3841b8b2a60e439cf0 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Tue, 27 Jun 2023 22:33:35 -0700 Subject: [PATCH 0153/1425] Clean up all Legion handles before the runtime is destroyed * Add a test case with a pending exception * Make sure all Legion handles are released before the runtime is destroyed See merge request legate/legate.core.internal!68 --- src/core/runtime/detail/runtime.cc | 6 ++++++ src/core/utilities/multi_set.h | 5 +++++ src/core/utilities/multi_set.inl | 6 ++++++ tests/cpp/integration/exception.cc | 28 ++++++++++++++++++++++++++-- 4 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index 8dbfa52724..57aa627241 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -826,6 +826,12 @@ void Runtime::destroy() // We're about to deallocate objects below, so let's block on all outstanding Legion operations issue_execution_fence(true); + // Any STL containers holding Legion handles need to be cleared here, otherwise they cause + // trouble when they get destroyed in the Legate runtime's destructor + inline_mapped_.clear(); + physical_region_refs_.clear(); + pending_exceptions_.clear(); + // We finally deallocate managers for (auto& [_, context] : libraries_) delete context; libraries_.clear(); diff --git a/src/core/utilities/multi_set.h b/src/core/utilities/multi_set.h index 6f049c0a55..4811a3884c 100644 --- a/src/core/utilities/multi_set.h +++ b/src/core/utilities/multi_set.h @@ -46,6 +46,11 @@ class MultiSet { */ bool contains(const T& value) const; + /** + * @brief Clears the container + */ + void clear(); + private: std::map map_; }; diff --git a/src/core/utilities/multi_set.inl b/src/core/utilities/multi_set.inl index 61f464fd70..b6cb7a99e6 100644 --- a/src/core/utilities/multi_set.inl +++ b/src/core/utilities/multi_set.inl @@ -50,4 +50,10 @@ bool MultiSet::contains(const T& value) const return map_.contains(value); } +template +void MultiSet::clear() +{ + map_.clear(); +} + } // namespace legate diff --git a/tests/cpp/integration/exception.cc b/tests/cpp/integration/exception.cc index 46c9bd447f..d712347990 100644 --- a/tests/cpp/integration/exception.cc +++ b/tests/cpp/integration/exception.cc @@ -36,6 +36,7 @@ constexpr size_t SIZE = 10; struct ExceptionTask : public legate::LegateTask { static void cpu_variant(legate::TaskContext& context) { + EXPECT_TRUE(context.can_raise_exception()); auto raise = context.scalars().at(0).value(); auto index = context.scalars().at(1).value(); // Make sure only some of the point tasks raise an exception @@ -163,16 +164,32 @@ void test_multi(bool use_auto_task) EXPECT_EQ(exn.value().error_message(), EXN_MSG); } +void test_pending() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + runtime->set_max_pending_exceptions(2); + auto task = runtime->create_task(context, EXCEPTION_TASK); + task->throws_exception(true); + task->add_scalar_arg(legate::Scalar(false)); + task->add_scalar_arg(legate::Scalar(12345)); + + runtime->submit(std::move(task)); + + // Finish the test with a pending exception to check if the runtime cleans things up correctly +} + } // namespace -TEST(Integration, ExceptionSingle) +TEST(Exception, Single) { legate::Core::perform_registration(); test_single(); } -TEST(Integration, ExceptionMulti) +TEST(Exception, Multi) { legate::Core::perform_registration(); @@ -180,4 +197,11 @@ TEST(Integration, ExceptionMulti) test_multi(false /* use_auto_task */); } +TEST(Exception, Pending) +{ + legate::Core::perform_registration(); + + test_pending(); +} + } // namespace exception From f9e874df49680509b06f6f00d97fb8558146ca0a Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 28 Jun 2023 20:10:52 -0700 Subject: [PATCH 0154/1425] Pimpl for operation types * Build C++ examples in CI * Fix examples and tests to catch up the API change * Missing files * Pimpl for operation types * Move operation specific files to the operation sub-directory See merge request legate/legate.core.internal!69 --- .../home/coder/.local/bin/build-all | 1 + .../home/coder/.local/bin/build-cpp-example | 23 ++ examples/cpp/hello/hello.cc | 22 +- examples/cpp/hello/hello_print.cc | 4 +- examples/cpp/io/io.cc | 40 +- legate_core_cpp.cmake | 30 +- src/core/comm/comm_cpu.cc | 2 +- src/core/comm/comm_nccl.cu | 2 +- src/core/data/detail/logical_store.cc | 2 +- src/core/data/logical_store.h | 3 + src/core/operation/copy.cc | 50 +++ src/core/operation/copy.h | 60 +++ .../{runtime => operation/detail}/copy.cc | 63 ++- src/core/operation/detail/copy.h | 71 ++++ .../detail/copy_launcher.cc | 6 +- .../detail/copy_launcher.h | 0 .../{runtime => operation}/detail/fill.cc | 29 +- src/core/{runtime => operation}/detail/fill.h | 21 +- .../detail/fill_launcher.cc | 4 +- .../detail/fill_launcher.h | 0 .../detail/launcher_arg.cc | 5 +- .../detail/launcher_arg.h | 0 src/core/operation/detail/operation.cc | 65 +++ src/core/operation/detail/operation.h | 77 ++++ .../detail/projection.cc | 3 +- .../detail/projection.h | 0 .../detail/req_analyzer.cc | 2 +- .../detail/req_analyzer.h | 2 +- .../{runtime => operation/detail}/task.cc | 150 +++---- src/core/operation/detail/task.h | 146 +++++++ .../detail/task_launcher.cc | 6 +- .../detail/task_launcher.h | 0 src/core/operation/task.cc | 130 ++++++ .../{runtime/operation.h => operation/task.h} | 371 +++++++----------- src/core/partitioning/constraint.cc | 4 +- src/core/partitioning/constraint.h | 11 +- src/core/partitioning/constraint_solver.cc | 2 +- src/core/partitioning/partitioner.cc | 2 +- src/core/runtime/detail/runtime.cc | 11 +- src/core/runtime/detail/runtime.h | 10 +- src/core/runtime/runtime.cc | 29 +- src/core/runtime/runtime.h | 39 +- src/legate.h | 3 +- .../cpp/integration/alignment_constraints.cc | 44 +-- .../cpp/integration/broadcast_constraints.cc | 28 +- tests/cpp/integration/copy_failure.cc | 40 +- tests/cpp/integration/copy_gather.cc | 24 +- tests/cpp/integration/copy_gather_scatter.cc | 36 +- tests/cpp/integration/copy_normal.cc | 14 +- tests/cpp/integration/copy_scatter.cc | 28 +- tests/cpp/integration/copy_util.inl | 16 +- tests/cpp/integration/cpu_communicator.cc | 10 +- tests/cpp/integration/exception.cc | 30 +- tests/cpp/integration/fill.cc | 16 +- tests/cpp/integration/inline_map.cc | 4 +- tests/cpp/integration/inout.cc | 4 +- tests/cpp/integration/machine_scope.cc | 36 +- tests/cpp/integration/manual_simple.cc | 6 +- tests/cpp/integration/multi_scalar_out.cc | 24 +- tests/cpp/integration/nccl.cu | 14 +- tests/cpp/integration/provenance.cc | 14 +- tests/cpp/integration/region_manager.cc | 4 +- tests/cpp/integration/weighted.cc | 15 +- 63 files changed, 1210 insertions(+), 698 deletions(-) create mode 100755 continuous_integration/home/coder/.local/bin/build-cpp-example create mode 100644 src/core/operation/copy.cc create mode 100644 src/core/operation/copy.h rename src/core/{runtime => operation/detail}/copy.cc (79%) create mode 100644 src/core/operation/detail/copy.h rename src/core/{runtime => operation}/detail/copy_launcher.cc (98%) rename src/core/{runtime => operation}/detail/copy_launcher.h (100%) rename src/core/{runtime => operation}/detail/fill.cc (73%) rename src/core/{runtime => operation}/detail/fill.h (62%) rename src/core/{runtime => operation}/detail/fill_launcher.cc (97%) rename src/core/{runtime => operation}/detail/fill_launcher.h (100%) rename src/core/{runtime => operation}/detail/launcher_arg.cc (96%) rename src/core/{runtime => operation}/detail/launcher_arg.h (100%) create mode 100644 src/core/operation/detail/operation.cc create mode 100644 src/core/operation/detail/operation.h rename src/core/{runtime => operation}/detail/projection.cc (98%) rename src/core/{runtime => operation}/detail/projection.h (100%) rename src/core/{runtime => operation}/detail/req_analyzer.cc (99%) rename src/core/{runtime => operation}/detail/req_analyzer.h (98%) rename src/core/{runtime => operation/detail}/task.cc (71%) create mode 100644 src/core/operation/detail/task.h rename src/core/{runtime => operation}/detail/task_launcher.cc (98%) rename src/core/{runtime => operation}/detail/task_launcher.h (100%) create mode 100644 src/core/operation/task.cc rename src/core/{runtime/operation.h => operation/task.h} (54%) diff --git a/continuous_integration/home/coder/.local/bin/build-all b/continuous_integration/home/coder/.local/bin/build-all index 9f1288c84c..bb1a37ec94 100755 --- a/continuous_integration/home/coder/.local/bin/build-all +++ b/continuous_integration/home/coder/.local/bin/build-all @@ -12,6 +12,7 @@ build_all() { build-legate-cpp; build-legate-wheel; build-legate-conda; + build-cpp-example; build-cpp-test; } diff --git a/continuous_integration/home/coder/.local/bin/build-cpp-example b/continuous_integration/home/coder/.local/bin/build-cpp-example new file mode 100755 index 0000000000..c071a29539 --- /dev/null +++ b/continuous_integration/home/coder/.local/bin/build-cpp-example @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +build_cpp_examples() { + set -ex; + + conda deactivate + + cd ~/legate/examples/cpp + + rm -rf build + cmake -B build -S . -D legate_core_ROOT=../../build -D CMAKE_BUILD_TYPE=Debug + cmake --build build -v -j 4 + + ( + cd ~/legate/examples/cpp/build; + cpack -G TGZ; + cp cpp_examples*.tar.gz /tmp/out/; + ); + + { set +x; } 2>/dev/null; +} + +build_cpp_examples "$@"; diff --git a/examples/cpp/hello/hello.cc b/examples/cpp/hello/hello.cc index 9bd0434422..a6be30d3d2 100644 --- a/examples/cpp/hello/hello.cc +++ b/examples/cpp/hello/hello.cc @@ -26,8 +26,8 @@ legate::LogicalStore iota(legate::LibraryContext* context, size_t size) auto runtime = legate::Runtime::get_runtime(); auto task = runtime->create_task(context, task::hello::IOTA); auto output = runtime->create_store({size}, legate::float32(), true); - auto part = task->declare_partition(); - task->add_output(output, part); + auto part = task.declare_partition(); + task.add_output(output, part); runtime->submit(std::move(task)); return output; } @@ -37,11 +37,11 @@ legate::LogicalStore square(legate::LibraryContext* context, legate::LogicalStor auto runtime = legate::Runtime::get_runtime(); auto task = runtime->create_task(context, task::hello::SQUARE); auto output = runtime->create_store(input.extents().data(), legate::float32(), true); - auto part1 = task->declare_partition(); - auto part2 = task->declare_partition(); + auto part1 = task.declare_partition(); + auto part2 = task.declare_partition(); - task->add_input(input, part1); - task->add_output(output, part2); + task.add_input(input, part1); + task.add_output(output, part2); runtime->submit(std::move(task)); return output; } @@ -56,11 +56,11 @@ legate::LogicalStore sum(legate::LibraryContext* context, auto output = runtime->create_store(legate::Scalar(legate::float32(), bytearray)); auto redop = input.type().find_reduction_operator(legate::ReductionOpKind::ADD); - auto part1 = task->declare_partition(); - auto part2 = task->declare_partition(); + auto part1 = task.declare_partition(); + auto part2 = task.declare_partition(); - task->add_input(input, part1); - task->add_reduction(output, redop, part2); + task.add_input(input, part1); + task.add_reduction(output, redop, part2); runtime->submit(std::move(task)); return output; } @@ -68,7 +68,7 @@ legate::LogicalStore sum(legate::LibraryContext* context, float to_scalar(legate::LibraryContext* context, legate::LogicalStore scalar) { auto runtime = legate::Runtime::get_runtime(); - auto p_scalar = scalar.get_physical_store(context); + auto p_scalar = scalar.get_physical_store(); auto acc = p_scalar->read_accessor(); float output = static_cast(acc[{0}]); return output; diff --git a/examples/cpp/hello/hello_print.cc b/examples/cpp/hello/hello_print.cc index 1c47d3e8fb..6516bea9b9 100644 --- a/examples/cpp/hello/hello_print.cc +++ b/examples/cpp/hello/hello_print.cc @@ -26,7 +26,7 @@ void test_print_hello(legate::LibraryContext* context, std::string str) auto runtime = legate::Runtime::get_runtime(); auto task = runtime->create_task(context, task::hello::HELLO_WORLD); - task->add_scalar_arg(legate::Scalar(str)); + task.add_scalar_arg(legate::Scalar(str)); runtime->submit(std::move(task)); } @@ -35,7 +35,7 @@ void test_print_hellos(legate::LibraryContext* context, std::string str, size_t auto runtime = legate::Runtime::get_runtime(); auto task = runtime->create_task(context, task::hello::HELLO_WORLD, legate::Shape({count})); - task->add_scalar_arg(legate::Scalar(str)); + task.add_scalar_arg(legate::Scalar(str)); runtime->submit(std::move(task)); } diff --git a/examples/cpp/io/io.cc b/examples/cpp/io/io.cc index 4aed904d1d..8dc2726f56 100644 --- a/examples/cpp/io/io.cc +++ b/examples/cpp/io/io.cc @@ -45,15 +45,15 @@ class IOArray : public Array { { auto runtime = legate::Runtime::get_runtime(); auto task = runtime->create_task(context_, task::legateio::WRITE_FILE); - auto part = task->declare_partition(); + auto part = task.declare_partition(); - task->add_scalar_arg(legate::Scalar(filename)); - task->add_input(store_, part); + task.add_scalar_arg(legate::Scalar(filename)); + task.add_input(store_, part); // Request a broadcasting for the input. Since this is the only store // argument to the task, Legate will launch a single task from this // task descriptor. - task->add_constraint(legate::broadcast(part, {0})); + task.add_constraint(legate::broadcast(part, {0})); runtime->submit(std::move(task)); } @@ -69,10 +69,10 @@ class IOArray : public Array { auto runtime = legate::Runtime::get_runtime(); auto task = runtime->create_task(context_, task::legateio::WRITE_UNEVEN_TILES); - auto part = task->declare_partition(); + auto part = task.declare_partition(); - task->add_scalar_arg(legate::Scalar(path)); - task->add_input(store_, part); + task.add_scalar_arg(legate::Scalar(path)); + task.add_input(store_, part); runtime->submit(std::move(task)); } @@ -90,14 +90,14 @@ class IOArray : public Array { auto launch_shape = store_partition.partition()->color_shape(); auto task = runtime->create_task(context_, task::legateio::WRITE_EVEN_TILES, launch_shape); - task->add_input(store_partition); - task->add_scalar_arg(legate::Scalar(path)); + task.add_input(store_partition); + task.add_scalar_arg(legate::Scalar(path)); auto extents = store_.extents(); EXPECT_EQ(extents.size(), 2); - task->add_scalar_arg( + task.add_scalar_arg( legate::Scalar(std::vector{(uint32_t)extents[0], (uint32_t)extents[1]})); - task->add_scalar_arg(legate::Scalar(std::vector{tile_shape, tile_shape})); + task.add_scalar_arg(legate::Scalar(std::vector{tile_shape, tile_shape})); runtime->submit(std::move(task)); } @@ -116,10 +116,10 @@ IOArray read_file(legate::LibraryContext* context, auto runtime = legate::Runtime::get_runtime(); auto output = runtime->create_store(std::move(dtype), 1); auto task = runtime->create_task(context, task::legateio::READ_FILE); - auto part = task->declare_partition(); + auto part = task.declare_partition(); - task->add_scalar_arg(legate::Scalar(filename)); - task->add_output(output, part); + task.add_scalar_arg(legate::Scalar(filename)); + task.add_output(output, part); runtime->submit(std::move(task)); return IOArray(context, output); @@ -139,8 +139,8 @@ IOArray read_file_parallel(legate::LibraryContext* context, auto task = runtime->create_task(context, task::legateio::READ_FILE, legate::Shape({parallelism})); - task->add_scalar_arg(legate::Scalar(filename)); - task->add_output(output); + task.add_scalar_arg(legate::Scalar(filename)); + task.add_output(output); runtime->submit(std::move(task)); return IOArray(context, output); @@ -202,8 +202,8 @@ IOArray read_uneven_tiles(legate::LibraryContext* context, std::string path) auto task = runtime->create_task(context, task::legateio::READ_UNEVEN_TILES, legate::Shape(color_shape)); - task->add_output(output); - task->add_scalar_arg(legate::Scalar(path)); + task.add_output(output); + task.add_scalar_arg(legate::Scalar(path)); runtime->submit(std::move(task)); return IOArray(context, output); @@ -256,8 +256,8 @@ IOArray read_even_tiles(legate::LibraryContext* context, std::string path) auto launch_shape = output_partition.partition()->color_shape(); auto task = runtime->create_task(context, task::legateio::READ_EVEN_TILES, launch_shape); - task->add_output(output_partition); - task->add_scalar_arg(legate::Scalar(path)); + task.add_output(output_partition); + task.add_scalar_arg(legate::Scalar(path)); runtime->submit(std::move(task)); // Unlike AutoTask, manually parallelized tasks don't update the "key" diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index dc3917a510..f3d1a02484 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -208,32 +208,35 @@ list(APPEND legate_core_SOURCES src/core/mapping/mapping.cc src/core/mapping/operation.cc src/core/mapping/store.cc + src/core/operation/copy.cc + src/core/operation/task.cc + src/core/operation/detail/copy.cc + src/core/operation/detail/copy_launcher.cc + src/core/operation/detail/fill.cc + src/core/operation/detail/fill_launcher.cc + src/core/operation/detail/launcher_arg.cc + src/core/operation/detail/operation.cc + src/core/operation/detail/projection.cc + src/core/operation/detail/req_analyzer.cc + src/core/operation/detail/task.cc + src/core/operation/detail/task_launcher.cc src/core/partitioning/constraint.cc src/core/partitioning/constraint_solver.cc src/core/partitioning/partition.cc src/core/partitioning/partitioner.cc src/core/partitioning/restriction.cc src/core/runtime/context.cc - src/core/runtime/copy.cc src/core/runtime/projection.cc src/core/runtime/runtime.cc - src/core/runtime/task.cc src/core/runtime/tracker.cc src/core/runtime/shard.cc src/core/runtime/detail/communicator_manager.cc - src/core/runtime/detail/copy_launcher.cc src/core/runtime/detail/field_manager.cc - src/core/runtime/detail/fill.cc - src/core/runtime/detail/fill_launcher.cc - src/core/runtime/detail/launcher_arg.cc src/core/runtime/detail/machine_manager.cc src/core/runtime/detail/partition_manager.cc - src/core/runtime/detail/projection.cc src/core/runtime/detail/provenance_manager.cc src/core/runtime/detail/region_manager.cc - src/core/runtime/detail/req_analyzer.cc src/core/runtime/detail/runtime.cc - src/core/runtime/detail/task_launcher.cc src/core/task/registrar.cc src/core/task/return.cc src/core/task/task.cc @@ -366,7 +369,8 @@ if (legate_core_BUILD_DOCS) src/core/runtime/runtime.inl src/core/runtime/context.h # operation - src/core/runtime/operation.h + src/core/operation/task.h + src/core/operation/copy.h # partitioning src/core/partitioning/constraint.h # mapping @@ -455,6 +459,11 @@ install( src/core/mapping/store.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/legate/core/mapping) +install( + FILES src/core/operation/copy.h + src/core/operation/task.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/legate/core/operation) + install( FILES src/core/partitioning/constraint.h src/core/partitioning/partition.h @@ -464,7 +473,6 @@ install( install( FILES src/core/runtime/context.h src/core/runtime/context.inl - src/core/runtime/operation.h src/core/runtime/resource.h src/core/runtime/projection.h src/core/runtime/runtime.h diff --git a/src/core/comm/comm_cpu.cc b/src/core/comm/comm_cpu.cc index 2cdfdef96f..ee822e6b61 100644 --- a/src/core/comm/comm_cpu.cc +++ b/src/core/comm/comm_cpu.cc @@ -15,9 +15,9 @@ */ #include "core/comm/comm_cpu.h" +#include "core/operation/detail/task_launcher.h" #include "core/runtime/detail/communicator_manager.h" #include "core/runtime/detail/runtime.h" -#include "core/runtime/detail/task_launcher.h" #include "legate.h" #include "core/comm/coll.h" diff --git a/src/core/comm/comm_nccl.cu b/src/core/comm/comm_nccl.cu index 3da787b16d..ccc4179754 100644 --- a/src/core/comm/comm_nccl.cu +++ b/src/core/comm/comm_nccl.cu @@ -18,9 +18,9 @@ #include "core/cuda/cuda_help.h" #include "core/cuda/stream_pool.h" #include "core/data/buffer.h" +#include "core/operation/detail/task_launcher.h" #include "core/runtime/detail/communicator_manager.h" #include "core/runtime/detail/runtime.h" -#include "core/runtime/detail/task_launcher.h" #include "core/utilities/nvtx_help.h" #include "core/utilities/typedefs.h" #include "legate.h" diff --git a/src/core/data/detail/logical_store.cc b/src/core/data/detail/logical_store.cc index f420819748..341a89897c 100644 --- a/src/core/data/detail/logical_store.cc +++ b/src/core/data/detail/logical_store.cc @@ -17,8 +17,8 @@ #include "core/data/detail/logical_store.h" #include "core/mapping/machine.h" +#include "core/operation/detail/projection.h" #include "core/runtime/detail/partition_manager.h" -#include "core/runtime/detail/projection.h" #include "core/runtime/detail/runtime.h" #include "core/type/type_traits.h" #include "core/utilities/buffer_builder.h" diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index cfaa087905..850ed9243b 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -351,6 +351,9 @@ class LogicalStorePartition { LogicalStore store() const; std::shared_ptr partition() const; + public: + std::shared_ptr impl() const { return impl_; } + private: std::shared_ptr impl_{nullptr}; }; diff --git a/src/core/operation/copy.cc b/src/core/operation/copy.cc new file mode 100644 index 0000000000..9d379af71c --- /dev/null +++ b/src/core/operation/copy.cc @@ -0,0 +1,50 @@ +/* Copyright 2021 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/operation/copy.h" + +#include "core/operation/detail/copy.h" + +namespace legate { + +void Copy::add_input(LogicalStore store) { impl_->add_input(store.impl()); } + +void Copy::add_output(LogicalStore store) { impl_->add_output(store.impl()); } + +void Copy::add_reduction(LogicalStore store, Legion::ReductionOpID redop) +{ + impl_->add_reduction(store.impl(), redop); +} + +void Copy::add_source_indirect(LogicalStore store) { impl_->add_source_indirect(store.impl()); } + +void Copy::add_target_indirect(LogicalStore store) { impl_->add_target_indirect(store.impl()); } + +void Copy::set_source_indirect_out_of_range(bool flag) +{ + impl_->set_source_indirect_out_of_range(flag); +} + +void Copy::set_target_indirect_out_of_range(bool flag) +{ + impl_->set_target_indirect_out_of_range(flag); +} + +Copy::~Copy() {} + +Copy::Copy(std::unique_ptr impl) : impl_(std::move(impl)) {} + +} // namespace legate diff --git a/src/core/operation/copy.h b/src/core/operation/copy.h new file mode 100644 index 0000000000..85cfd1ebfb --- /dev/null +++ b/src/core/operation/copy.h @@ -0,0 +1,60 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include + +#include "core/data/logical_store.h" +/** + * @file + * @brief Class definition for legate::Copy + */ + +namespace legate::detail { +class Copy; +} // namespace legate::detail + +namespace legate { + +class Copy { + public: + void add_input(LogicalStore store); + void add_output(LogicalStore store); + void add_reduction(LogicalStore store, Legion::ReductionOpID redop); + void add_source_indirect(LogicalStore store); + void add_target_indirect(LogicalStore store); + + public: + void set_source_indirect_out_of_range(bool flag); + void set_target_indirect_out_of_range(bool flag); + + public: + Copy(const Copy&) = delete; + Copy(Copy&&) = default; + Copy& operator=(const Copy&) = delete; + Copy& operator=(Copy&&) = default; + + public: + ~Copy(); + + private: + friend class Runtime; + Copy(std::unique_ptr impl); + std::unique_ptr impl_; +}; + +} // namespace legate diff --git a/src/core/runtime/copy.cc b/src/core/operation/detail/copy.cc similarity index 79% rename from src/core/runtime/copy.cc rename to src/core/operation/detail/copy.cc index 3f2818c5e5..f09c14bfaa 100644 --- a/src/core/runtime/copy.cc +++ b/src/core/operation/detail/copy.cc @@ -1,4 +1,4 @@ -/* Copyright 2021 NVIDIA Corporation +/* Copyright 2023 NVIDIA Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,19 +14,16 @@ * */ -#include "core/runtime/operation.h" +#include "core/operation/detail/copy.h" -#include "core/data/detail/logical_store.h" -#include "core/data/logical_store.h" +#include "core/operation/detail/copy_launcher.h" +#include "core/operation/detail/projection.h" #include "core/partitioning/constraint.h" #include "core/partitioning/constraint_solver.h" #include "core/partitioning/partition.h" #include "core/partitioning/partitioner.h" -#include "core/runtime/context.h" -#include "core/runtime/detail/copy_launcher.h" -#include "core/runtime/detail/projection.h" -namespace legate { +namespace legate::detail { Copy::Copy(int64_t unique_id, mapping::MachineDesc&& machine) : Operation(unique_id, std::move(machine)) @@ -34,75 +31,71 @@ Copy::Copy(int64_t unique_id, mapping::MachineDesc&& machine) } void Copy::add_store(std::vector& store_args, - LogicalStore& store, + std::shared_ptr store, const Variable* partition_symbol) { - auto store_impl = store.impl(); - store_args.push_back(StoreArg(store_impl.get(), partition_symbol)); - record_partition(partition_symbol, std::move(store_impl)); + store_args.push_back(StoreArg(store.get(), partition_symbol)); + record_partition(partition_symbol, std::move(store)); } void Copy::add_store(std::optional& store_arg, - LogicalStore& store, + std::shared_ptr store, const Variable* partition_symbol) { - auto store_impl = store.impl(); - store_arg = StoreArg(store_impl.get(), partition_symbol); - record_partition(partition_symbol, std::move(store_impl)); + store_arg = StoreArg(store.get(), partition_symbol); + record_partition(partition_symbol, std::move(store)); } -void check_store(LogicalStore store) +void check_store(std::shared_ptr& store) { - if (store.unbound() || store.impl()->has_scalar_storage() || store.transformed()) { - std::string msg = "Copy accepts only normal, not transformed, region-backed store"; - throw std::runtime_error(msg); + if (store->unbound() || store->has_scalar_storage() || store->transformed()) { + throw std::runtime_error("Copy accepts only normal, not transformed, region-backed stores"); } } -void Copy::add_input(LogicalStore store) +void Copy::add_input(std::shared_ptr&& store) { check_store(store); - add_store(inputs_, store, declare_partition()); + add_store(inputs_, std::move(store), declare_partition()); } -void Copy::add_output(LogicalStore store) +void Copy::add_output(std::shared_ptr&& store) { check_store(store); if (reductions_.size() > 0) throw std::runtime_error("Copy targets must be either all normal outputs or reductions"); - add_store(outputs_, store, declare_partition()); + add_store(outputs_, std::move(store), declare_partition()); } -void Copy::add_reduction(LogicalStore store, Legion::ReductionOpID redop) +void Copy::add_reduction(std::shared_ptr&& store, Legion::ReductionOpID redop) { check_store(store); if (outputs_.size() > 0) throw std::runtime_error("Copy targets must be either all normal outputs or reductions"); - add_store(reductions_, store, declare_partition()); + add_store(reductions_, std::move(store), declare_partition()); reduction_ops_.push_back(redop); } -void Copy::add_source_indirect(LogicalStore store) +void Copy::add_source_indirect(std::shared_ptr&& store) { check_store(store); - add_store(source_indirect_, store, declare_partition()); + add_store(source_indirect_, std::move(store), declare_partition()); } -void Copy::add_target_indirect(LogicalStore store) +void Copy::add_target_indirect(std::shared_ptr&& store) { check_store(store); - add_store(target_indirect_, store, declare_partition()); + add_store(target_indirect_, std::move(store), declare_partition()); } void Copy::set_source_indirect_out_of_range(bool flag) { source_indirect_out_of_range_ = flag; } void Copy::set_target_indirect_out_of_range(bool flag) { target_indirect_out_of_range_ = flag; } -void Copy::launch(detail::Strategy* p_strategy) +void Copy::launch(Strategy* p_strategy) { auto& strategy = *p_strategy; - detail::CopyLauncher launcher( - machine_, source_indirect_out_of_range_, target_indirect_out_of_range_); + CopyLauncher launcher(machine_, source_indirect_out_of_range_, target_indirect_out_of_range_); auto launch_domain = strategy.launch_domain(this); auto create_projection_info = [&strategy, &launch_domain](auto& store, auto& var) { @@ -153,7 +146,7 @@ void Copy::launch(detail::Strategy* p_strategy) } } -void Copy::add_to_solver(detail::ConstraintSolver& solver) +void Copy::add_to_solver(ConstraintSolver& solver) { bool gather = source_indirect_.has_value(); bool scatter = target_indirect_.has_value(); @@ -211,4 +204,4 @@ void Copy::add_to_solver(detail::ConstraintSolver& solver) std::string Copy::to_string() const { return "Copy:" + std::to_string(unique_id_); } -} // namespace legate +} // namespace legate::detail diff --git a/src/core/operation/detail/copy.h b/src/core/operation/detail/copy.h new file mode 100644 index 0000000000..e8194eb053 --- /dev/null +++ b/src/core/operation/detail/copy.h @@ -0,0 +1,71 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include + +#include "core/data/detail/logical_store.h" +#include "core/operation/detail/operation.h" +#include "core/partitioning/constraint.h" + +namespace legate::detail { + +class ConstraintSolver; +class Runtime; + +class Copy : public Operation { + private: + friend class detail::Runtime; + Copy(int64_t unique_id, mapping::MachineDesc&& machine); + + public: + void add_input(std::shared_ptr&& store); + void add_output(std::shared_ptr&& store); + void add_reduction(std::shared_ptr&& store, Legion::ReductionOpID redop); + void add_source_indirect(std::shared_ptr&& store); + void add_target_indirect(std::shared_ptr&& store); + + private: + void add_store(std::vector& store_args, + std::shared_ptr store, + const Variable* partition_symbol); + void add_store(std::optional& store_arg, + std::shared_ptr store, + const Variable* partition_symbol); + + public: + void set_source_indirect_out_of_range(bool flag); + void set_target_indirect_out_of_range(bool flag); + + public: + void launch(detail::Strategy* strategy) override; + + public: + void add_to_solver(detail::ConstraintSolver& solver) override; + + public: + std::string to_string() const override; + + private: + std::vector> constraints_{}; + std::optional source_indirect_{}; + std::optional target_indirect_{}; + bool source_indirect_out_of_range_{true}; + bool target_indirect_out_of_range_{true}; +}; + +} // namespace legate::detail diff --git a/src/core/runtime/detail/copy_launcher.cc b/src/core/operation/detail/copy_launcher.cc similarity index 98% rename from src/core/runtime/detail/copy_launcher.cc rename to src/core/operation/detail/copy_launcher.cc index d85946efaf..9a7c5bf090 100644 --- a/src/core/runtime/detail/copy_launcher.cc +++ b/src/core/operation/detail/copy_launcher.cc @@ -14,12 +14,12 @@ * */ -#include "core/runtime/detail/copy_launcher.h" +#include "core/operation/detail/copy_launcher.h" #include "core/data/detail/logical_store.h" #include "core/mapping/machine.h" +#include "core/operation/detail/launcher_arg.h" +#include "core/operation/detail/projection.h" #include "core/runtime/context.h" -#include "core/runtime/detail/launcher_arg.h" -#include "core/runtime/detail/projection.h" #include "core/runtime/detail/runtime.h" #include "core/utilities/buffer_builder.h" diff --git a/src/core/runtime/detail/copy_launcher.h b/src/core/operation/detail/copy_launcher.h similarity index 100% rename from src/core/runtime/detail/copy_launcher.h rename to src/core/operation/detail/copy_launcher.h diff --git a/src/core/runtime/detail/fill.cc b/src/core/operation/detail/fill.cc similarity index 73% rename from src/core/runtime/detail/fill.cc rename to src/core/operation/detail/fill.cc index 8edc84cdd2..fecab65bd8 100644 --- a/src/core/runtime/detail/fill.cc +++ b/src/core/operation/detail/fill.cc @@ -14,22 +14,24 @@ * */ -#include "core/runtime/detail/fill.h" +#include "core/operation/detail/fill.h" #include "core/data/detail/logical_store.h" +#include "core/operation/detail/fill_launcher.h" +#include "core/operation/detail/projection.h" #include "core/partitioning/constraint_solver.h" #include "core/partitioning/partitioner.h" -#include "core/runtime/context.h" -#include "core/runtime/detail/fill_launcher.h" -#include "core/runtime/detail/projection.h" -namespace legate { +namespace legate::detail { -Fill::Fill(LogicalStore lhs, LogicalStore value, int64_t unique_id, mapping::MachineDesc&& machine) +Fill::Fill(std::shared_ptr&& lhs, + std::shared_ptr&& value, + int64_t unique_id, + mapping::MachineDesc&& machine) : Operation(unique_id, std::move(machine)), lhs_var_(declare_partition()), - lhs_(lhs.impl()), - value_(value.impl()) + lhs_(std::move(lhs)), + value_(std::move(value)) { store_mappings_[*lhs_var_] = lhs_.get(); if (lhs_->unbound() || lhs_->has_scalar_storage()) @@ -39,9 +41,9 @@ Fill::Fill(LogicalStore lhs, LogicalStore value, int64_t unique_id, mapping::Mac throw std::runtime_error("Fill value should be a Future-back store"); } -void Fill::launch(detail::Strategy* strategy) +void Fill::launch(Strategy* strategy) { - detail::FillLauncher launcher(machine_); + FillLauncher launcher(machine_); auto launch_domain = strategy->launch_domain(this); auto part = (*strategy)[lhs_var_]; auto lhs_proj = lhs_->create_partition(part)->create_projection_info(launch_domain); @@ -55,9 +57,6 @@ void Fill::launch(detail::Strategy* strategy) std::string Fill::to_string() const { return "Fill:" + std::to_string(unique_id_); } -void Fill::add_to_solver(detail::ConstraintSolver& solver) -{ - solver.add_partition_symbol(lhs_var_); -} +void Fill::add_to_solver(ConstraintSolver& solver) { solver.add_partition_symbol(lhs_var_); } -} // namespace legate +} // namespace legate::detail diff --git a/src/core/runtime/detail/fill.h b/src/core/operation/detail/fill.h similarity index 62% rename from src/core/runtime/detail/fill.h rename to src/core/operation/detail/fill.h index 4d506f88e4..54b29508f4 100644 --- a/src/core/runtime/detail/fill.h +++ b/src/core/operation/detail/fill.h @@ -16,28 +16,31 @@ #pragma once -#include "core/runtime/operation.h" +#include "core/operation/detail/operation.h" -namespace legate { +namespace legate::detail { class Fill : public Operation { private: - friend class detail::Runtime; - Fill(LogicalStore lhs, LogicalStore value, int64_t unique_id, mapping::MachineDesc&& machine); + friend class Runtime; + Fill(std::shared_ptr&& lhs, + std::shared_ptr&& value, + int64_t unique_id, + mapping::MachineDesc&& machine); public: - void launch(detail::Strategy* strategy) override; + void launch(Strategy* strategy) override; public: std::string to_string() const override; public: - void add_to_solver(detail::ConstraintSolver& solver) override; + void add_to_solver(ConstraintSolver& solver) override; private: const Variable* lhs_var_; - std::shared_ptr lhs_; - std::shared_ptr value_; + std::shared_ptr lhs_; + std::shared_ptr value_; }; -} // namespace legate +} // namespace legate::detail diff --git a/src/core/runtime/detail/fill_launcher.cc b/src/core/operation/detail/fill_launcher.cc similarity index 97% rename from src/core/runtime/detail/fill_launcher.cc rename to src/core/operation/detail/fill_launcher.cc index 9231f67631..a525cc5f48 100644 --- a/src/core/runtime/detail/fill_launcher.cc +++ b/src/core/operation/detail/fill_launcher.cc @@ -14,11 +14,11 @@ * */ -#include "core/runtime/detail/fill_launcher.h" +#include "core/operation/detail/fill_launcher.h" #include "core/data/detail/logical_store.h" #include "core/mapping/machine.h" +#include "core/operation/detail/projection.h" #include "core/runtime/context.h" -#include "core/runtime/detail/projection.h" #include "core/runtime/detail/runtime.h" #include "core/utilities/buffer_builder.h" diff --git a/src/core/runtime/detail/fill_launcher.h b/src/core/operation/detail/fill_launcher.h similarity index 100% rename from src/core/runtime/detail/fill_launcher.h rename to src/core/operation/detail/fill_launcher.h diff --git a/src/core/runtime/detail/launcher_arg.cc b/src/core/operation/detail/launcher_arg.cc similarity index 96% rename from src/core/runtime/detail/launcher_arg.cc rename to src/core/operation/detail/launcher_arg.cc index 946d9336b7..c317a2bd70 100644 --- a/src/core/runtime/detail/launcher_arg.cc +++ b/src/core/operation/detail/launcher_arg.cc @@ -14,9 +14,10 @@ * */ -#include "core/runtime/detail/launcher_arg.h" +#include "core/operation/detail/launcher_arg.h" + #include "core/data/detail/logical_region_field.h" -#include "core/runtime/detail/req_analyzer.h" +#include "core/operation/detail/req_analyzer.h" namespace legate::detail { diff --git a/src/core/runtime/detail/launcher_arg.h b/src/core/operation/detail/launcher_arg.h similarity index 100% rename from src/core/runtime/detail/launcher_arg.h rename to src/core/operation/detail/launcher_arg.h diff --git a/src/core/operation/detail/operation.cc b/src/core/operation/detail/operation.cc new file mode 100644 index 0000000000..b127f4e053 --- /dev/null +++ b/src/core/operation/detail/operation.cc @@ -0,0 +1,65 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/operation/detail/operation.h" + +#include "core/partitioning/constraint.h" +#include "core/runtime/detail/runtime.h" + +namespace legate::detail { + +Operation::Operation(uint64_t unique_id, mapping::MachineDesc&& machine) + : unique_id_(unique_id), + machine_(std::move(machine)), + provenance_(Runtime::get_runtime()->provenance_manager()->get_provenance()) +{ +} + +const Variable* Operation::find_or_declare_partition(std::shared_ptr store) +{ + auto finder = part_mappings_.find(store); + if (finder != part_mappings_.end()) return finder->second; + const auto* symb = declare_partition(); + part_mappings_.insert({std::move(store), symb}); + return symb; +} + +const Variable* Operation::declare_partition() +{ + partition_symbols_.emplace_back(new Variable(this, next_part_id_++)); + return partition_symbols_.back().get(); +} + +LogicalStore* Operation::find_store(const Variable* part_symb) const +{ + return store_mappings_.at(*part_symb); +} + +void Operation::record_partition(const Variable* variable, std::shared_ptr store) +{ + auto finder = store_mappings_.find(*variable); + if (finder != store_mappings_.end()) { + if (finder->second->id() != store->id()) + throw std::invalid_argument("Variable " + variable->to_string() + + " is already assigned to another store"); + return; + } + if (part_mappings_.find(store) == part_mappings_.end()) part_mappings_.insert({store, variable}); + store_mappings_[*variable] = store.get(); + all_stores_.insert(std::move(store)); +} + +} // namespace legate::detail diff --git a/src/core/operation/detail/operation.h b/src/core/operation/detail/operation.h new file mode 100644 index 0000000000..134333ba11 --- /dev/null +++ b/src/core/operation/detail/operation.h @@ -0,0 +1,77 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include + +#include "core/data/detail/logical_store.h" +#include "core/mapping/machine.h" + +namespace legate { +class Variable; +} // namespace legate + +namespace legate::detail { +class ConstraintSolver; +class LogicalStore; +class Strategy; + +class Operation { + protected: + using StoreArg = std::pair; + Operation(uint64_t unique_id, mapping::MachineDesc&& machine); + + public: + virtual ~Operation() {} + + public: + virtual void add_to_solver(ConstraintSolver& solver) = 0; + virtual void launch(Strategy* strategy) = 0; + virtual std::string to_string() const = 0; + + public: + const Variable* find_or_declare_partition(std::shared_ptr store); + const Variable* declare_partition(); + LogicalStore* find_store(const Variable* variable) const; + + public: + const mapping::MachineDesc& machine() const { return machine_; } + const std::string& provenance() const { return provenance_; } + + protected: + void record_partition(const Variable* variable, std::shared_ptr store); + + protected: + uint64_t unique_id_; + + protected: + std::set> all_stores_{}; + std::vector inputs_{}; + std::vector outputs_{}; + std::vector reductions_{}; + std::vector reduction_ops_{}; + + protected: + uint32_t next_part_id_{0}; + std::vector> partition_symbols_{}; + std::map store_mappings_{}; + std::map, const Variable*> part_mappings_{}; + std::string provenance_; + mapping::MachineDesc machine_; +}; + +} // namespace legate::detail diff --git a/src/core/runtime/detail/projection.cc b/src/core/operation/detail/projection.cc similarity index 98% rename from src/core/runtime/detail/projection.cc rename to src/core/operation/detail/projection.cc index 3a7b8fbf2f..1d00b12613 100644 --- a/src/core/runtime/detail/projection.cc +++ b/src/core/operation/detail/projection.cc @@ -14,7 +14,8 @@ * */ -#include "core/runtime/detail/projection.h" +#include "core/operation/detail/projection.h" + #include "core/runtime/detail/runtime.h" namespace legate::detail { diff --git a/src/core/runtime/detail/projection.h b/src/core/operation/detail/projection.h similarity index 100% rename from src/core/runtime/detail/projection.h rename to src/core/operation/detail/projection.h diff --git a/src/core/runtime/detail/req_analyzer.cc b/src/core/operation/detail/req_analyzer.cc similarity index 99% rename from src/core/runtime/detail/req_analyzer.cc rename to src/core/operation/detail/req_analyzer.cc index ad7bb75180..006f567ed9 100644 --- a/src/core/runtime/detail/req_analyzer.cc +++ b/src/core/operation/detail/req_analyzer.cc @@ -14,7 +14,7 @@ * */ -#include "core/runtime/detail/req_analyzer.h" +#include "core/operation/detail/req_analyzer.h" #include "core/runtime/detail/runtime.h" namespace legate::detail { diff --git a/src/core/runtime/detail/req_analyzer.h b/src/core/operation/detail/req_analyzer.h similarity index 98% rename from src/core/runtime/detail/req_analyzer.h rename to src/core/operation/detail/req_analyzer.h index 5a433dd6b7..e2109d512e 100644 --- a/src/core/runtime/detail/req_analyzer.h +++ b/src/core/operation/detail/req_analyzer.h @@ -16,7 +16,7 @@ #pragma once -#include "core/runtime/detail/projection.h" +#include "core/operation/detail/projection.h" #include "core/runtime/detail/runtime.h" namespace legate::detail { diff --git a/src/core/runtime/task.cc b/src/core/operation/detail/task.cc similarity index 71% rename from src/core/runtime/task.cc rename to src/core/operation/detail/task.cc index 9e7e3e2c37..14f0a91466 100644 --- a/src/core/runtime/task.cc +++ b/src/core/operation/detail/task.cc @@ -14,74 +14,22 @@ * */ -#include "core/runtime/operation.h" +#include "core/operation/detail/task.h" #include -#include -#include "core/data/detail/logical_store.h" #include "core/data/scalar.h" -#include "core/partitioning/constraint.h" +#include "core/operation/detail/projection.h" +#include "core/operation/detail/task_launcher.h" #include "core/partitioning/constraint_solver.h" #include "core/partitioning/partition.h" #include "core/partitioning/partitioner.h" #include "core/runtime/context.h" #include "core/runtime/detail/communicator_manager.h" -#include "core/runtime/detail/projection.h" #include "core/runtime/detail/provenance_manager.h" #include "core/runtime/detail/runtime.h" -#include "core/runtime/detail/task_launcher.h" -namespace legate { - -//////////////////////////////////////////////////// -// legate::Operation -//////////////////////////////////////////////////// - -Operation::Operation(uint64_t unique_id, mapping::MachineDesc&& machine) - : unique_id_(unique_id), - machine_(std::move(machine)), - provenance_(detail::Runtime::get_runtime()->provenance_manager()->get_provenance()) -{ -} - -const Variable* Operation::find_or_declare_partition(LogicalStore store) -{ - auto* impl = store.impl().get(); - auto finder = part_mappings_.find(impl); - if (finder != part_mappings_.end()) return finder->second; - const auto* symb = declare_partition(); - part_mappings_.insert({impl, symb}); - return symb; -} - -const Variable* Operation::declare_partition() -{ - partition_symbols_.emplace_back(new Variable(this, next_part_id_++)); - return partition_symbols_.back().get(); -} - -detail::LogicalStore* Operation::find_store(const Variable* part_symb) const -{ - return store_mappings_.at(*part_symb); -} - -void Operation::record_partition(const Variable* variable, - std::shared_ptr store) -{ - auto finder = store_mappings_.find(*variable); - if (finder != store_mappings_.end()) { - if (finder->second->id() != store->id()) - throw std::invalid_argument("Variable " + variable->to_string() + - " is already assigned to another store"); - return; - } - auto* p_store = store.get(); - store_mappings_[*variable] = p_store; - all_stores_.insert(std::move(store)); - if (part_mappings_.find(p_store) == part_mappings_.end()) - part_mappings_.insert({p_store, variable}); -} +namespace legate::detail { //////////////////////////////////////////////////// // legate::Task @@ -114,7 +62,7 @@ void Task::add_communicator(const std::string& name) communicator_factories_.push_back(comm_mgr->find_factory(name)); } -void Task::launch(detail::Strategy* p_strategy) +void Task::launch(Strategy* p_strategy) { auto& strategy = *p_strategy; detail::TaskLauncher launcher(library_, machine_, provenance_, task_id_); @@ -287,36 +235,35 @@ AutoTask::AutoTask(const LibraryContext* library, { } -void AutoTask::add_input(LogicalStore store, const Variable* partition_symbol) +void AutoTask::add_input(std::shared_ptr&& store, const Variable* partition_symbol) { - add_store(inputs_, store, partition_symbol); + add_store(inputs_, std::move(store), partition_symbol); } -void AutoTask::add_output(LogicalStore store, const Variable* partition_symbol) +void AutoTask::add_output(std::shared_ptr&& store, const Variable* partition_symbol) { - if (store.impl()->has_scalar_storage()) + if (store->has_scalar_storage()) scalar_outputs_.push_back(outputs_.size()); - else if (store.impl()->unbound()) + else if (store->unbound()) unbound_outputs_.push_back(outputs_.size()); - add_store(outputs_, store, partition_symbol); + add_store(outputs_, std::move(store), partition_symbol); } -void AutoTask::add_reduction(LogicalStore store, +void AutoTask::add_reduction(std::shared_ptr&& store, Legion::ReductionOpID redop, const Variable* partition_symbol) { - if (store.impl()->has_scalar_storage()) scalar_reductions_.push_back(reductions_.size()); - add_store(reductions_, store, partition_symbol); + if (store->has_scalar_storage()) scalar_reductions_.push_back(reductions_.size()); + add_store(reductions_, std::move(store), partition_symbol); reduction_ops_.push_back(redop); } void AutoTask::add_store(std::vector& store_args, - LogicalStore& store, + std::shared_ptr store, const Variable* partition_symbol) { - auto store_impl = store.impl(); - store_args.push_back(StoreArg(store_impl.get(), partition_symbol)); - record_partition(partition_symbol, store_impl); + store_args.push_back(StoreArg(store.get(), partition_symbol)); + record_partition(partition_symbol, std::move(store)); } void AutoTask::add_constraint(std::unique_ptr constraint) @@ -349,64 +296,67 @@ ManualTask::ManualTask(const LibraryContext* library, strategy_->set_launch_shape(this, launch_shape); } -void ManualTask::add_input(LogicalStore store) { add_store(inputs_, store, create_no_partition()); } - -void ManualTask::add_output(LogicalStore store) +void ManualTask::add_input(std::shared_ptr&& store) { - if (store.impl()->has_scalar_storage()) - scalar_outputs_.push_back(outputs_.size()); - else if (store.impl()->unbound()) - unbound_outputs_.push_back(outputs_.size()); - add_store(outputs_, store, create_no_partition()); + add_store(inputs_, std::move(store), create_no_partition()); } -void ManualTask::add_reduction(LogicalStore store, Legion::ReductionOpID redop) +void ManualTask::add_input(std::shared_ptr&& store_partition) { - if (store.impl()->has_scalar_storage()) scalar_reductions_.push_back(reductions_.size()); - add_store(reductions_, store, create_no_partition()); - reduction_ops_.push_back(redop); + add_store(inputs_, store_partition->store(), store_partition->partition()); } -void ManualTask::add_input(LogicalStorePartition store_partition) +void ManualTask::add_output(std::shared_ptr&& store) { - add_store(inputs_, store_partition.store(), store_partition.partition()); + if (store->has_scalar_storage()) + scalar_outputs_.push_back(outputs_.size()); + else if (store->unbound()) + unbound_outputs_.push_back(outputs_.size()); + add_store(outputs_, std::move(store), create_no_partition()); } -void ManualTask::add_output(LogicalStorePartition store_partition) +void ManualTask::add_output(std::shared_ptr&& store_partition) { #ifdef DEBUG_LEGATE - assert(!store_partition.store().unbound()); + // TODO: We need to raise an exception for the user error in this case + assert(!store_partition->store()->unbound()); #endif - if (store_partition.store().impl()->has_scalar_storage()) - scalar_outputs_.push_back(outputs_.size()); - add_store(outputs_, store_partition.store(), store_partition.partition()); + if (store_partition->store()->has_scalar_storage()) scalar_outputs_.push_back(outputs_.size()); + add_store(outputs_, store_partition->store(), store_partition->partition()); +} + +void ManualTask::add_reduction(std::shared_ptr&& store, Legion::ReductionOpID redop) +{ + if (store->has_scalar_storage()) scalar_reductions_.push_back(reductions_.size()); + add_store(reductions_, std::move(store), create_no_partition()); + reduction_ops_.push_back(redop); } -void ManualTask::add_reduction(LogicalStorePartition store_partition, Legion::ReductionOpID redop) +void ManualTask::add_reduction(std::shared_ptr&& store_partition, + Legion::ReductionOpID redop) { - if (store_partition.store().impl()->has_scalar_storage()) + if (store_partition->store()->has_scalar_storage()) scalar_reductions_.push_back(reductions_.size()); - add_store(reductions_, store_partition.store(), store_partition.partition()); + add_store(reductions_, store_partition->store(), store_partition->partition()); reduction_ops_.push_back(redop); } void ManualTask::add_store(std::vector& store_args, - const LogicalStore& store, + std::shared_ptr store, std::shared_ptr partition) { - auto store_impl = store.impl(); auto partition_symbol = declare_partition(); - store_args.push_back(StoreArg(store_impl.get(), partition_symbol)); - all_stores_.insert(std::move(store_impl)); - if (store.unbound()) { + store_args.push_back(StoreArg(store.get(), partition_symbol)); + if (store->unbound()) { auto field_space = detail::Runtime::get_runtime()->create_field_space(); strategy_->insert(partition_symbol, std::move(partition), field_space); } else strategy_->insert(partition_symbol, std::move(partition)); + all_stores_.insert(std::move(store)); } -void ManualTask::launch(detail::Strategy*) { Task::launch(strategy_.get()); } +void ManualTask::launch(Strategy*) { Task::launch(strategy_.get()); } -void ManualTask::add_to_solver(detail::ConstraintSolver& solver) {} +void ManualTask::add_to_solver(ConstraintSolver& solver) {} -} // namespace legate +} // namespace legate::detail diff --git a/src/core/operation/detail/task.h b/src/core/operation/detail/task.h new file mode 100644 index 0000000000..626fb21039 --- /dev/null +++ b/src/core/operation/detail/task.h @@ -0,0 +1,146 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include + +#include "core/operation/detail/operation.h" +#include "core/partitioning/constraint.h" + +namespace legate { +class Constraint; +class LibraryContext; +class Scalar; +} // namespace legate + +namespace legate::detail { +class CommunicatorFactory; +class ConstraintSolver; +class LogicalStore; +class LogicalStorePartition; +class Strategy; +class Runtime; + +class Task : public Operation { + protected: + Task(const LibraryContext* library, + int64_t task_id, + uint64_t unique_id, + mapping::MachineDesc&& machine); + + public: + virtual ~Task() {} + + public: + void add_scalar_arg(const Scalar& scalar); + void add_scalar_arg(Scalar&& scalar); + void set_concurrent(bool concurrent); + void set_side_effect(bool has_side_effect); + void throws_exception(bool can_throw_exception); + void add_communicator(const std::string& name); + + public: + virtual void launch(Strategy* strategy) override; + + private: + void demux_scalar_stores(const Legion::Future& result); + void demux_scalar_stores(const Legion::FutureMap& result, const Domain& launch_domain); + + public: + std::string to_string() const override; + + protected: + const LibraryContext* library_; + int64_t task_id_; + bool concurrent_{false}; + bool has_side_effect_{false}; + bool can_throw_exception_{false}; + std::vector scalars_{}; + std::vector unbound_outputs_{}; + std::vector scalar_outputs_{}; + std::vector scalar_reductions_{}; + std::vector communicator_factories_{}; +}; + +class AutoTask : public Task { + private: + friend class Runtime; + AutoTask(const LibraryContext* library, + int64_t task_id, + uint64_t unique_id, + mapping::MachineDesc&& machine); + + public: + ~AutoTask() {} + + public: + void add_input(std::shared_ptr&& store, const Variable* partition_symbol); + void add_output(std::shared_ptr&& store, const Variable* partition_symbol); + void add_reduction(std::shared_ptr&& store, + Legion::ReductionOpID redop, + const Variable* partition_symbol); + + private: + void add_store(std::vector& store_args, + std::shared_ptr store, + const Variable* partition_symbol); + + public: + void add_constraint(std::unique_ptr constraint); + void add_to_solver(ConstraintSolver& solver) override; + + private: + std::vector> constraints_{}; +}; + +class ManualTask : public Task { + private: + friend class Runtime; + ManualTask(const LibraryContext* library, + int64_t task_id, + const Shape& launch_shape, + uint64_t unique_id, + mapping::MachineDesc&& machine); + + public: + ~ManualTask(); + + public: + void add_input(std::shared_ptr&& store); + void add_input(std::shared_ptr&& store_partition); + void add_output(std::shared_ptr&& store); + void add_output(std::shared_ptr&& store_partition); + void add_reduction(std::shared_ptr&& store, Legion::ReductionOpID redop); + void add_reduction(std::shared_ptr&& store_partition, + Legion::ReductionOpID redop); + + private: + void add_store(std::vector& store_args, + std::shared_ptr store, + std::shared_ptr partition); + + public: + void launch(Strategy* strategy) override; + + public: + void add_to_solver(ConstraintSolver& solver) override; + + private: + std::unique_ptr strategy_; +}; + +} // namespace legate::detail diff --git a/src/core/runtime/detail/task_launcher.cc b/src/core/operation/detail/task_launcher.cc similarity index 98% rename from src/core/runtime/detail/task_launcher.cc rename to src/core/operation/detail/task_launcher.cc index d238dbb778..41f926baf8 100644 --- a/src/core/runtime/detail/task_launcher.cc +++ b/src/core/operation/detail/task_launcher.cc @@ -14,15 +14,15 @@ * */ -#include "core/runtime/detail/task_launcher.h" +#include "core/operation/detail/task_launcher.h" #include "core/data/detail/logical_region_field.h" #include "core/data/detail/logical_store.h" #include "core/data/scalar.h" #include "core/mapping/machine.h" +#include "core/operation/detail/launcher_arg.h" +#include "core/operation/detail/req_analyzer.h" #include "core/runtime/context.h" -#include "core/runtime/detail/launcher_arg.h" #include "core/runtime/detail/partition_manager.h" -#include "core/runtime/detail/req_analyzer.h" #include "core/runtime/detail/runtime.h" #include "core/runtime/shard.h" #include "core/utilities/buffer_builder.h" diff --git a/src/core/runtime/detail/task_launcher.h b/src/core/operation/detail/task_launcher.h similarity index 100% rename from src/core/runtime/detail/task_launcher.h rename to src/core/operation/detail/task_launcher.h diff --git a/src/core/operation/task.cc b/src/core/operation/task.cc new file mode 100644 index 0000000000..e4ac878afa --- /dev/null +++ b/src/core/operation/task.cc @@ -0,0 +1,130 @@ +/* Copyright 2021 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/operation/task.h" + +#include "core/operation/detail/task.h" + +namespace legate { + +//////////////////////////////////////////////////// +// legate::AutoTask +//////////////////////////////////////////////////// + +void AutoTask::add_input(LogicalStore store, const Variable* partition_symbol) +{ + impl_->add_input(store.impl(), partition_symbol); +} + +void AutoTask::add_output(LogicalStore store, const Variable* partition_symbol) +{ + impl_->add_output(store.impl(), partition_symbol); +} + +void AutoTask::add_reduction(LogicalStore store, + Legion::ReductionOpID redop, + const Variable* partition_symbol) +{ + impl_->add_reduction(store.impl(), redop, partition_symbol); +} + +void AutoTask::add_scalar_arg(const Scalar& scalar) { impl_->add_scalar_arg(scalar); } + +void AutoTask::add_scalar_arg(Scalar&& scalar) { impl_->add_scalar_arg(scalar); } + +void AutoTask::add_constraint(std::unique_ptr constraint) +{ + impl_->add_constraint(std::move(constraint)); +} + +const Variable* AutoTask::find_or_declare_partition(LogicalStore store) +{ + return impl_->find_or_declare_partition(store.impl()); +} + +const Variable* AutoTask::declare_partition() { return impl_->declare_partition(); } + +const mapping::MachineDesc& AutoTask::machine() const { return impl_->machine(); } + +const std::string& AutoTask::provenance() const { return impl_->provenance(); } + +void AutoTask::set_concurrent(bool concurrent) { impl_->set_concurrent(concurrent); } + +void AutoTask::set_side_effect(bool has_side_effect) { impl_->set_side_effect(has_side_effect); } + +void AutoTask::throws_exception(bool can_throw_exception) +{ + impl_->throws_exception(can_throw_exception); +} + +void AutoTask::add_communicator(const std::string& name) { impl_->add_communicator(name); } + +AutoTask::~AutoTask() {} + +AutoTask::AutoTask(std::unique_ptr impl) : impl_(std::move(impl)) {} + +//////////////////////////////////////////////////// +// legate::ManualTask +//////////////////////////////////////////////////// + +void ManualTask::add_input(LogicalStore store) { impl_->add_input(store.impl()); } + +void ManualTask::add_input(LogicalStorePartition store_partition) +{ + impl_->add_input(store_partition.impl()); +} + +void ManualTask::add_output(LogicalStore store) { impl_->add_output(store.impl()); } + +void ManualTask::add_output(LogicalStorePartition store_partition) +{ + impl_->add_output(store_partition.impl()); +} + +void ManualTask::add_reduction(LogicalStore store, Legion::ReductionOpID redop) +{ + impl_->add_reduction(store.impl(), redop); +} + +void ManualTask::add_reduction(LogicalStorePartition store_partition, Legion::ReductionOpID redop) +{ + impl_->add_reduction(store_partition.impl(), redop); +} + +void ManualTask::add_scalar_arg(const Scalar& scalar) { impl_->add_scalar_arg(scalar); } + +void ManualTask::add_scalar_arg(Scalar&& scalar) { impl_->add_scalar_arg(scalar); } + +const mapping::MachineDesc& ManualTask::machine() const { return impl_->machine(); } + +const std::string& ManualTask::provenance() const { return impl_->provenance(); } + +void ManualTask::set_concurrent(bool concurrent) { impl_->set_concurrent(concurrent); } + +void ManualTask::set_side_effect(bool has_side_effect) { impl_->set_side_effect(has_side_effect); } + +void ManualTask::throws_exception(bool can_throw_exception) +{ + impl_->throws_exception(can_throw_exception); +} + +void ManualTask::add_communicator(const std::string& name) { impl_->add_communicator(name); } + +ManualTask::~ManualTask() {} + +ManualTask::ManualTask(std::unique_ptr impl) : impl_(std::move(impl)) {} + +} // namespace legate diff --git a/src/core/runtime/operation.h b/src/core/operation/task.h similarity index 54% rename from src/core/runtime/operation.h rename to src/core/operation/task.h index 646d948f3c..b8d52d7b9a 100644 --- a/src/core/runtime/operation.h +++ b/src/core/operation/task.h @@ -1,4 +1,4 @@ -/* Copyright 2021 NVIDIA Corporation +/* Copyright 2023 NVIDIA Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,51 +17,83 @@ #pragma once #include -#include #include "core/data/logical_store.h" #include "core/mapping/machine.h" -#include "core/partitioning/constraint.h" -#include "legion.h" - -/** @defgroup op Task and non-task operation descriptors - */ /** * @file - * @brief Class definitions for various operation kinds + * @brief Class definitions for legate::AutoTask and legate::ManualTask */ namespace legate::detail { -class CommunicatorFactory; -class ConstraintSolver; -class LogicalStore; -class Strategy; -class Runtime; +class AutoTask; +class ManualTask; } // namespace legate::detail namespace legate { class Constraint; -class LibraryContext; class Scalar; +class Variable; /** * @ingroup op - * @brief A base class for all operation kinds + * @brief A class for auto-parallelized task desciptors */ -class Operation { - protected: - using StoreArg = std::pair; - Operation(uint64_t unique_id, mapping::MachineDesc&& machine); - +class AutoTask { public: - virtual ~Operation() {} - - public: - virtual void add_to_solver(detail::ConstraintSolver& solver) = 0; - virtual void launch(detail::Strategy* strategy) = 0; - virtual std::string to_string() const = 0; + /** + * @brief Adds a store to the task as input + * + * Partitioning of the store is controlled by constraints on the partition symbol + * associated with the store + * + * @param store A store to add to the task as input + * @param partition_symbol A partition symbol for the store + */ + void add_input(LogicalStore store, const Variable* partition_symbol); + /** + * @brief Adds a store to the task as output + * + * Partitioning of the store is controlled by constraints on the partition symbol + * associated with the store + * + * @param store A store to add to the task as output + * @param partition_symbol A partition symbol for the store + */ + void add_output(LogicalStore store, const Variable* partition_symbol); + /** + * @brief Adds a store to the task for reductions + * + * Partitioning of the store is controlled by constraints on the partition symbol + * associated with the store + * + * @param store A store to add to the task for reductions + * @param redop ID of the reduction operator to use + * @param partition_symbol A partition symbol for the store + */ + void add_reduction(LogicalStore store, + Legion::ReductionOpID redop, + const Variable* partition_symbol); + /** + * @brief Adds a by-value scalar argument to the task + * + * @param scalar A scalar to add to the task + */ + void add_scalar_arg(const Scalar& scalar); + /** + * @brief Adds a by-value scalar argument to the task + * + * @param scalar A scalar to add to the task + */ + void add_scalar_arg(Scalar&& scalar); + /** + * @brief Adds a partitioning constraint to the task + * + * @param constraint A partitioning constraint + */ + void add_constraint(std::unique_ptr constraint); public: /** @@ -78,73 +110,20 @@ class Operation { * @return A new symbol that can be used when passing a store to an operation */ const Variable* declare_partition(); - detail::LogicalStore* find_store(const Variable* variable) const; - - public: /** * @brief Returns the machine of the scope in which this operation is issued * * @return The machine of the scope */ - const mapping::MachineDesc& machine() const { return machine_; } - + const mapping::MachineDesc& machine() const; /** * @brief Returns the provenance information of this operation * * @return Provenance */ - const std::string& provenance() const { return provenance_; } - - protected: - void record_partition(const Variable* variable, std::shared_ptr store); - - protected: - uint64_t unique_id_; - - protected: - std::set> all_stores_{}; - std::vector inputs_{}; - std::vector outputs_{}; - std::vector reductions_{}; - std::vector reduction_ops_{}; - - protected: - uint32_t next_part_id_{0}; - std::vector> partition_symbols_{}; - std::map store_mappings_{}; - std::map part_mappings_{}; - std::string provenance_; - mapping::MachineDesc machine_; -}; - -/** - * @ingroup op - * @brief A base class for tasks inherited by two kinds of task descriptors - * (auto-parallelized and manually parallelized task descriptors) - */ -class Task : public Operation { - protected: - Task(const LibraryContext* library, - int64_t task_id, - uint64_t unique_id, - mapping::MachineDesc&& machine); - - public: - virtual ~Task() {} + const std::string& provenance() const; public: - /** - * @brief Adds a by-value scalar argument to the task - * - * @param scalar A scalar to add to the task - */ - void add_scalar_arg(const Scalar& scalar); - /** - * @brief Adds a by-value scalar argument to the task - * - * @param scalar A scalar to add to the task - */ - void add_scalar_arg(Scalar&& scalar); /** * @brief Sets whether the task needs a concurrent task launch. * @@ -177,214 +156,142 @@ class Task : public Operation { void add_communicator(const std::string& name); public: - virtual void launch(detail::Strategy* strategy) override; - - private: - void demux_scalar_stores(const Legion::Future& result); - void demux_scalar_stores(const Legion::FutureMap& result, const Domain& launch_domain); + AutoTask(const AutoTask&) = delete; + AutoTask(AutoTask&&) = default; + AutoTask& operator=(const AutoTask&) = delete; + AutoTask& operator=(AutoTask&&) = default; public: - std::string to_string() const override; + ~AutoTask(); - protected: - const LibraryContext* library_; - int64_t task_id_; - bool concurrent_{false}; - bool has_side_effect_{false}; - bool can_throw_exception_{false}; - std::vector scalars_{}; - std::vector unbound_outputs_{}; - std::vector scalar_outputs_{}; - std::vector scalar_reductions_{}; - std::vector communicator_factories_{}; + private: + friend class Runtime; + AutoTask(std::unique_ptr impl); + std::unique_ptr impl_; }; /** * @ingroup op - * @brief A class for auto-parallelized task desciptors + * @brief A class for manually parallelized task descriptors */ -class AutoTask : public Task { - public: - friend class detail::Runtime; - AutoTask(const LibraryContext* library, - int64_t task_id, - uint64_t unique_id, - mapping::MachineDesc&& machine); - - public: - ~AutoTask() {} - +class ManualTask { public: /** * @brief Adds a store to the task as input * - * Partitioning of the store is controlled by constraints on the partition symbol - * associated with the store + * The store will be unpartitioned but broadcasted to all the tasks * * @param store A store to add to the task as input - * @param partition_symbol A partition symbol for the store */ - void add_input(LogicalStore store, const Variable* partition_symbol); + void add_input(LogicalStore store); + /** + * @brief Adds a store partition to the task as input + * + * @param store_partition A store partition to add to the task as input + */ + void add_input(LogicalStorePartition store_partition); /** * @brief Adds a store to the task as output * - * Partitioning of the store is controlled by constraints on the partition symbol - * associated with the store + * The store will be unpartitioned but broadcasted to all the tasks * * @param store A store to add to the task as output - * @param partition_symbol A partition symbol for the store */ - void add_output(LogicalStore store, const Variable* partition_symbol); + void add_output(LogicalStore store); + /** + * @brief Adds a store partition to the task as output + * + * @param store_partition A store partition to add to the task as output + */ + void add_output(LogicalStorePartition store_partition); /** * @brief Adds a store to the task for reductions * - * Partitioning of the store is controlled by constraints on the partition symbol - * associated with the store + * The store will be unpartitioned but broadcasted to all the tasks * * @param store A store to add to the task for reductions - * @param redop ID of the reduction operator to use - * @param partition_symbol A partition symbol for the store + * @param redop ID of the reduction operator */ - void add_reduction(LogicalStore store, - Legion::ReductionOpID redop, - const Variable* partition_symbol); - - private: - void add_store(std::vector& store_args, - LogicalStore& store, - const Variable* partition_symbol); - - public: + void add_reduction(LogicalStore store, Legion::ReductionOpID redop); /** - * @brief Adds a partitioning constraint to the task + * @brief Adds a store partition to the task for reductions * - * @param constraint A partitioning constraint + * @param store_partition A store partition to add to the task for reductions + * @param redop ID of the reduction operator */ - void add_constraint(std::unique_ptr constraint); - void add_to_solver(detail::ConstraintSolver& solver) override; - - private: - std::vector> constraints_{}; -}; - -/** - * @ingroup op - * @brief A class for manually parallelized task descriptors - */ -class ManualTask : public Task { - private: - friend class detail::Runtime; - ManualTask(const LibraryContext* library, - int64_t task_id, - const Shape& launch_shape, - uint64_t unique_id, - mapping::MachineDesc&& machine); - - public: - ~ManualTask(); - - public: + void add_reduction(LogicalStorePartition store_partition, Legion::ReductionOpID redop); /** - * @brief Adds a store to the task as input - * - * The store will be unpartitioned but broadcasted to all the tasks + * @brief Adds a by-value scalar argument to the task * - * @param store A store to add to the task as input + * @param scalar A scalar to add to the task */ - void add_input(LogicalStore store); + void add_scalar_arg(const Scalar& scalar); /** - * @brief Adds a store to the task as output - * - * The store will be unpartitioned but broadcasted to all the tasks + * @brief Adds a by-value scalar argument to the task * - * @param store A store to add to the task as output + * @param scalar A scalar to add to the task */ - void add_output(LogicalStore store); + void add_scalar_arg(Scalar&& scalar); + + public: /** - * @brief Adds a store to the task for reductions + * @brief Returns the machine of the scope in which this operation is issued * - * The store will be unpartitioned but broadcasted to all the tasks + * @return The machine of the scope + */ + const mapping::MachineDesc& machine() const; + /** + * @brief Returns the provenance information of this operation * - * @param store A store to add to the task for reductions - * @param redop ID of the reduction operator + * @return Provenance */ - void add_reduction(LogicalStore store, Legion::ReductionOpID redop); + const std::string& provenance() const; public: /** - * @brief Adds a store partition to the task as input + * @brief Sets whether the task needs a concurrent task launch. * - * @param store_partition A store partition to add to the task as input + * Any task with at least one communicator will implicitly use concurrent task launch, so this + * method is to be used when the task needs a concurrent task launch for a reason unknown to + * Legate. + * + * @param concurrent A boolean value indicating whether the task needs a concurrent task launch */ - void add_input(LogicalStorePartition store_partition); + void set_concurrent(bool concurrent); /** - * @brief Adds a store partition to the task as output + * @brief Sets whether the task has side effects or not. * - * @param store_partition A store partition to add to the task as output + * A task is assumed to be free of side effects by default if the task only has scalar arguments. + * + * @param has_side_effect A boolean value indicating whether the task has side effects */ - void add_output(LogicalStorePartition store_partition); + void set_side_effect(bool has_side_effect); /** - * @brief Adds a store partition to the task for reductions + * @brief Sets whether the task can throw an exception or not. * - * @param store_partition A store partition to add to the task for reductions - * @param redop ID of the reduction operator + * @param can_throw_exception A boolean value indicating whether the task can throw an exception */ - void add_reduction(LogicalStorePartition store_partition, Legion::ReductionOpID redop); - - private: - void add_store(std::vector& store_args, - const LogicalStore& store, - std::shared_ptr partition); - - public: - void launch(detail::Strategy* strategy) override; - - public: - void add_to_solver(detail::ConstraintSolver& solver) override; - - private: - std::unique_ptr strategy_; -}; - -class Copy : public Operation { - private: - friend class detail::Runtime; - Copy(int64_t unique_id, mapping::MachineDesc&& machine); - - public: - void add_input(LogicalStore store); - void add_output(LogicalStore store); - void add_reduction(LogicalStore store, Legion::ReductionOpID redop); - void add_source_indirect(LogicalStore store); - void add_target_indirect(LogicalStore store); - - private: - void add_store(std::vector& store_args, - LogicalStore& store, - const Variable* partition_symbol); - void add_store(std::optional& store_arg, - LogicalStore& store, - const Variable* partition_symbol); - - public: - void set_source_indirect_out_of_range(bool flag); - void set_target_indirect_out_of_range(bool flag); - - public: - void launch(detail::Strategy* strategy) override; + void throws_exception(bool can_throw_exception); + /** + * @brief Requests a communicator for this task. + * + * @param name The name of the communicator to use for this task + */ + void add_communicator(const std::string& name); public: - void add_to_solver(detail::ConstraintSolver& solver) override; + ManualTask(const ManualTask&) = delete; + ManualTask(ManualTask&&) = default; + ManualTask& operator=(const ManualTask&) = delete; + ManualTask& operator=(ManualTask&&) = default; public: - std::string to_string() const override; + ~ManualTask(); private: - std::vector> constraints_{}; - std::optional source_indirect_{}; - std::optional target_indirect_{}; - bool source_indirect_out_of_range_{true}; - bool target_indirect_out_of_range_{true}; + friend class Runtime; + ManualTask(std::unique_ptr impl); + std::unique_ptr impl_; }; } // namespace legate diff --git a/src/core/partitioning/constraint.cc b/src/core/partitioning/constraint.cc index 6a7a295b0a..4d66d903b4 100644 --- a/src/core/partitioning/constraint.cc +++ b/src/core/partitioning/constraint.cc @@ -17,9 +17,9 @@ #include #include "core/data/scalar.h" +#include "core/operation/detail/operation.h" #include "core/partitioning/constraint.h" #include "core/partitioning/partition.h" -#include "core/runtime/operation.h" namespace legate { @@ -29,7 +29,7 @@ std::string Literal::to_string() const { return partition_->to_string(); } void Literal::find_partition_symbols(std::vector& partition_symbols) const {} -Variable::Variable(const Operation* op, int32_t id) : op_(op), id_(id) {} +Variable::Variable(const detail::Operation* op, int32_t id) : op_(op), id_(id) {} bool operator==(const Variable& lhs, const Variable& rhs) { diff --git a/src/core/partitioning/constraint.h b/src/core/partitioning/constraint.h index 75f0d96cca..98280d3766 100644 --- a/src/core/partitioning/constraint.h +++ b/src/core/partitioning/constraint.h @@ -30,13 +30,16 @@ * @brief Class definitions for partitioning constraint language */ +namespace legate::detail { +class Operation; +} // namespace legate::detail + namespace legate { class Alignment; class Broadcast; class Constraint; class Literal; -class Operation; class Partition; class Variable; @@ -115,7 +118,7 @@ class Literal : public Expr { */ class Variable : public Expr { public: - Variable(const Operation* op, int32_t id); + Variable(const detail::Operation* op, int32_t id); public: Variable(const Variable&) = default; @@ -138,10 +141,10 @@ class Variable : public Expr { const Variable* as_variable() const override { return this; } public: - const Operation* operation() const { return op_; } + const detail::Operation* operation() const { return op_; } private: - const Operation* op_; + const detail::Operation* op_; int32_t id_; }; diff --git a/src/core/partitioning/constraint_solver.cc b/src/core/partitioning/constraint_solver.cc index ac693110e1..f7641e8464 100644 --- a/src/core/partitioning/constraint_solver.cc +++ b/src/core/partitioning/constraint_solver.cc @@ -19,9 +19,9 @@ #include "legion.h" #include "core/data/detail/logical_store.h" +#include "core/operation/detail/operation.h" #include "core/partitioning/constraint.h" #include "core/partitioning/constraint_solver.h" -#include "core/runtime/operation.h" namespace legate { extern Legion::Logger log_legate; diff --git a/src/core/partitioning/partitioner.cc b/src/core/partitioning/partitioner.cc index a4544aef43..597f378bfb 100644 --- a/src/core/partitioning/partitioner.cc +++ b/src/core/partitioning/partitioner.cc @@ -18,11 +18,11 @@ #include "core/data/detail/logical_store.h" #include "core/data/logical_store.h" #include "core/data/scalar.h" +#include "core/operation/detail/operation.h" #include "core/partitioning/constraint.h" #include "core/partitioning/constraint_solver.h" #include "core/partitioning/partition.h" #include "core/runtime/detail/runtime.h" -#include "core/runtime/operation.h" namespace legate::detail { diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index 57aa627241..d87a22cc20 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -21,9 +21,11 @@ #include "core/data/detail/logical_store.h" #include "core/mapping/core_mapper.h" #include "core/mapping/default_mapper.h" +#include "core/operation/detail/copy.h" +#include "core/operation/detail/fill.h" +#include "core/operation/detail/task.h" +#include "core/operation/detail/task_launcher.h" #include "core/partitioning/partitioner.h" -#include "core/runtime/detail/fill.h" -#include "core/runtime/detail/task_launcher.h" #include "core/runtime/projection.h" #include "core/runtime/shard.h" #include "env_defaults.h" @@ -186,10 +188,11 @@ std::unique_ptr Runtime::create_copy() return std::unique_ptr(copy); } -void Runtime::issue_fill(legate::LogicalStore lhs, legate::LogicalStore value) +void Runtime::issue_fill(std::shared_ptr lhs, std::shared_ptr value) { auto machine = machine_manager_->get_machine(); - submit(std::unique_ptr(new Fill(lhs, value, next_unique_id_++, std::move(machine)))); + submit(std::unique_ptr( + new Fill(std::move(lhs), std::move(value), next_unique_id_++, std::move(machine)))); } void Runtime::flush_scheduling_window() diff --git a/src/core/runtime/detail/runtime.h b/src/core/runtime/detail/runtime.h index 22fdc589db..6248c9a64a 100644 --- a/src/core/runtime/detail/runtime.h +++ b/src/core/runtime/detail/runtime.h @@ -35,17 +35,17 @@ #include "core/utilities/multi_set.h" namespace legate { -class AutoTask; -class Copy; class LibraryContext; -class ManualTask; -class Operation; } // namespace legate namespace legate::detail { +class AutoTask; +class Copy; class LogicalRegionField; class LogicalStore; +class ManualTask; +class Operation; class Runtime { public: @@ -73,7 +73,7 @@ class Runtime { int64_t task_id, const Shape& launch_shape); std::unique_ptr create_copy(); - void issue_fill(legate::LogicalStore lhs, legate::LogicalStore value); + void issue_fill(std::shared_ptr lhs, std::shared_ptr value); void flush_scheduling_window(); void submit(std::unique_ptr op); diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index be3955e6f6..460f259ae9 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -16,9 +16,11 @@ #include "core/runtime/runtime.h" +#include "core/operation/detail/copy.h" +#include "core/operation/detail/operation.h" +#include "core/operation/detail/task.h" #include "core/runtime/context.h" #include "core/runtime/detail/runtime.h" -#include "core/runtime/operation.h" namespace legate { @@ -151,28 +153,33 @@ void Runtime::record_reduction_operator(int32_t type_uid, int32_t op_kind, int32 } // This function should be moved to the library context -std::unique_ptr Runtime::create_task(LibraryContext* library, int64_t task_id) +AutoTask Runtime::create_task(LibraryContext* library, int64_t task_id) { - return impl_->create_task(library, task_id); + return AutoTask(impl_->create_task(library, task_id)); } -std::unique_ptr Runtime::create_task(LibraryContext* library, - int64_t task_id, - const Shape& launch_shape) +ManualTask Runtime::create_task(LibraryContext* library, int64_t task_id, const Shape& launch_shape) { - return impl_->create_task(library, task_id, launch_shape); + return ManualTask(impl_->create_task(library, task_id, launch_shape)); } -std::unique_ptr Runtime::create_copy() { return impl_->create_copy(); } +Copy Runtime::create_copy() { return Copy(impl_->create_copy()); } -void Runtime::issue_fill(LogicalStore lhs, LogicalStore value) { impl_->issue_fill(lhs, value); } +void Runtime::issue_fill(LogicalStore lhs, LogicalStore value) +{ + impl_->issue_fill(lhs.impl(), value.impl()); +} void Runtime::issue_fill(LogicalStore lhs, const Scalar& value) { - issue_fill(lhs, create_store(value)); + issue_fill(std::move(lhs), create_store(value)); } -void Runtime::submit(std::unique_ptr op) { impl_->submit(std::move(op)); } +void Runtime::submit(AutoTask&& task) { impl_->submit(std::move(task.impl_)); } + +void Runtime::submit(ManualTask&& task) { impl_->submit(std::move(task.impl_)); } + +void Runtime::submit(Copy&& copy) { impl_->submit(std::move(copy.impl_)); } LogicalStore Runtime::create_store(std::unique_ptr type, int32_t dim) { diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 4baf525bcf..0443301ebd 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -22,6 +22,8 @@ #include "core/data/logical_store.h" #include "core/data/shape.h" #include "core/data/store.h" +#include "core/operation/copy.h" +#include "core/operation/task.h" #include "core/runtime/resource.h" #include "core/task/exception.h" #include "core/utilities/typedefs.h" @@ -86,11 +88,6 @@ struct Core { static bool has_socket_mem; }; -class AutoTask; -class Copy; -class ManualTask; -class Operation; - /** * @ingroup runtime * @brief Class that implements the Legate runtime @@ -149,7 +146,7 @@ class Runtime { * * @return Task object */ - std::unique_ptr create_task(LibraryContext* library, int64_t task_id); + AutoTask create_task(LibraryContext* library, int64_t task_id); /** * @brief Creates a ManualTask * @@ -159,15 +156,13 @@ class Runtime { * * @return Task object */ - std::unique_ptr create_task(LibraryContext* library, - int64_t task_id, - const Shape& launch_shape); + ManualTask create_task(LibraryContext* library, int64_t task_id, const Shape& launch_shape); /** * @brief Creates a Copy * * @return Copy object */ - std::unique_ptr create_copy(); + Copy create_copy(); /** * @brief Fills a given store with a constant * @@ -183,14 +178,32 @@ class Runtime { */ void issue_fill(LogicalStore lhs, const Scalar& value); /** - * @brief Submits an operation for execution + * @brief Submits an AutoTask for execution + * + * Each submitted operation goes through multiple pipeline steps to eventually get scheduled + * for execution. It's not guaranteed that the submitted operation starts executing immediately. + * + * @param task An AutoTask to execute + */ + void submit(AutoTask&& task); + /** + * @brief Submits a ManualTask for execution + * + * Each submitted operation goes through multiple pipeline steps to eventually get scheduled + * for execution. It's not guaranteed that the submitted operation starts executing immediately. + * + * @param task A ManualTask to execute + */ + void submit(ManualTask&& task); + /** + * @brief Submits a copy for execution * * Each submitted operation goes through multiple pipeline steps to eventually get scheduled * for execution. It's not guaranteed that the submitted operation starts executing immediately. * - * @param op Operation to execute + * @param copy A Copy to execute */ - void submit(std::unique_ptr op); + void submit(Copy&& copy); public: /** diff --git a/src/legate.h b/src/legate.h index 9305dce523..f3b1564918 100644 --- a/src/legate.h +++ b/src/legate.h @@ -30,9 +30,10 @@ #include "core/data/store.h" #include "core/legate_c.h" #include "core/mapping/mapping.h" +#include "core/operation/copy.h" +#include "core/operation/task.h" #include "core/partitioning/constraint.h" #include "core/partitioning/partition.h" -#include "core/runtime/operation.h" #include "core/runtime/runtime.h" #include "core/runtime/tracker.h" #include "core/task/exception.h" diff --git a/tests/cpp/integration/alignment_constraints.cc b/tests/cpp/integration/alignment_constraints.cc index e5ad231ed6..5224e960c7 100644 --- a/tests/cpp/integration/alignment_constraints.cc +++ b/tests/cpp/integration/alignment_constraints.cc @@ -101,16 +101,16 @@ void test_alignment() auto store3 = runtime->create_store(extents, legate::int64()); auto task = runtime->create_task(context, ALIGNMENT_TESTER + extents.size()); - auto part1 = task->declare_partition(); - auto part2 = task->declare_partition(); - auto part3 = task->declare_partition(); + auto part1 = task.declare_partition(); + auto part2 = task.declare_partition(); + auto part3 = task.declare_partition(); - task->add_output(store1, part1); - task->add_output(store2, part2); - task->add_output(store3, part3); + task.add_output(store1, part1); + task.add_output(store2, part2); + task.add_output(store3, part3); - task->add_constraint(legate::align(part1, part2)); - task->add_constraint(legate::align(part2, part3)); + task.add_constraint(legate::align(part1, part2)); + task.add_constraint(legate::align(part2, part3)); runtime->submit(std::move(task)); }; @@ -130,16 +130,16 @@ void test_alignment_and_broadcast() auto store2 = runtime->create_store(extents, legate::int64()); auto task = runtime->create_task(context, ALIGNMENT_BROADCAST_TESTER + extents.size()); - auto part1 = task->declare_partition(); - auto part2 = task->declare_partition(); + auto part1 = task.declare_partition(); + auto part2 = task.declare_partition(); - task->add_output(store1, part1); - task->add_output(store2, part2); + task.add_output(store1, part1); + task.add_output(store2, part2); - task->add_scalar_arg(legate::Scalar(extents[0])); + task.add_scalar_arg(legate::Scalar(extents[0])); - task->add_constraint(legate::align(part1, part2)); - task->add_constraint(legate::broadcast(part1, {0})); + task.add_constraint(legate::align(part1, part2)); + task.add_constraint(legate::broadcast(part1, {0})); runtime->submit(std::move(task)); }; @@ -155,8 +155,8 @@ void initialize(legate::LogicalStore store) auto context = runtime->find_library(library_name); auto task = runtime->create_task(context, INIT); - auto part = task->declare_partition(); - task->add_output(store, part); + auto part = task.declare_partition(); + task.add_output(store, part); runtime->submit(std::move(task)); } @@ -168,13 +168,13 @@ void test_alignment_transformed() auto launch_tester = [&](auto store1, auto store2) { auto task = runtime->create_task(context, TRANSFORMED_TESTER + store1.dim()); - auto part1 = task->declare_partition(); - auto part2 = task->declare_partition(); + auto part1 = task.declare_partition(); + auto part2 = task.declare_partition(); - task->add_input(store1, part1); - task->add_input(store2, part2); + task.add_input(store1, part1); + task.add_input(store2, part2); - task->add_constraint(legate::align(part1, part2)); + task.add_constraint(legate::align(part1, part2)); runtime->submit(std::move(task)); }; diff --git a/tests/cpp/integration/broadcast_constraints.cc b/tests/cpp/integration/broadcast_constraints.cc index 3dcb929c3e..dcd34a7c4b 100644 --- a/tests/cpp/integration/broadcast_constraints.cc +++ b/tests/cpp/integration/broadcast_constraints.cc @@ -67,12 +67,12 @@ void test_normal_store() for (auto dim : dims) extents[dim] = EXT_LARGE; auto store = runtime->create_store(extents, legate::int64()); auto task = runtime->create_task(context, TESTER); - auto part = task->declare_partition(); - task->add_output(store, part); - task->add_scalar_arg(legate::Scalar(EXT_LARGE)); - task->add_scalar_arg(legate::Scalar(dims)); - task->add_scalar_arg(legate::Scalar(false)); - task->add_constraint(legate::broadcast(part, dims)); + auto part = task.declare_partition(); + task.add_output(store, part); + task.add_scalar_arg(legate::Scalar(EXT_LARGE)); + task.add_scalar_arg(legate::Scalar(dims)); + task.add_scalar_arg(legate::Scalar(false)); + task.add_constraint(legate::broadcast(part, dims)); runtime->submit(std::move(task)); }; @@ -92,8 +92,8 @@ void test_promoted_store() auto initialize = [&](auto store) { auto task = runtime->create_task(context, INITIALIZER); - auto part = task->declare_partition(); - task->add_output(store, part); + auto part = task.declare_partition(); + task.add_output(store, part); runtime->submit(std::move(task)); }; @@ -104,12 +104,12 @@ void test_promoted_store() initialize(store); auto task = runtime->create_task(context, TESTER); - auto part = task->declare_partition(); - task->add_input(store.promote(2, EXT_LARGE), part); - task->add_scalar_arg(legate::Scalar(EXT_LARGE)); - task->add_scalar_arg(legate::Scalar(std::vector{dim})); - task->add_scalar_arg(legate::Scalar(true)); - task->add_constraint(legate::broadcast(part, {dim})); + auto part = task.declare_partition(); + task.add_input(store.promote(2, EXT_LARGE), part); + task.add_scalar_arg(legate::Scalar(EXT_LARGE)); + task.add_scalar_arg(legate::Scalar(std::vector{dim})); + task.add_scalar_arg(legate::Scalar(true)); + task.add_constraint(legate::broadcast(part, {dim})); runtime->submit(std::move(task)); }; diff --git a/tests/cpp/integration/copy_failure.cc b/tests/cpp/integration/copy_failure.cc index fc63fbf689..0bf78ef54c 100644 --- a/tests/cpp/integration/copy_failure.cc +++ b/tests/cpp/integration/copy_failure.cc @@ -32,9 +32,9 @@ void test_input_output_failure() auto out_store = runtime->create_store(extents, legate::int64()); // fill input store with some values auto copy = runtime->create_copy(); - copy->add_input(in_store1); - copy->add_input(in_store2); - copy->add_output(out_store); + copy.add_input(in_store1); + copy.add_input(in_store2); + copy.add_output(out_store); EXPECT_THROW(runtime->submit(std::move(copy)), std::runtime_error); } @@ -52,11 +52,11 @@ void test_indirect_failure() auto indirect_store = runtime->create_store(extents, legate::int64()); auto copy = runtime->create_copy(); - copy->add_input(in_store1); - copy->add_input(in_store2); - copy->add_output(out_store1); - copy->add_output(out_store2); - copy->add_target_indirect(indirect_store); + copy.add_input(in_store1); + copy.add_input(in_store2); + copy.add_output(out_store1); + copy.add_output(out_store2); + copy.add_target_indirect(indirect_store); EXPECT_THROW(runtime->submit(std::move(copy)), std::runtime_error); } @@ -72,33 +72,33 @@ void test_shape_check_failure() { auto copy = runtime->create_copy(); - copy->add_input(store1); - copy->add_output(store2); + copy.add_input(store1); + copy.add_output(store2); EXPECT_THROW(runtime->submit(std::move(copy)), std::runtime_error); } { auto copy = runtime->create_copy(); - copy->add_input(store1); - copy->add_source_indirect(store2); - copy->add_output(store3); + copy.add_input(store1); + copy.add_source_indirect(store2); + copy.add_output(store3); EXPECT_THROW(runtime->submit(std::move(copy)), std::runtime_error); } { auto copy = runtime->create_copy(); - copy->add_input(store1); - copy->add_target_indirect(store2); - copy->add_output(store3); + copy.add_input(store1); + copy.add_target_indirect(store2); + copy.add_output(store3); EXPECT_THROW(runtime->submit(std::move(copy)), std::runtime_error); } { auto copy = runtime->create_copy(); - copy->add_input(store1); - copy->add_source_indirect(store2); - copy->add_target_indirect(store3); - copy->add_output(store1); + copy.add_input(store1); + copy.add_source_indirect(store2); + copy.add_target_indirect(store3); + copy.add_output(store1); EXPECT_THROW(runtime->submit(std::move(copy)), std::runtime_error); } } diff --git a/tests/cpp/integration/copy_gather.cc b/tests/cpp/integration/copy_gather.cc index c47013a805..ca6489f5a1 100644 --- a/tests/cpp/integration/copy_gather.cc +++ b/tests/cpp/integration/copy_gather.cc @@ -100,16 +100,16 @@ void check_gather_output(legate::LibraryContext* context, auto machine = runtime->get_machine(); int32_t task_id = CHECK_GATHER_TASK + ind.dim() * TEST_MAX_DIM + src.dim(); auto task = runtime->create_task(context, task_id); - auto src_part = task->declare_partition(); - auto tgt_part = task->declare_partition(); - auto ind_part = task->declare_partition(); - task->add_input(src, src_part); - task->add_input(tgt, tgt_part); - task->add_input(ind, ind_part); + auto src_part = task.declare_partition(); + auto tgt_part = task.declare_partition(); + auto ind_part = task.declare_partition(); + task.add_input(src, src_part); + task.add_input(tgt, tgt_part); + task.add_input(ind, ind_part); - task->add_constraint(legate::broadcast(src_part, legate::from_range(src.dim()))); - task->add_constraint(legate::broadcast(tgt_part, legate::from_range(tgt.dim()))); - task->add_constraint(legate::broadcast(ind_part, legate::from_range(ind.dim()))); + task.add_constraint(legate::broadcast(src_part, legate::from_range(src.dim()))); + task.add_constraint(legate::broadcast(tgt_part, legate::from_range(tgt.dim()))); + task.add_constraint(legate::broadcast(ind_part, legate::from_range(ind.dim()))); runtime->submit(std::move(task)); } @@ -145,9 +145,9 @@ void test_gather(const GatherSpec& spec) fill_indirect(context, ind, src); auto copy = runtime->create_copy(); - copy->add_input(src); - copy->add_output(tgt); - copy->add_source_indirect(ind); + copy.add_input(src); + copy.add_output(tgt); + copy.add_source_indirect(ind); runtime->submit(std::move(copy)); check_gather_output(context, src, tgt, ind); diff --git a/tests/cpp/integration/copy_gather_scatter.cc b/tests/cpp/integration/copy_gather_scatter.cc index b878ff4402..f5478a315a 100644 --- a/tests/cpp/integration/copy_gather_scatter.cc +++ b/tests/cpp/integration/copy_gather_scatter.cc @@ -137,20 +137,20 @@ void check_gather_scatter_output(legate::LibraryContext* context, auto task = runtime->create_task(context, task_id); - auto src_part = task->declare_partition(); - auto tgt_part = task->declare_partition(); - auto src_ind_part = task->declare_partition(); - auto tgt_ind_part = task->declare_partition(); - task->add_input(src, src_part); - task->add_input(tgt, tgt_part); - task->add_input(src_ind, src_ind_part); - task->add_input(tgt_ind, tgt_ind_part); - task->add_scalar_arg(init); - - task->add_constraint(legate::broadcast(src_part, legate::from_range(src.dim()))); - task->add_constraint(legate::broadcast(tgt_part, legate::from_range(tgt.dim()))); - task->add_constraint(legate::broadcast(src_ind_part, legate::from_range(src_ind.dim()))); - task->add_constraint(legate::broadcast(tgt_ind_part, legate::from_range(tgt_ind.dim()))); + auto src_part = task.declare_partition(); + auto tgt_part = task.declare_partition(); + auto src_ind_part = task.declare_partition(); + auto tgt_ind_part = task.declare_partition(); + task.add_input(src, src_part); + task.add_input(tgt, tgt_part); + task.add_input(src_ind, src_ind_part); + task.add_input(tgt_ind, tgt_ind_part); + task.add_scalar_arg(init); + + task.add_constraint(legate::broadcast(src_part, legate::from_range(src.dim()))); + task.add_constraint(legate::broadcast(tgt_part, legate::from_range(tgt.dim()))); + task.add_constraint(legate::broadcast(src_ind_part, legate::from_range(src_ind.dim()))); + task.add_constraint(legate::broadcast(tgt_ind_part, legate::from_range(tgt_ind.dim()))); runtime->submit(std::move(task)); } @@ -175,10 +175,10 @@ void test_gather_scatter(const GatherScatterSpec& spec) runtime->issue_fill(tgt, spec.init); auto copy = runtime->create_copy(); - copy->add_input(src); - copy->add_output(tgt); - copy->add_source_indirect(src_ind); - copy->add_target_indirect(tgt_ind); + copy.add_input(src); + copy.add_output(tgt); + copy.add_source_indirect(src_ind); + copy.add_target_indirect(tgt_ind); runtime->submit(std::move(copy)); check_gather_scatter_output(context, src, tgt, src_ind, tgt_ind, spec.init); diff --git a/tests/cpp/integration/copy_normal.cc b/tests/cpp/integration/copy_normal.cc index fb0278f427..013f001ebd 100644 --- a/tests/cpp/integration/copy_normal.cc +++ b/tests/cpp/integration/copy_normal.cc @@ -76,12 +76,12 @@ void check_output(legate::LibraryContext* context, auto machine = runtime->get_machine(); auto task = runtime->create_task(context, CHECK_TASK + tgt.dim()); - auto src_part = task->declare_partition(); - auto tgt_part = task->declare_partition(); + auto src_part = task.declare_partition(); + auto tgt_part = task.declare_partition(); - task->add_input(src, src_part); - task->add_input(tgt, tgt_part); - task->add_constraint(legate::align(src_part, tgt_part)); + task.add_input(src, src_part); + task.add_input(tgt, tgt_part); + task.add_constraint(legate::align(src_part, tgt_part)); runtime->submit(std::move(task)); } @@ -112,8 +112,8 @@ void test_normal_copies(const std::vector specs) } auto copy = runtime->create_copy(); - for (auto& input : inputs) copy->add_input(input); - for (auto& output : outputs) copy->add_output(output); + for (auto& input : inputs) copy.add_input(input); + for (auto& output : outputs) copy.add_output(output); runtime->submit(std::move(copy)); // check the result of copy diff --git a/tests/cpp/integration/copy_scatter.cc b/tests/cpp/integration/copy_scatter.cc index 14bf080819..e1e2454354 100644 --- a/tests/cpp/integration/copy_scatter.cc +++ b/tests/cpp/integration/copy_scatter.cc @@ -139,17 +139,17 @@ void check_scatter_output(legate::LibraryContext* context, auto task = runtime->create_task(context, task_id); - auto src_part = task->declare_partition(); - auto tgt_part = task->declare_partition(); - auto ind_part = task->declare_partition(); - task->add_input(src, src_part); - task->add_input(tgt, tgt_part); - task->add_input(ind, ind_part); - task->add_scalar_arg(init); - - task->add_constraint(legate::broadcast(src_part, legate::from_range(src.dim()))); - task->add_constraint(legate::broadcast(tgt_part, legate::from_range(tgt.dim()))); - task->add_constraint(legate::broadcast(ind_part, legate::from_range(ind.dim()))); + auto src_part = task.declare_partition(); + auto tgt_part = task.declare_partition(); + auto ind_part = task.declare_partition(); + task.add_input(src, src_part); + task.add_input(tgt, tgt_part); + task.add_input(ind, ind_part); + task.add_scalar_arg(init); + + task.add_constraint(legate::broadcast(src_part, legate::from_range(src.dim()))); + task.add_constraint(legate::broadcast(tgt_part, legate::from_range(tgt.dim()))); + task.add_constraint(legate::broadcast(ind_part, legate::from_range(ind.dim()))); runtime->submit(std::move(task)); } @@ -172,9 +172,9 @@ void test_scatter(const ScatterSpec& spec) runtime->issue_fill(tgt, spec.init); auto copy = runtime->create_copy(); - copy->add_input(src); - copy->add_output(tgt); - copy->add_target_indirect(ind); + copy.add_input(src); + copy.add_output(tgt); + copy.add_target_indirect(ind); runtime->submit(std::move(copy)); check_scatter_output(context, src, tgt, ind, spec.init); diff --git a/tests/cpp/integration/copy_util.inl b/tests/cpp/integration/copy_util.inl index e7183221e7..877ef47565 100644 --- a/tests/cpp/integration/copy_util.inl +++ b/tests/cpp/integration/copy_util.inl @@ -120,8 +120,8 @@ void fill_input(legate::LibraryContext* context, auto runtime = legate::Runtime::get_runtime(); auto machine = runtime->get_machine(); auto task = runtime->create_task(context, FILL_TASK + src.dim()); - task->add_output(src, task->declare_partition()); - task->add_scalar_arg(value); + task.add_output(src, task.declare_partition()); + task.add_scalar_arg(value); runtime->submit(std::move(task)); } @@ -133,24 +133,24 @@ void fill_indirect(legate::LibraryContext* context, auto machine = runtime->get_machine(); int32_t task_id = FILL_INDIRECT_TASK + ind.dim() * TEST_MAX_DIM + data.dim(); auto task = runtime->create_task(context, task_id); - auto part = task->declare_partition(); - task->add_output(ind, part); + auto part = task.declare_partition(); + task.add_output(ind, part); // Technically indirection fields for gather coipes can have repeated points // and thus be initialized in parallel, but we always serialize the // initialization to simplify the logic - task->add_constraint(legate::broadcast(part, legate::from_range(ind.dim()))); + task.add_constraint(legate::broadcast(part, legate::from_range(ind.dim()))); auto domain = legate::to_domain(data.extents()); switch (data.dim()) { case 1: { - task->add_scalar_arg(legate::Rect<1>(domain)); + task.add_scalar_arg(legate::Rect<1>(domain)); break; } case 2: { - task->add_scalar_arg(legate::Rect<2>(domain)); + task.add_scalar_arg(legate::Rect<2>(domain)); break; } case 3: { - task->add_scalar_arg(legate::Rect<3>(domain)); + task.add_scalar_arg(legate::Rect<3>(domain)); break; } default: { diff --git a/tests/cpp/integration/cpu_communicator.cc b/tests/cpp/integration/cpu_communicator.cc index 445a93497e..fe621def19 100644 --- a/tests/cpp/integration/cpu_communicator.cc +++ b/tests/cpp/integration/cpu_communicator.cc @@ -63,9 +63,9 @@ void test_cpu_communicator_auto(int32_t ndim) auto store = runtime->create_store(std::vector(ndim, SIZE), legate::int32()); auto task = runtime->create_task(context, CPU_COMM_TESTER); - auto part = task->declare_partition(); - task->add_output(store, part); - task->add_communicator("cpu"); + auto part = task.declare_partition(); + task.add_output(store, part); + task.add_communicator("cpu"); runtime->submit(std::move(task)); } @@ -85,8 +85,8 @@ void test_cpu_communicator_manual(int32_t ndim) auto part = store.partition_by_tiling(std::move(tile_shape)); auto task = runtime->create_task(context, CPU_COMM_TESTER, std::move(launch_shape)); - task->add_output(part); - task->add_communicator("cpu"); + task.add_output(part); + task.add_communicator("cpu"); runtime->submit(std::move(task)); } diff --git a/tests/cpp/integration/exception.cc b/tests/cpp/integration/exception.cc index d712347990..9ee1da566f 100644 --- a/tests/cpp/integration/exception.cc +++ b/tests/cpp/integration/exception.cc @@ -72,9 +72,9 @@ void test_single() auto issue_task = [&](int32_t index, bool raise) { auto task = runtime->create_task(context, EXCEPTION_TASK); - task->add_scalar_arg(legate::Scalar(raise)); - task->add_scalar_arg(legate::Scalar(index)); - task->throws_exception(true); + task.add_scalar_arg(legate::Scalar(raise)); + task.add_scalar_arg(legate::Scalar(index)); + task.throws_exception(true); runtime->submit(std::move(task)); }; @@ -139,22 +139,22 @@ void test_multi(bool use_auto_task) auto store = runtime->create_store({SIZE, SIZE}, legate::int64()); if (use_auto_task) { auto task = runtime->create_task(context, EXCEPTION_TASK); - auto part = task->declare_partition(); + auto part = task.declare_partition(); // Dummy store argument to trigger parallelization - task->add_output(store, part); - task->throws_exception(true); - task->add_scalar_arg(legate::Scalar(true)); - task->add_scalar_arg(legate::Scalar(12345)); + task.add_output(store, part); + task.throws_exception(true); + task.add_scalar_arg(legate::Scalar(true)); + task.add_scalar_arg(legate::Scalar(12345)); runtime->submit(std::move(task)); } else { auto task = runtime->create_task(context, EXCEPTION_TASK, {2, 2}); auto part = store.partition_by_tiling({SIZE / 2, SIZE / 2}); // Dummy store argument to trigger parallelization - task->add_output(part); - task->throws_exception(true); - task->add_scalar_arg(legate::Scalar(true)); - task->add_scalar_arg(legate::Scalar(12345)); + task.add_output(part); + task.throws_exception(true); + task.add_scalar_arg(legate::Scalar(true)); + task.add_scalar_arg(legate::Scalar(12345)); runtime->submit(std::move(task)); } @@ -171,9 +171,9 @@ void test_pending() runtime->set_max_pending_exceptions(2); auto task = runtime->create_task(context, EXCEPTION_TASK); - task->throws_exception(true); - task->add_scalar_arg(legate::Scalar(false)); - task->add_scalar_arg(legate::Scalar(12345)); + task.throws_exception(true); + task.add_scalar_arg(legate::Scalar(false)); + task.add_scalar_arg(legate::Scalar(12345)); runtime->submit(std::move(task)); diff --git a/tests/cpp/integration/fill.cc b/tests/cpp/integration/fill.cc index e0b3a6573b..435d920a06 100644 --- a/tests/cpp/integration/fill.cc +++ b/tests/cpp/integration/fill.cc @@ -94,9 +94,9 @@ void check_output(legate::LogicalStore store, int64_t value) auto context = runtime->find_library(library_name); auto task = runtime->create_task(context, CHECK_TASK + store.dim()); - auto part = task->declare_partition(); - task->add_input(store, part); - task->add_scalar_arg(value); + auto part = task.declare_partition(); + task.add_input(store, part); + task.add_scalar_arg(value); runtime->submit(std::move(task)); } @@ -109,11 +109,11 @@ void check_output_slice(legate::LogicalStore store, auto context = runtime->find_library(library_name); auto task = runtime->create_task(context, CHECK_SLICE_TASK + store.dim()); - auto part = task->declare_partition(); - task->add_input(store, part); - task->add_scalar_arg(value_in_slice); - task->add_scalar_arg(value_outside_slice); - task->add_scalar_arg(offset); + auto part = task.declare_partition(); + task.add_input(store, part); + task.add_scalar_arg(value_in_slice); + task.add_scalar_arg(value_outside_slice); + task.add_scalar_arg(offset); runtime->submit(std::move(task)); } diff --git a/tests/cpp/integration/inline_map.cc b/tests/cpp/integration/inline_map.cc index e248dea8e2..d8dcb527ad 100644 --- a/tests/cpp/integration/inline_map.cc +++ b/tests/cpp/integration/inline_map.cc @@ -86,8 +86,8 @@ void test_inline_map_and_task(legate::Runtime* runtime, legate::LibraryContext* acc[2] = 42; } auto task = runtime->create_task(context, ADDER, {1}); - task->add_input(l_store); - task->add_output(l_store); + task.add_input(l_store); + task.add_output(l_store); runtime->submit(std::move(task)); auto p_store = l_store.get_physical_store(); auto acc = p_store->read_accessor(); diff --git a/tests/cpp/integration/inout.cc b/tests/cpp/integration/inout.cc index 06e8429117..73001785cf 100644 --- a/tests/cpp/integration/inout.cc +++ b/tests/cpp/integration/inout.cc @@ -30,8 +30,8 @@ void test_inout() runtime->issue_fill(store, legate::Scalar(int64_t(0))); auto task = runtime->create_task(context, task::simple::HELLO); - task->add_input(store, task->find_or_declare_partition(store)); - task->add_output(store, task->find_or_declare_partition(store)); + task.add_input(store, task.find_or_declare_partition(store)); + task.add_output(store, task.find_or_declare_partition(store)); runtime->submit(std::move(task)); } diff --git a/tests/cpp/integration/machine_scope.cc b/tests/cpp/integration/machine_scope.cc index 3ab058d990..e1b277c4f3 100644 --- a/tests/cpp/integration/machine_scope.cc +++ b/tests/cpp/integration/machine_scope.cc @@ -68,35 +68,35 @@ void test_scoping(legate::LibraryContext* context) auto store = runtime->create_store({5, 5}, legate::int64()); auto machine = runtime->get_machine(); auto task = runtime->create_task(context, MULTI_VARIANT); - auto part = task->declare_partition(); - task->add_output(store, part); - task->add_scalar_arg(machine.count()); + auto part = task.declare_partition(); + task.add_output(store, part); + task.add_scalar_arg(machine.count()); runtime->submit(std::move(task)); if (machine.count(legate::mapping::TaskTarget::CPU) > 0) { legate::MachineTracker tracker(machine.only(legate::mapping::TaskTarget::CPU)); auto task_scoped = runtime->create_task(context, MULTI_VARIANT); - auto part_scoped = task_scoped->declare_partition(); - task_scoped->add_output(store, part_scoped); - task_scoped->add_scalar_arg(machine.count(legate::mapping::TaskTarget::CPU)); + auto part_scoped = task_scoped.declare_partition(); + task_scoped.add_output(store, part_scoped); + task_scoped.add_scalar_arg(machine.count(legate::mapping::TaskTarget::CPU)); runtime->submit(std::move(task_scoped)); } if (machine.count(legate::mapping::TaskTarget::OMP) > 0) { legate::MachineTracker tracker(machine.only(legate::mapping::TaskTarget::OMP)); auto task_scoped = runtime->create_task(context, MULTI_VARIANT); - auto part_scoped = task_scoped->declare_partition(); - task_scoped->add_output(store, part_scoped); - task_scoped->add_scalar_arg(machine.count(legate::mapping::TaskTarget::OMP)); + auto part_scoped = task_scoped.declare_partition(); + task_scoped.add_output(store, part_scoped); + task_scoped.add_scalar_arg(machine.count(legate::mapping::TaskTarget::OMP)); runtime->submit(std::move(task_scoped)); } if (machine.count(legate::mapping::TaskTarget::GPU) > 0) { legate::MachineTracker tracker(machine.only(legate::mapping::TaskTarget::GPU)); auto task_scoped = runtime->create_task(context, MULTI_VARIANT); - auto part_scoped = task_scoped->declare_partition(); - task_scoped->add_output(store, part_scoped); - task_scoped->add_scalar_arg(machine.count(legate::mapping::TaskTarget::GPU)); + auto part_scoped = task_scoped.declare_partition(); + task_scoped.add_output(store, part_scoped); + task_scoped.add_scalar_arg(machine.count(legate::mapping::TaskTarget::GPU)); runtime->submit(std::move(task_scoped)); } } @@ -107,17 +107,17 @@ void test_cpu_only(legate::LibraryContext* context) auto store = runtime->create_store({5, 5}, legate::int64()); auto machine = runtime->get_machine(); auto task = runtime->create_task(context, CPU_VARIANT); - auto part = task->declare_partition(); - task->add_output(store, part); - task->add_scalar_arg(machine.count(legate::mapping::TaskTarget::CPU)); + auto part = task.declare_partition(); + task.add_output(store, part); + task.add_scalar_arg(machine.count(legate::mapping::TaskTarget::CPU)); runtime->submit(std::move(task)); if (machine.count(legate::mapping::TaskTarget::CPU) > 0) { legate::MachineTracker tracker(machine.only(legate::mapping::TaskTarget::CPU)); auto task_scoped = runtime->create_task(context, CPU_VARIANT); - auto part_scoped = task_scoped->declare_partition(); - task_scoped->add_output(store, part_scoped); - task_scoped->add_scalar_arg(machine.count(legate::mapping::TaskTarget::CPU)); + auto part_scoped = task_scoped.declare_partition(); + task_scoped.add_output(store, part_scoped); + task_scoped.add_scalar_arg(machine.count(legate::mapping::TaskTarget::CPU)); runtime->submit(std::move(task_scoped)); } diff --git a/tests/cpp/integration/manual_simple.cc b/tests/cpp/integration/manual_simple.cc index a4750c5905..a09de7ace4 100644 --- a/tests/cpp/integration/manual_simple.cc +++ b/tests/cpp/integration/manual_simple.cc @@ -25,8 +25,8 @@ void test_auto_task(legate::LibraryContext* context, legate::LogicalStore store) { auto runtime = legate::Runtime::get_runtime(); auto task = runtime->create_task(context, task::simple::HELLO); - auto part = task->declare_partition(); - task->add_output(store, part); + auto part = task.declare_partition(); + task.add_output(store, part); runtime->submit(std::move(task)); } @@ -35,7 +35,7 @@ void test_manual_task(legate::LibraryContext* context, legate::LogicalStore stor auto runtime = legate::Runtime::get_runtime(); auto task = runtime->create_task(context, task::simple::HELLO, {3, 3}); auto part = store.partition_by_tiling({2, 2}); - task->add_output(part); + task.add_output(part); runtime->submit(std::move(task)); } diff --git a/tests/cpp/integration/multi_scalar_out.cc b/tests/cpp/integration/multi_scalar_out.cc index 57fe24cc90..3a97140f37 100644 --- a/tests/cpp/integration/multi_scalar_out.cc +++ b/tests/cpp/integration/multi_scalar_out.cc @@ -27,10 +27,10 @@ void test_writer_auto(legate::LibraryContext* context, { auto runtime = legate::Runtime::get_runtime(); auto task = runtime->create_task(context, task::simple::WRITER); - auto part1 = task->declare_partition(); - auto part2 = task->declare_partition(); - task->add_output(scalar1, part1); - task->add_output(scalar2, part2); + auto part1 = task.declare_partition(); + auto part2 = task.declare_partition(); + task.add_output(scalar1, part1); + task.add_output(scalar2, part2); runtime->submit(std::move(task)); } @@ -41,14 +41,14 @@ void test_reducer_auto(legate::LibraryContext* context, { auto runtime = legate::Runtime::get_runtime(); auto task = runtime->create_task(context, task::simple::REDUCER); - auto part1 = task->declare_partition(); - auto part2 = task->declare_partition(); - auto part3 = task->declare_partition(); + auto part1 = task.declare_partition(); + auto part2 = task.declare_partition(); + auto part3 = task.declare_partition(); auto redop1 = scalar1.type().find_reduction_operator(legate::ReductionOpKind::ADD); auto redop2 = scalar2.type().find_reduction_operator(legate::ReductionOpKind::MUL); - task->add_reduction(scalar1, redop1, part1); - task->add_reduction(scalar2, redop2, part2); - task->add_output(store, part3); + task.add_reduction(scalar1, redop1, part1); + task.add_reduction(scalar2, redop2, part2); + task.add_output(store, part3); runtime->submit(std::move(task)); } @@ -60,8 +60,8 @@ void test_reducer_manual(legate::LibraryContext* context, auto task = runtime->create_task(context, task::simple::REDUCER, legate::Shape({2})); auto redop1 = scalar1.type().find_reduction_operator(legate::ReductionOpKind::ADD); auto redop2 = scalar2.type().find_reduction_operator(legate::ReductionOpKind::MUL); - task->add_reduction(scalar1, redop1); - task->add_reduction(scalar2, redop2); + task.add_reduction(scalar1, redop1); + task.add_reduction(scalar2, redop2); runtime->submit(std::move(task)); } diff --git a/tests/cpp/integration/nccl.cu b/tests/cpp/integration/nccl.cu index c15b6d3fae..a233515349 100644 --- a/tests/cpp/integration/nccl.cu +++ b/tests/cpp/integration/nccl.cu @@ -75,10 +75,10 @@ void test_nccl_auto(int32_t ndim) auto store = runtime->create_store(std::vector(ndim, SIZE), legate::int32()); auto task = runtime->create_task(context, NCCL_TESTER); - auto part = task->declare_partition(); - task->add_output(store, part); - task->add_communicator("cpu"); // This requested will be ignored - task->add_communicator("nccl"); + auto part = task.declare_partition(); + task.add_output(store, part); + task.add_communicator("cpu"); // This requested will be ignored + task.add_communicator("nccl"); runtime->submit(std::move(task)); } @@ -98,9 +98,9 @@ void test_nccl_manual(int32_t ndim) auto part = store.partition_by_tiling(std::move(tile_shape)); auto task = runtime->create_task(context, NCCL_TESTER, std::move(launch_shape)); - task->add_output(part); - task->add_communicator("cpu"); // This requested will be ignored - task->add_communicator("nccl"); + task.add_output(part); + task.add_communicator("cpu"); // This requested will be ignored + task.add_communicator("nccl"); runtime->submit(std::move(task)); } diff --git a/tests/cpp/integration/provenance.cc b/tests/cpp/integration/provenance.cc index 00a9b3c2c5..859268bd07 100644 --- a/tests/cpp/integration/provenance.cc +++ b/tests/cpp/integration/provenance.cc @@ -53,7 +53,7 @@ void test_manual_provenance(legate::LibraryContext* context) runtime->impl()->provenance_manager()->set_provenance(provenance); // auto task auto task = runtime->create_task(context, PROVENANCE); - task->add_scalar_arg(legate::Scalar(provenance)); + task.add_scalar_arg(legate::Scalar(provenance)); runtime->submit(std::move(task)); } @@ -65,7 +65,7 @@ void test_push_provenance(legate::LibraryContext* context) EXPECT_EQ(runtime->impl()->provenance_manager()->get_provenance(), provenance); // auto task auto task = runtime->create_task(context, PROVENANCE); - task->add_scalar_arg(legate::Scalar(provenance)); + task.add_scalar_arg(legate::Scalar(provenance)); runtime->submit(std::move(task)); } @@ -78,7 +78,7 @@ void test_pop_provenance(legate::LibraryContext* context) // auto task auto task = runtime->create_task(context, PROVENANCE); std::string provenance = ""; - task->add_scalar_arg(legate::Scalar(provenance)); + task.add_scalar_arg(legate::Scalar(provenance)); runtime->submit(std::move(task)); } @@ -99,7 +99,7 @@ void test_clear_provenance(legate::LibraryContext* context) // auto task auto task = runtime->create_task(context, PROVENANCE); std::string provenance = ""; - task->add_scalar_arg(legate::Scalar(provenance)); + task.add_scalar_arg(legate::Scalar(provenance)); runtime->submit(std::move(task)); } @@ -110,7 +110,7 @@ void test_provenance_tracker(legate::LibraryContext* context) // auto task auto task = runtime->create_task(context, PROVENANCE); std::string provenance = "provenance.cc:108"; - task->add_scalar_arg(legate::Scalar(provenance)); + task.add_scalar_arg(legate::Scalar(provenance)); runtime->submit(std::move(task)); } @@ -122,7 +122,7 @@ void test_nested_provenance_tracker(legate::LibraryContext* context) auto runtime = legate::Runtime::get_runtime(); auto task = runtime->create_task(context, PROVENANCE); std::string provenance = "provenance.cc:119"; - task->add_scalar_arg(legate::Scalar(provenance)); + task.add_scalar_arg(legate::Scalar(provenance)); runtime->submit(std::move(task)); } @@ -132,7 +132,7 @@ void test_manual_tracker(legate::LibraryContext* context) auto runtime = legate::Runtime::get_runtime(); // auto task auto task = runtime->create_task(context, PROVENANCE); - task->add_scalar_arg(legate::Scalar(track.get_current_provenance())); + task.add_scalar_arg(legate::Scalar(track.get_current_provenance())); runtime->submit(std::move(task)); } diff --git a/tests/cpp/integration/region_manager.cc b/tests/cpp/integration/region_manager.cc index b503602fd3..34c216f429 100644 --- a/tests/cpp/integration/region_manager.cc +++ b/tests/cpp/integration/region_manager.cc @@ -32,8 +32,8 @@ TEST(Integration, RegionManager) std::vector stores; for (uint32_t idx = 0; idx < LEGION_MAX_FIELDS * 2; ++idx) { auto store = runtime->create_store({10}, legate::int64()); - auto part = task->declare_partition(); - task->add_output(store, part); + auto part = task.declare_partition(); + task.add_output(store, part); stores.push_back(store); } runtime->submit(std::move(task)); diff --git a/tests/cpp/integration/weighted.cc b/tests/cpp/integration/weighted.cc index 1916914f51..8804a38a84 100644 --- a/tests/cpp/integration/weighted.cc +++ b/tests/cpp/integration/weighted.cc @@ -72,10 +72,7 @@ void initialize(legate::Runtime* runtime, auto task = runtime->create_task(context, INIT, {NUM_TASKS}); std::vector parts; - for (auto& output : outputs) { - auto part = task->declare_partition(); - task->add_output(output); - } + for (auto& output : outputs) task.add_output(output); runtime->submit(std::move(task)); } @@ -87,12 +84,12 @@ void check(legate::Runtime* runtime, auto task = runtime->create_task(context, CHECK); for (auto& input : inputs) { - auto part_in = task->declare_partition(); + auto part_in = task.declare_partition(); auto output = runtime->create_store(input.extents(), input.type()); - auto part_out = task->declare_partition(); - task->add_input(input, part_in); - task->add_output(output, part_out); - task->add_constraint(legate::align(part_in, part_out)); + auto part_out = task.declare_partition(); + task.add_input(input, part_in); + task.add_output(output, part_out); + task.add_constraint(legate::align(part_in, part_out)); } runtime->submit(std::move(task)); From 0533cb9e47e564edc4622555bb94c6f546ff2527 Mon Sep 17 00:00:00 2001 From: Joy Shen Date: Thu, 29 Jun 2023 09:09:59 -0700 Subject: [PATCH 0155/1425] Add unit test for Scalar and Span. * Fix comments. * Add unit test for Scalar and Span. * fix review comments. * Add unit test for Scalar and Span. * fix review comments. * Add unit test for Scalar and Span. * Add unit test for Scalar and Span. * fix review comments. * Add unit test for Scalar and Span. * fix review comments. * Add unit test for Scalar and Span. * Fix comments. * Add unit test for Scalar and Span. * fix review comments. * Add unit test for Scalar and Span. * fix review comments. * Add unit test for Scalar and Span. * fix review comments. * Add unit test for Scalar and Span. * fix review comments. * Add unit test for Scalar and Span. * Add unit test for Scalar and Span. See merge request legate/legate.core.internal!53 --- tests/cpp/unit/scalar.cc | 305 +++++++++++++++++++++++++++++++++++++++ tests/cpp/unit/span.cc | 94 ++++++++++++ 2 files changed, 399 insertions(+) create mode 100644 tests/cpp/unit/scalar.cc create mode 100644 tests/cpp/unit/span.cc diff --git a/tests/cpp/unit/scalar.cc b/tests/cpp/unit/scalar.cc new file mode 100644 index 0000000000..8ea5f43337 --- /dev/null +++ b/tests/cpp/unit/scalar.cc @@ -0,0 +1,305 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "core/utilities/buffer_builder.h" +#include "legate.h" + +namespace scalar_test { + +constexpr bool BOOL_VALUE = true; +constexpr int8_t INT8_VALUE = 10; +constexpr int16_t INT16_VALUE = -1000; +constexpr int32_t INT32_VALUE = 2700; +constexpr int64_t INT64_VALUE = -28; +constexpr uint8_t UINT8_VALUE = 128; +constexpr uint16_t UINT16_VALUE = 65535; +constexpr uint32_t UINT32_VALUE = 999; +constexpr uint64_t UINT64_VALUE = 100; +constexpr float FLOAT_VALUE = 1.23f; +constexpr double DOUBLE_VALUE = -4.567d; +const std::string STRING_VALUE = "123"; +const __half FLOAT16_VALUE(0.1f); +const complex COMPLEX_FLOAT_VALUE{0, 1}; +const complex COMPLEX_DOUBLE_VALUE{0, 1}; +constexpr uint32_t DATA_SIZE = 10; + +struct PaddingStructData { + bool bool_data; + int32_t int32_data; + uint64_t uint64_data; +}; + +struct __attribute__((packed)) NoPaddingStructData { + bool bool_data; + int32_t int32_data; + uint64_t uint64_data; +}; + +class ScalarUnitTestDeserializer : public legate::BaseDeserializer { + public: + ScalarUnitTestDeserializer(const void* args, size_t arglen); + + public: + using BaseDeserializer::_unpack; +}; + +ScalarUnitTestDeserializer::ScalarUnitTestDeserializer(const void* args, size_t arglen) + : BaseDeserializer(args, arglen) +{ +} + +TEST(ScalarUnit, CreateWithObject) +{ + // constructor with Scalar object + legate::Scalar scalar1(INT32_VALUE); + legate::Scalar scalar2(scalar1); + EXPECT_EQ(scalar2.type().code, scalar1.type().code); + EXPECT_EQ(scalar2.size(), scalar1.size()); + EXPECT_EQ(scalar2.size(), legate::uint32()->size()); + EXPECT_NE(scalar2.ptr(), nullptr); + EXPECT_EQ(scalar2.value(), scalar1.value()); + EXPECT_EQ(scalar2.values().size(), scalar1.values().size()); + EXPECT_EQ(*scalar2.values().begin(), *scalar1.values().begin()); + + // Invalid type + EXPECT_THROW(scalar2.value(), std::invalid_argument); + EXPECT_THROW(scalar2.values(), std::invalid_argument); +} + +TEST(ScalarUnit, CreateSharedScalar) +{ + auto data_vec = std::vector(DATA_SIZE, UINT64_VALUE); + const auto* data = data_vec.data(); + legate::Scalar scalar(legate::uint64(), data); + EXPECT_EQ(scalar.type().code, legate::Type::Code::UINT64); + EXPECT_EQ(scalar.size(), legate::uint64()->size()); + EXPECT_EQ(scalar.ptr(), data); + + EXPECT_EQ(scalar.value(), UINT64_VALUE); + legate::Span actualValues(scalar.values()); + legate::Span expectedValues = legate::Span(data, 1); + EXPECT_EQ(*actualValues.begin(), *expectedValues.begin()); + EXPECT_EQ(actualValues.size(), expectedValues.size()); + + // Invalid type + EXPECT_THROW(scalar.value(), std::invalid_argument); + EXPECT_THROW(scalar.values(), std::invalid_argument); +} + +template +void checkType(T value) +{ + legate::Scalar scalar(value); + EXPECT_EQ(scalar.type().code, legate::legate_type_code_of); + EXPECT_EQ(scalar.size(), sizeof(T)); + EXPECT_EQ(scalar.value(), value); + EXPECT_EQ(scalar.values().size(), 1); + EXPECT_NE(scalar.ptr(), nullptr); +} + +TEST(ScalarUnit, CreateWithValue) +{ + checkType(BOOL_VALUE); + checkType(INT8_VALUE); + checkType(INT16_VALUE); + checkType(INT32_VALUE); + checkType(INT64_VALUE); + checkType(UINT8_VALUE); + checkType(UINT16_VALUE); + checkType(UINT32_VALUE); + checkType(UINT64_VALUE); + checkType(FLOAT_VALUE); + checkType(DOUBLE_VALUE); + checkType(FLOAT16_VALUE); + checkType(COMPLEX_FLOAT_VALUE); + checkType(COMPLEX_DOUBLE_VALUE); +} + +TEST(ScalarUnit, CreateWithVector) +{ + // constructor with arrays + std::vector scalar_data{INT32_VALUE, INT32_VALUE}; + legate::Scalar scalar(scalar_data); + EXPECT_EQ(scalar.type().code, legate::Type::Code::FIXED_ARRAY); + auto fixed_type = legate::fixed_array_type(legate::int32(), scalar_data.size()); + EXPECT_EQ(scalar.size(), fixed_type->size()); + + // check values here. Note: no value allowed for a fixed arrays scalar + std::vector data_vec = {INT32_VALUE, INT32_VALUE}; + const auto* data = data_vec.data(); + legate::Span expectedValues = legate::Span(data, scalar_data.size()); + legate::Span actualValues(scalar.values()); + EXPECT_EQ(*actualValues.begin(), *expectedValues.begin()); + EXPECT_EQ(*actualValues.end(), *expectedValues.end()); + EXPECT_EQ(actualValues.size(), expectedValues.size()); + + // Invalid type + EXPECT_THROW(scalar.value(), std::invalid_argument); + EXPECT_THROW(scalar.values(), std::invalid_argument); +} + +TEST(ScalarUnit, CreateWithString) +{ + // constructor with string + auto inputString = STRING_VALUE; + legate::Scalar scalar(inputString); + EXPECT_EQ(scalar.type().code, legate::Type::Code::STRING); + auto expectedSize = sizeof(uint32_t) + sizeof(char) * inputString.size(); + EXPECT_EQ(scalar.size(), expectedSize); + EXPECT_NE(scalar.ptr(), inputString.data()); + + // Check values + EXPECT_EQ(scalar.value(), inputString); + legate::Span actualValues(scalar.values()); + EXPECT_EQ(actualValues.size(), 1); + EXPECT_EQ(*actualValues.begin(), inputString.substr(0, 1)); + + // Invalid type + EXPECT_THROW(scalar.value(), std::invalid_argument); + EXPECT_THROW(scalar.values(), std::invalid_argument); +} + +TEST(ScalarUnit, CreateWithStructType) +{ + // with struct padding + { + PaddingStructData structData = {BOOL_VALUE, INT32_VALUE, UINT64_VALUE}; + legate::Scalar scalar( + structData, legate::struct_type(true, legate::bool_(), legate::int32(), legate::uint64())); + EXPECT_EQ(scalar.type().code, legate::Type::Code::STRUCT); + EXPECT_EQ(scalar.size(), sizeof(PaddingStructData)); + EXPECT_NE(scalar.ptr(), nullptr); + + // Check value + PaddingStructData actualData = scalar.value(); + EXPECT_EQ(actualData.bool_data, structData.bool_data); + EXPECT_EQ(actualData.int32_data, structData.int32_data); + EXPECT_EQ(actualData.uint64_data, structData.uint64_data); + + // Check values + legate::Span actualValues(scalar.values()); + legate::Span expectedValues = legate::Span(&structData, 1); + EXPECT_EQ(actualValues.size(), expectedValues.size()); + EXPECT_EQ(actualValues.begin()->bool_data, expectedValues.begin()->bool_data); + EXPECT_EQ(actualValues.begin()->int32_data, expectedValues.begin()->int32_data); + EXPECT_EQ(actualValues.begin()->uint64_data, expectedValues.begin()->uint64_data); + EXPECT_NE(actualValues.ptr(), expectedValues.ptr()); + } + + // without struct padding + { + NoPaddingStructData structData = {BOOL_VALUE, INT32_VALUE, UINT64_VALUE}; + legate::Scalar scalar( + structData, legate::struct_type(false, legate::bool_(), legate::int32(), legate::uint64())); + EXPECT_EQ(scalar.type().code, legate::Type::Code::STRUCT); + EXPECT_EQ(scalar.size(), sizeof(NoPaddingStructData)); + EXPECT_NE(scalar.ptr(), nullptr); + + // Check value + NoPaddingStructData actualData = scalar.value(); + EXPECT_EQ(actualData.bool_data, structData.bool_data); + EXPECT_EQ(actualData.int32_data, structData.int32_data); + EXPECT_EQ(actualData.uint64_data, structData.uint64_data); + + // Check values + legate::Span actualValues(scalar.values()); + legate::Span expectedValues = legate::Span(&structData, 1); + EXPECT_EQ(actualValues.size(), expectedValues.size()); + EXPECT_EQ(actualValues.begin()->bool_data, expectedValues.begin()->bool_data); + EXPECT_EQ(actualValues.begin()->int32_data, expectedValues.begin()->int32_data); + EXPECT_EQ(actualValues.begin()->uint64_data, expectedValues.begin()->uint64_data); + EXPECT_NE(actualValues.ptr(), expectedValues.ptr()); + } +} + +TEST(ScalarUnit, OperatorEqual) +{ + legate::Scalar scalar1(INT32_VALUE); + legate::Scalar scalar2(UINT64_VALUE); + scalar2 = scalar1; + EXPECT_EQ(scalar2.type().code, scalar1.type().code); + EXPECT_EQ(scalar2.size(), scalar2.size()); + EXPECT_EQ(scalar2.value(), scalar1.value()); + EXPECT_EQ(scalar2.values().size(), scalar1.values().size()); +} + +void checkPack(const legate::Scalar& scalar) +{ + legate::BufferBuilder buf; + scalar.pack(buf); + auto legion_buffer = buf.to_legion_buffer(); + EXPECT_NE(legion_buffer.get_ptr(), nullptr); + legate::BaseDeserializer deserializer(legion_buffer.get_ptr(), + legion_buffer.get_size()); + auto scalar_unpack = deserializer._unpack_scalar(); + EXPECT_EQ(scalar_unpack.type().code, scalar.type().code); + EXPECT_EQ(scalar_unpack.size(), scalar.size()); +} + +TEST(ScalarUnit, Pack) +{ + // test pack for a fixed array scalar + { + legate::Scalar scalar(std::vector{UINT32_VALUE, UINT32_VALUE}); + checkPack(scalar); + } + + // test pack for a single value scalar + { + legate::Scalar scalar(BOOL_VALUE); + checkPack(scalar); + } + + // test pack for string scalar + { + legate::Scalar scalar(STRING_VALUE); + checkPack(scalar); + } + + // test pack for padding struct type scalar + { + PaddingStructData structData = {BOOL_VALUE, INT32_VALUE, UINT64_VALUE}; + legate::Scalar scalar( + structData, legate::struct_type(true, legate::bool_(), legate::int32(), legate::uint64())); + checkPack(scalar); + } + + // test pack for no padding struct type scalar + { + NoPaddingStructData structData = {BOOL_VALUE, INT32_VALUE, UINT64_VALUE}; + legate::Scalar scalar( + structData, legate::struct_type(false, legate::bool_(), legate::int32(), legate::uint64())); + checkPack(scalar); + } + + // test pack for a shared scalar + { + auto data_vec = std::vector(DATA_SIZE, UINT64_VALUE); + const auto* data = data_vec.data(); + legate::Scalar scalar(legate::uint64(), data); + checkPack(scalar); + } + + // test pack for scalar created by another scalar + { + legate::Scalar scalar1(INT32_VALUE); + legate::Scalar scalar2(scalar1); + checkPack(scalar2); + } +} +} // namespace scalar_test diff --git a/tests/cpp/unit/span.cc b/tests/cpp/unit/span.cc new file mode 100644 index 0000000000..4f4376bbb1 --- /dev/null +++ b/tests/cpp/unit/span.cc @@ -0,0 +1,94 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include "legate.h" + +namespace span_test { + + +constexpr bool BOOL_VALUE = true; +constexpr int8_t INT8_VALUE = 10; +constexpr int16_t INT16_VALUE = -1000; +constexpr int32_t INT32_VALUE = 2700; +constexpr int64_t INT64_VALUE = -28; +constexpr uint8_t UINT8_VALUE = 128; +constexpr uint16_t UINT16_VALUE = 65535; +constexpr uint32_t UINT32_VALUE = 999; +constexpr uint64_t UINT64_VALUE = 100; +constexpr float FLOAT_VALUE = 1.23f; +constexpr double DOUBLE_VALUE = -4.567d; +const std::string STRING_VALUE = "123"; +const __half FLOAT16_VALUE1(0.1f); +const __half FLOAT16_VALUE2(0.2f); +const __half FLOAT16_VALUE3(0.3f); +const complex COMPLEX_FLOAT_VALUE1{0, 1}; +const complex COMPLEX_FLOAT_VALUE2{2, 3}; +const complex COMPLEX_FLOAT_VALUE3{4, 5}; +const complex COMPLEX_DOUBLE_VALUE1{6, 7}; +const complex COMPLEX_DOUBLE_VALUE2{8, 9}; +const complex COMPLEX_DOUBLE_VALUE3{10, 11}; + + +constexpr uint32_t DATA_SIZE = 3; + +template +void create(T value1, T value2, T value3) +{ + std::array data = {value1, value2, value3}; + legate::Span span = legate::Span(data.data(), DATA_SIZE); + EXPECT_EQ(span.ptr(), data.data()); + EXPECT_EQ(span.size(), DATA_SIZE); + EXPECT_EQ(span.end() - span.begin(), DATA_SIZE); + for (auto& to_compare : span) { + auto i = std::distance(span.begin(), &to_compare); + EXPECT_EQ(data.at(i), to_compare); + } + EXPECT_EQ(span[0], value1); + EXPECT_EQ(span[DATA_SIZE - 1], value3); +} + +TEST(SpanUnit, Create) +{ + create(BOOL_VALUE, BOOL_VALUE, !BOOL_VALUE); + create(INT8_VALUE, static_cast(INT8_VALUE + 1), static_cast(INT8_VALUE + 2)); + create(INT16_VALUE, static_cast(INT16_VALUE + 1), static_cast(INT16_VALUE + 2)); + create(INT32_VALUE, static_cast(INT32_VALUE + 1), static_cast(INT32_VALUE + 2)); + create(INT64_VALUE, static_cast(INT64_VALUE + 1), static_cast(INT64_VALUE + 2)); + create(UINT8_VALUE, static_cast(UINT8_VALUE + 1), static_cast(UINT8_VALUE + 2)); + create(UINT16_VALUE, static_cast(UINT16_VALUE + 1), static_cast(UINT16_VALUE + 2)); + create(UINT32_VALUE, static_cast(UINT32_VALUE + 1), static_cast(UINT32_VALUE + 2)); + create(UINT64_VALUE, static_cast(UINT64_VALUE + 1), static_cast(UINT64_VALUE + 2)); + create(FLOAT_VALUE, FLOAT_VALUE + 1.0f, FLOAT_VALUE + 2.0f); + create(DOUBLE_VALUE, DOUBLE_VALUE + 1.0d, DOUBLE_VALUE + 2.0d); + create(STRING_VALUE, STRING_VALUE + "1", STRING_VALUE + "2"); + create(FLOAT16_VALUE1, FLOAT16_VALUE2, FLOAT16_VALUE3); + create(COMPLEX_FLOAT_VALUE1, COMPLEX_FLOAT_VALUE2, COMPLEX_FLOAT_VALUE3); + create(COMPLEX_DOUBLE_VALUE1, COMPLEX_DOUBLE_VALUE2, COMPLEX_DOUBLE_VALUE3); +} + +TEST(SpanUnit, Subspan) +{ + auto data_vec = std::vector(DATA_SIZE, UINT64_VALUE); + const auto* data = data_vec.data(); + legate::Span span = legate::Span(data, DATA_SIZE); + legate::Span subspan = span.subspan(DATA_SIZE - 1); + EXPECT_EQ(subspan.ptr(), &data[DATA_SIZE - 1]); + EXPECT_EQ(subspan.size(), 1); + EXPECT_EQ(subspan.end() - subspan.begin(), 1); + for (auto& to_compare : span) EXPECT_EQ(UINT64_VALUE, to_compare); +} +} // namespace span_test From 4ac8e10f876a1a47d6c097f4544f3ebb45a485e6 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 30 Jun 2023 00:14:37 -0700 Subject: [PATCH 0156/1425] Validation hooks * Split up Copy into four classes (Copy, Gather, Scatter, ScatterGather) * Provide direct APIs for copies and not expose the Copy class * Start validating operations immediately upon submission See merge request legate/legate.core.internal!70 --- legate_core_cpp.cmake | 8 +- src/core/operation/copy.cc | 50 ----- src/core/operation/copy.h | 60 ------ src/core/operation/detail/copy.cc | 180 +++--------------- src/core/operation/detail/copy.h | 39 +--- src/core/operation/detail/copy_launcher.cc | 10 +- src/core/operation/detail/copy_launcher.h | 13 +- src/core/operation/detail/fill.cc | 2 + src/core/operation/detail/fill.h | 1 + src/core/operation/detail/gather.cc | 92 +++++++++ src/core/operation/detail/gather.h | 58 ++++++ src/core/operation/detail/operation.cc | 11 ++ src/core/operation/detail/operation.h | 11 +- src/core/operation/detail/scatter.cc | 92 +++++++++ src/core/operation/detail/scatter.h | 58 ++++++ src/core/operation/detail/scatter_gather.cc | 117 ++++++++++++ src/core/operation/detail/scatter_gather.h | 62 ++++++ src/core/operation/detail/task.cc | 29 +-- src/core/operation/detail/task.h | 22 ++- src/core/partitioning/constraint.cc | 22 ++- src/core/partitioning/constraint.h | 22 ++- src/core/partitioning/partitioner.h | 3 - src/core/runtime/detail/runtime.cc | 48 ++++- src/core/runtime/detail/runtime.h | 12 +- src/core/runtime/runtime.cc | 26 ++- src/core/runtime/runtime.h | 54 ++++-- src/core/type/type_info.cc | 30 ++- src/core/type/type_info.h | 13 +- src/legate.h | 1 - .../cpp/integration/alignment_constraints.cc | 36 +++- .../cpp/integration/broadcast_constraints.cc | 27 ++- tests/cpp/integration/copy_failure.cc | 122 +++++------- tests/cpp/integration/copy_gather.cc | 6 +- tests/cpp/integration/copy_gather_scatter.cc | 8 +- tests/cpp/integration/copy_normal.cc | 55 +----- tests/cpp/integration/copy_scatter.cc | 7 +- 36 files changed, 895 insertions(+), 512 deletions(-) delete mode 100644 src/core/operation/copy.cc delete mode 100644 src/core/operation/copy.h create mode 100644 src/core/operation/detail/gather.cc create mode 100644 src/core/operation/detail/gather.h create mode 100644 src/core/operation/detail/scatter.cc create mode 100644 src/core/operation/detail/scatter.h create mode 100644 src/core/operation/detail/scatter_gather.cc create mode 100644 src/core/operation/detail/scatter_gather.h diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index 785c3794d3..0a8b063adf 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -218,16 +218,18 @@ list(APPEND legate_core_SOURCES src/core/mapping/mapping.cc src/core/mapping/operation.cc src/core/mapping/store.cc - src/core/operation/copy.cc src/core/operation/task.cc src/core/operation/detail/copy.cc src/core/operation/detail/copy_launcher.cc src/core/operation/detail/fill.cc src/core/operation/detail/fill_launcher.cc + src/core/operation/detail/gather.cc src/core/operation/detail/launcher_arg.cc src/core/operation/detail/operation.cc src/core/operation/detail/projection.cc src/core/operation/detail/req_analyzer.cc + src/core/operation/detail/scatter.cc + src/core/operation/detail/scatter_gather.cc src/core/operation/detail/task.cc src/core/operation/detail/task_launcher.cc src/core/partitioning/constraint.cc @@ -380,7 +382,6 @@ if (legate_core_BUILD_DOCS) src/core/runtime/context.h # operation src/core/operation/task.h - src/core/operation/copy.h # partitioning src/core/partitioning/constraint.h # mapping @@ -470,8 +471,7 @@ install( DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/legate/core/mapping) install( - FILES src/core/operation/copy.h - src/core/operation/task.h + FILES src/core/operation/task.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/legate/core/operation) install( diff --git a/src/core/operation/copy.cc b/src/core/operation/copy.cc deleted file mode 100644 index 9d379af71c..0000000000 --- a/src/core/operation/copy.cc +++ /dev/null @@ -1,50 +0,0 @@ -/* Copyright 2021 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#include "core/operation/copy.h" - -#include "core/operation/detail/copy.h" - -namespace legate { - -void Copy::add_input(LogicalStore store) { impl_->add_input(store.impl()); } - -void Copy::add_output(LogicalStore store) { impl_->add_output(store.impl()); } - -void Copy::add_reduction(LogicalStore store, Legion::ReductionOpID redop) -{ - impl_->add_reduction(store.impl(), redop); -} - -void Copy::add_source_indirect(LogicalStore store) { impl_->add_source_indirect(store.impl()); } - -void Copy::add_target_indirect(LogicalStore store) { impl_->add_target_indirect(store.impl()); } - -void Copy::set_source_indirect_out_of_range(bool flag) -{ - impl_->set_source_indirect_out_of_range(flag); -} - -void Copy::set_target_indirect_out_of_range(bool flag) -{ - impl_->set_target_indirect_out_of_range(flag); -} - -Copy::~Copy() {} - -Copy::Copy(std::unique_ptr impl) : impl_(std::move(impl)) {} - -} // namespace legate diff --git a/src/core/operation/copy.h b/src/core/operation/copy.h deleted file mode 100644 index 85cfd1ebfb..0000000000 --- a/src/core/operation/copy.h +++ /dev/null @@ -1,60 +0,0 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#pragma once - -#include - -#include "core/data/logical_store.h" -/** - * @file - * @brief Class definition for legate::Copy - */ - -namespace legate::detail { -class Copy; -} // namespace legate::detail - -namespace legate { - -class Copy { - public: - void add_input(LogicalStore store); - void add_output(LogicalStore store); - void add_reduction(LogicalStore store, Legion::ReductionOpID redop); - void add_source_indirect(LogicalStore store); - void add_target_indirect(LogicalStore store); - - public: - void set_source_indirect_out_of_range(bool flag); - void set_target_indirect_out_of_range(bool flag); - - public: - Copy(const Copy&) = delete; - Copy(Copy&&) = default; - Copy& operator=(const Copy&) = delete; - Copy& operator=(Copy&&) = default; - - public: - ~Copy(); - - private: - friend class Runtime; - Copy(std::unique_ptr impl); - std::unique_ptr impl_; -}; - -} // namespace legate diff --git a/src/core/operation/detail/copy.cc b/src/core/operation/detail/copy.cc index f09c14bfaa..f41e123887 100644 --- a/src/core/operation/detail/copy.cc +++ b/src/core/operation/detail/copy.cc @@ -17,7 +17,6 @@ #include "core/operation/detail/copy.h" #include "core/operation/detail/copy_launcher.h" -#include "core/operation/detail/projection.h" #include "core/partitioning/constraint.h" #include "core/partitioning/constraint_solver.h" #include "core/partitioning/partition.h" @@ -25,119 +24,39 @@ namespace legate::detail { -Copy::Copy(int64_t unique_id, mapping::MachineDesc&& machine) - : Operation(unique_id, std::move(machine)) +Copy::Copy(std::shared_ptr target, + std::shared_ptr source, + int64_t unique_id, + mapping::MachineDesc&& machine) + : Operation(unique_id, std::move(machine)), + target_{target.get(), declare_partition()}, + source_{source.get(), declare_partition()}, + constraint_(legate::align(target_.variable, source_.variable)) { + record_partition(target_.variable, std::move(target)); + record_partition(source_.variable, std::move(source)); } -void Copy::add_store(std::vector& store_args, - std::shared_ptr store, - const Variable* partition_symbol) +void Copy::validate() { - store_args.push_back(StoreArg(store.get(), partition_symbol)); - record_partition(partition_symbol, std::move(store)); -} - -void Copy::add_store(std::optional& store_arg, - std::shared_ptr store, - const Variable* partition_symbol) -{ - store_arg = StoreArg(store.get(), partition_symbol); - record_partition(partition_symbol, std::move(store)); -} - -void check_store(std::shared_ptr& store) -{ - if (store->unbound() || store->has_scalar_storage() || store->transformed()) { - throw std::runtime_error("Copy accepts only normal, not transformed, region-backed stores"); - } -} - -void Copy::add_input(std::shared_ptr&& store) -{ - check_store(store); - add_store(inputs_, std::move(store), declare_partition()); -} - -void Copy::add_output(std::shared_ptr&& store) -{ - check_store(store); - if (reductions_.size() > 0) - throw std::runtime_error("Copy targets must be either all normal outputs or reductions"); - add_store(outputs_, std::move(store), declare_partition()); -} - -void Copy::add_reduction(std::shared_ptr&& store, Legion::ReductionOpID redop) -{ - check_store(store); - if (outputs_.size() > 0) - throw std::runtime_error("Copy targets must be either all normal outputs or reductions"); - add_store(reductions_, std::move(store), declare_partition()); - reduction_ops_.push_back(redop); -} - -void Copy::add_source_indirect(std::shared_ptr&& store) -{ - check_store(store); - add_store(source_indirect_, std::move(store), declare_partition()); -} - -void Copy::add_target_indirect(std::shared_ptr&& store) -{ - check_store(store); - add_store(target_indirect_, std::move(store), declare_partition()); + auto validate_store = [](auto* store) { + if (store->unbound() || store->has_scalar_storage() || store->transformed()) { + throw std::invalid_argument("Copy accepts only normal, untransformed, region-backed stores"); + } + }; + validate_store(target_.store); + validate_store(source_.store); + constraint_->validate(); } -void Copy::set_source_indirect_out_of_range(bool flag) { source_indirect_out_of_range_ = flag; } - -void Copy::set_target_indirect_out_of_range(bool flag) { target_indirect_out_of_range_ = flag; } - void Copy::launch(Strategy* p_strategy) { auto& strategy = *p_strategy; - CopyLauncher launcher(machine_, source_indirect_out_of_range_, target_indirect_out_of_range_); + CopyLauncher launcher(machine_); auto launch_domain = strategy.launch_domain(this); - auto create_projection_info = [&strategy, &launch_domain](auto& store, auto& var) { - auto store_partition = store->create_partition(strategy[var]); - auto proj_info = store_partition->create_projection_info(launch_domain); - proj_info->tag = strategy.is_key_partition(var) ? LEGATE_CORE_KEY_STORE_TAG : 0; - return std::move(proj_info); - }; - - for (auto& [store, var] : inputs_) launcher.add_input(store, create_projection_info(store, var)); - // FIXME: today a copy is a scatter copy only when a target indirection - // is given. In the future, we may pass store transforms directly to - // Legion and some transforms turn some copies into scatter copies even - // when no target indirection is provided. So, this scatter copy check - // will need to be extended accordingly. - bool scatter = target_indirect_.has_value(); - for (auto& [store, var] : outputs_) { - if (scatter) - launcher.add_inout(store, create_projection_info(store, var)); - else - launcher.add_output(store, create_projection_info(store, var)); - store->set_key_partition(machine(), strategy[var].get()); - } - uint32_t idx = 0; - for (auto& [store, var] : reductions_) { - auto store_partition = store->create_partition(strategy[var]); - auto proj = store_partition->create_projection_info(launch_domain); - bool read_write = store_partition->is_disjoint_for(launch_domain); - auto redop = reduction_ops_[idx++]; - proj->set_reduction_op(redop); - launcher.add_reduction(store, std::move(proj), read_write); - } - - if (source_indirect_.has_value()) { - auto& [store, var] = source_indirect_.value(); - launcher.add_source_indirect(store, create_projection_info(store, var)); - } - - if (target_indirect_.has_value()) { - auto& [store, var] = target_indirect_.value(); - launcher.add_target_indirect(store, create_projection_info(store, var)); - } + launcher.add_input(source_.store, create_projection_info(strategy, launch_domain, source_)); + launcher.add_output(target_.store, create_projection_info(strategy, launch_domain, target_)); if (launch_domain != nullptr) { return launcher.execute(*launch_domain); @@ -148,58 +67,9 @@ void Copy::launch(Strategy* p_strategy) void Copy::add_to_solver(ConstraintSolver& solver) { - bool gather = source_indirect_.has_value(); - bool scatter = target_indirect_.has_value(); - - if (inputs_.size() != outputs_.size()) - throw std::runtime_error( - "Number of inputs and outputs should be the same in the Copy operation"); - - if (gather && inputs_.size() != 1) - throw std::runtime_error("when source indirect is specified, there could be only one input"); - - if (scatter && outputs_.size() != 1) - throw std::runtime_error("when target indirect is specified, there could be only one output"); - - // fill constraints - if (!gather && !scatter) { - for (size_t i = 0; i < inputs_.size(); i++) { - const auto& [in, in_part] = inputs_[i]; - const auto& [out, out_part] = outputs_[i]; - if (in->extents() != out->extents()) - throw std::runtime_error( - "Each output must have the same shape as the corresponding input in a copy"); - constraints_.push_back(align(in_part, out_part)); - } - } else if (gather && scatter) { - const auto& [src_indirect, src_indirect_part] = source_indirect_.value(); - const auto& [tgt_indirect, tgt_indirect_part] = target_indirect_.value(); - if (src_indirect->extents() != tgt_indirect->extents()) - throw std::runtime_error( - "Source and target indirect must have the same shape in a gather-scatter copy"); - constraints_.push_back(align(src_indirect_part, tgt_indirect_part)); - } else if (gather) { - const auto& [out, out_part] = reductions_.empty() ? outputs_.front() : reductions_.front(); - const auto& [indirect, indirect_part] = source_indirect_.value(); - if (out->extents() != indirect->extents()) - throw std::runtime_error( - "Source indirect must have the same shape as the output in a gather copy"); - constraints_.push_back(align(out_part, indirect_part)); - } else if (scatter) { - const auto& [in, in_part] = inputs_.front(); - const auto& [indirect, indirect_part] = target_indirect_.value(); - if (in->extents() != indirect->extents()) - throw std::runtime_error( - "Target indirect must have the same shape as the input in a scatter copy"); - constraints_.push_back(align(in_part, indirect_part)); - } - - for (auto& constraint : constraints_) solver.add_constraint(constraint.get()); - for (auto& [_, symb] : inputs_) solver.add_partition_symbol(symb); - for (auto& [_, symb] : outputs_) solver.add_partition_symbol(symb, true); - for (auto& [_, symb] : reductions_) solver.add_partition_symbol(symb); - if (source_indirect_.has_value()) solver.add_partition_symbol(source_indirect_.value().second); - if (target_indirect_.has_value()) solver.add_partition_symbol(target_indirect_.value().second); + solver.add_constraint(constraint_.get()); + solver.add_partition_symbol(target_.variable); + solver.add_partition_symbol(source_.variable); } std::string Copy::to_string() const { return "Copy:" + std::to_string(unique_id_); } diff --git a/src/core/operation/detail/copy.h b/src/core/operation/detail/copy.h index e8194eb053..a70dc2016c 100644 --- a/src/core/operation/detail/copy.h +++ b/src/core/operation/detail/copy.h @@ -25,47 +25,28 @@ namespace legate::detail { class ConstraintSolver; -class Runtime; class Copy : public Operation { - private: - friend class detail::Runtime; - Copy(int64_t unique_id, mapping::MachineDesc&& machine); - - public: - void add_input(std::shared_ptr&& store); - void add_output(std::shared_ptr&& store); - void add_reduction(std::shared_ptr&& store, Legion::ReductionOpID redop); - void add_source_indirect(std::shared_ptr&& store); - void add_target_indirect(std::shared_ptr&& store); - - private: - void add_store(std::vector& store_args, - std::shared_ptr store, - const Variable* partition_symbol); - void add_store(std::optional& store_arg, - std::shared_ptr store, - const Variable* partition_symbol); - public: - void set_source_indirect_out_of_range(bool flag); - void set_target_indirect_out_of_range(bool flag); + Copy(std::shared_ptr target, + std::shared_ptr source, + int64_t unique_id, + mapping::MachineDesc&& machine); public: - void launch(detail::Strategy* strategy) override; + void validate() override; + void launch(Strategy* strategy) override; public: - void add_to_solver(detail::ConstraintSolver& solver) override; + void add_to_solver(ConstraintSolver& solver) override; public: std::string to_string() const override; private: - std::vector> constraints_{}; - std::optional source_indirect_{}; - std::optional target_indirect_{}; - bool source_indirect_out_of_range_{true}; - bool target_indirect_out_of_range_{true}; + StoreArg target_; + StoreArg source_; + std::unique_ptr constraint_; }; } // namespace legate::detail diff --git a/src/core/operation/detail/copy_launcher.cc b/src/core/operation/detail/copy_launcher.cc index 9a7c5bf090..3b7e78c289 100644 --- a/src/core/operation/detail/copy_launcher.cc +++ b/src/core/operation/detail/copy_launcher.cc @@ -80,14 +80,8 @@ void CopyArg::pack(BufferBuilder& buffer) const buffer.pack(field_id_); } -CopyLauncher::CopyLauncher(const mapping::MachineDesc& machine, - bool source_indirect_out_of_range, - bool target_indirect_out_of_range, - int64_t tag) - : machine_(machine), - tag_(tag), - source_indirect_out_of_range_(source_indirect_out_of_range), - target_indirect_out_of_range_(target_indirect_out_of_range) +CopyLauncher::CopyLauncher(const mapping::MachineDesc& machine, int64_t tag) + : machine_(machine), tag_(tag) { mapper_arg_ = new BufferBuilder(); machine_.pack(*mapper_arg_); diff --git a/src/core/operation/detail/copy_launcher.h b/src/core/operation/detail/copy_launcher.h index 10b79df385..ce9ef85140 100644 --- a/src/core/operation/detail/copy_launcher.h +++ b/src/core/operation/detail/copy_launcher.h @@ -41,10 +41,7 @@ class RequirementAnalyzer; class CopyLauncher { public: - CopyLauncher(const mapping::MachineDesc& machine, - bool source_indirect_out_of_range, - bool target_indirect_out_of_range, - int64_t tag = 0); + CopyLauncher(const mapping::MachineDesc& machine, int64_t tag = 0); ~CopyLauncher(); public: @@ -63,6 +60,10 @@ class CopyLauncher { std::unique_ptr proj_info, Legion::PrivilegeMode privilege); + public: + void set_source_indirect_out_of_range(bool flag) { source_indirect_out_of_range_ = flag; } + void set_target_indirect_out_of_range(bool flag) { target_indirect_out_of_range_ = flag; } + public: void execute(const Legion::Domain& launch_domain); void execute_single(); @@ -88,8 +89,8 @@ class CopyLauncher { std::vector target_indirect_{}; private: - bool source_indirect_out_of_range_; - bool target_indirect_out_of_range_; + bool source_indirect_out_of_range_{true}; + bool target_indirect_out_of_range_{true}; }; } // namespace legate::detail diff --git a/src/core/operation/detail/fill.cc b/src/core/operation/detail/fill.cc index fecab65bd8..06b1c74cd4 100644 --- a/src/core/operation/detail/fill.cc +++ b/src/core/operation/detail/fill.cc @@ -41,6 +41,8 @@ Fill::Fill(std::shared_ptr&& lhs, throw std::runtime_error("Fill value should be a Future-back store"); } +void Fill::validate() {} + void Fill::launch(Strategy* strategy) { FillLauncher launcher(machine_); diff --git a/src/core/operation/detail/fill.h b/src/core/operation/detail/fill.h index 54b29508f4..da6ea51c47 100644 --- a/src/core/operation/detail/fill.h +++ b/src/core/operation/detail/fill.h @@ -29,6 +29,7 @@ class Fill : public Operation { mapping::MachineDesc&& machine); public: + void validate() override; void launch(Strategy* strategy) override; public: diff --git a/src/core/operation/detail/gather.cc b/src/core/operation/detail/gather.cc new file mode 100644 index 0000000000..217f34847d --- /dev/null +++ b/src/core/operation/detail/gather.cc @@ -0,0 +1,92 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/operation/detail/gather.h" + +#include "core/operation/detail/copy_launcher.h" +#include "core/partitioning/constraint.h" +#include "core/partitioning/constraint_solver.h" +#include "core/partitioning/partition.h" +#include "core/partitioning/partitioner.h" + +namespace legate::detail { + +Gather::Gather(std::shared_ptr target, + std::shared_ptr source, + std::shared_ptr source_indirect, + int64_t unique_id, + mapping::MachineDesc&& machine) + : Operation(unique_id, std::move(machine)), + target_{target.get(), declare_partition()}, + source_{source.get(), declare_partition()}, + source_indirect_{source_indirect.get(), declare_partition()}, + constraint_(legate::align(target_.variable, source_indirect_.variable)) +{ + record_partition(target_.variable, std::move(target)); + record_partition(source_.variable, std::move(source)); + record_partition(source_indirect_.variable, std::move(source_indirect)); +} + +void Gather::validate() +{ + auto validate_store = [](auto* store) { + if (store->unbound() || store->has_scalar_storage() || store->transformed()) { + throw std::invalid_argument( + "Gather accepts only normal, untransformed, region-backed stores"); + } + }; + validate_store(target_.store); + validate_store(source_.store); + validate_store(source_indirect_.store); + + if (!is_point_type(source_indirect_.store->type(), source_.store->dim())) { + throw std::invalid_argument("Indirection store should contain " + + std::to_string(source_.store->dim()) + "-D points"); + } + + constraint_->validate(); +} + +void Gather::launch(Strategy* p_strategy) +{ + auto& strategy = *p_strategy; + CopyLauncher launcher(machine_); + auto launch_domain = strategy.launch_domain(this); + + launcher.add_input(source_.store, create_projection_info(strategy, launch_domain, source_)); + launcher.add_source_indirect(source_indirect_.store, + create_projection_info(strategy, launch_domain, source_indirect_)); + launcher.add_output(target_.store, create_projection_info(strategy, launch_domain, target_)); + launcher.set_source_indirect_out_of_range(out_of_range_); + + if (launch_domain != nullptr) { + return launcher.execute(*launch_domain); + } else { + return launcher.execute_single(); + } +} + +void Gather::add_to_solver(ConstraintSolver& solver) +{ + solver.add_constraint(constraint_.get()); + solver.add_partition_symbol(target_.variable); + solver.add_partition_symbol(source_.variable); + solver.add_partition_symbol(source_indirect_.variable); +} + +std::string Gather::to_string() const { return "Gather:" + std::to_string(unique_id_); } + +} // namespace legate::detail diff --git a/src/core/operation/detail/gather.h b/src/core/operation/detail/gather.h new file mode 100644 index 0000000000..83930aef7b --- /dev/null +++ b/src/core/operation/detail/gather.h @@ -0,0 +1,58 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include + +#include "core/data/detail/logical_store.h" +#include "core/operation/detail/operation.h" +#include "core/partitioning/constraint.h" + +namespace legate::detail { + +class ConstraintSolver; + +class Gather : public Operation { + public: + Gather(std::shared_ptr target, + std::shared_ptr source, + std::shared_ptr source_indirect, + int64_t unique_id, + mapping::MachineDesc&& machine); + + public: + void set_indirect_out_of_range(bool flag) { out_of_range_ = flag; } + + public: + void validate() override; + void launch(Strategy* strategy) override; + + public: + void add_to_solver(ConstraintSolver& solver) override; + + public: + std::string to_string() const override; + + private: + bool out_of_range_{true}; + StoreArg target_; + StoreArg source_; + StoreArg source_indirect_; + std::unique_ptr constraint_; +}; + +} // namespace legate::detail diff --git a/src/core/operation/detail/operation.cc b/src/core/operation/detail/operation.cc index b127f4e053..5b65ccddbe 100644 --- a/src/core/operation/detail/operation.cc +++ b/src/core/operation/detail/operation.cc @@ -17,6 +17,7 @@ #include "core/operation/detail/operation.h" #include "core/partitioning/constraint.h" +#include "core/partitioning/partitioner.h" #include "core/runtime/detail/runtime.h" namespace legate::detail { @@ -62,4 +63,14 @@ void Operation::record_partition(const Variable* variable, std::shared_ptr Operation::create_projection_info(const Strategy& strategy, + const Domain* launch_domain, + const StoreArg& arg) const +{ + auto store_partition = arg.store->create_partition(strategy[arg.variable]); + auto proj_info = store_partition->create_projection_info(launch_domain); + proj_info->tag = strategy.is_key_partition(arg.variable) ? LEGATE_CORE_KEY_STORE_TAG : 0; + return std::move(proj_info); +} + } // namespace legate::detail diff --git a/src/core/operation/detail/operation.h b/src/core/operation/detail/operation.h index 134333ba11..dc069d7899 100644 --- a/src/core/operation/detail/operation.h +++ b/src/core/operation/detail/operation.h @@ -20,6 +20,7 @@ #include "core/data/detail/logical_store.h" #include "core/mapping/machine.h" +#include "core/operation/detail/projection.h" namespace legate { class Variable; @@ -32,13 +33,17 @@ class Strategy; class Operation { protected: - using StoreArg = std::pair; + struct StoreArg { + LogicalStore* store; + const Variable* variable; + }; Operation(uint64_t unique_id, mapping::MachineDesc&& machine); public: virtual ~Operation() {} public: + virtual void validate() = 0; virtual void add_to_solver(ConstraintSolver& solver) = 0; virtual void launch(Strategy* strategy) = 0; virtual std::string to_string() const = 0; @@ -54,6 +59,10 @@ class Operation { protected: void record_partition(const Variable* variable, std::shared_ptr store); + // Helper methods + std::unique_ptr create_projection_info(const Strategy& strategy, + const Domain* launch_domain, + const StoreArg& arg) const; protected: uint64_t unique_id_; diff --git a/src/core/operation/detail/scatter.cc b/src/core/operation/detail/scatter.cc new file mode 100644 index 0000000000..2bc6db7fc9 --- /dev/null +++ b/src/core/operation/detail/scatter.cc @@ -0,0 +1,92 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/operation/detail/scatter.h" + +#include "core/operation/detail/copy_launcher.h" +#include "core/partitioning/constraint.h" +#include "core/partitioning/constraint_solver.h" +#include "core/partitioning/partition.h" +#include "core/partitioning/partitioner.h" + +namespace legate::detail { + +Scatter::Scatter(std::shared_ptr target, + std::shared_ptr target_indirect, + std::shared_ptr source, + int64_t unique_id, + mapping::MachineDesc&& machine) + : Operation(unique_id, std::move(machine)), + target_{target.get(), declare_partition()}, + target_indirect_{target_indirect.get(), declare_partition()}, + source_{source.get(), declare_partition()}, + constraint_(legate::align(source_.variable, target_indirect_.variable)) +{ + record_partition(target_.variable, std::move(target)); + record_partition(target_indirect_.variable, std::move(target_indirect)); + record_partition(source_.variable, std::move(source)); +} + +void Scatter::validate() +{ + auto validate_store = [](auto* store) { + if (store->unbound() || store->has_scalar_storage() || store->transformed()) { + throw std::invalid_argument( + "Scatter accepts only normal, untransformed, region-backed stores"); + } + }; + validate_store(target_.store); + validate_store(target_indirect_.store); + validate_store(source_.store); + + if (!is_point_type(target_indirect_.store->type(), target_.store->dim())) { + throw std::invalid_argument("Indirection store should contain " + + std::to_string(target_.store->dim()) + "-D points"); + } + + constraint_->validate(); +} + +void Scatter::launch(Strategy* p_strategy) +{ + auto& strategy = *p_strategy; + CopyLauncher launcher(machine_); + auto launch_domain = strategy.launch_domain(this); + + launcher.add_input(source_.store, create_projection_info(strategy, launch_domain, source_)); + launcher.add_inout(target_.store, create_projection_info(strategy, launch_domain, target_)); + launcher.add_target_indirect(target_indirect_.store, + create_projection_info(strategy, launch_domain, target_indirect_)); + launcher.set_target_indirect_out_of_range(out_of_range_); + + if (launch_domain != nullptr) { + return launcher.execute(*launch_domain); + } else { + return launcher.execute_single(); + } +} + +void Scatter::add_to_solver(ConstraintSolver& solver) +{ + solver.add_constraint(constraint_.get()); + solver.add_partition_symbol(target_.variable); + solver.add_partition_symbol(target_indirect_.variable); + solver.add_partition_symbol(source_.variable); +} + +std::string Scatter::to_string() const { return "Scatter:" + std::to_string(unique_id_); } + +} // namespace legate::detail diff --git a/src/core/operation/detail/scatter.h b/src/core/operation/detail/scatter.h new file mode 100644 index 0000000000..23f94a058e --- /dev/null +++ b/src/core/operation/detail/scatter.h @@ -0,0 +1,58 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include + +#include "core/data/detail/logical_store.h" +#include "core/operation/detail/operation.h" +#include "core/partitioning/constraint.h" + +namespace legate::detail { + +class ConstraintSolver; + +class Scatter : public Operation { + public: + Scatter(std::shared_ptr target, + std::shared_ptr target_indirect, + std::shared_ptr source, + int64_t unique_id, + mapping::MachineDesc&& machine); + + public: + void set_indirect_out_of_range(bool flag) { out_of_range_ = flag; } + + public: + void validate() override; + void launch(Strategy* strategy) override; + + public: + void add_to_solver(ConstraintSolver& solver) override; + + public: + std::string to_string() const override; + + private: + bool out_of_range_{true}; + StoreArg target_; + StoreArg target_indirect_; + StoreArg source_; + std::unique_ptr constraint_; +}; + +} // namespace legate::detail diff --git a/src/core/operation/detail/scatter_gather.cc b/src/core/operation/detail/scatter_gather.cc new file mode 100644 index 0000000000..26e1eeb4ed --- /dev/null +++ b/src/core/operation/detail/scatter_gather.cc @@ -0,0 +1,117 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/operation/detail/scatter_gather.h" + +#include "core/operation/detail/copy_launcher.h" +#include "core/partitioning/constraint.h" +#include "core/partitioning/constraint_solver.h" +#include "core/partitioning/partition.h" +#include "core/partitioning/partitioner.h" + +namespace legate::detail { + +ScatterGather::ScatterGather(std::shared_ptr target, + std::shared_ptr target_indirect, + std::shared_ptr source, + std::shared_ptr source_indirect, + int64_t unique_id, + mapping::MachineDesc&& machine) + : Operation(unique_id, std::move(machine)), + target_{target.get(), declare_partition()}, + target_indirect_{target_indirect.get(), declare_partition()}, + source_{source.get(), declare_partition()}, + source_indirect_{source_indirect.get(), declare_partition()}, + constraint_(legate::align(source_indirect_.variable, target_indirect_.variable)) +{ + record_partition(target_.variable, std::move(target)); + record_partition(target_indirect_.variable, std::move(target_indirect)); + record_partition(source_.variable, std::move(source)); + record_partition(source_indirect_.variable, std::move(source_indirect)); +} + +void ScatterGather::set_source_indirect_out_of_range(bool flag) +{ + source_indirect_out_of_range_ = flag; +} + +void ScatterGather::set_target_indirect_out_of_range(bool flag) +{ + target_indirect_out_of_range_ = flag; +} + +void ScatterGather::validate() +{ + auto validate_store = [](auto* store) { + if (store->unbound() || store->has_scalar_storage() || store->transformed()) { + throw std::invalid_argument( + "ScatterGather accepts only normal, untransformed, region-backed stores"); + } + }; + validate_store(target_.store); + validate_store(target_indirect_.store); + validate_store(source_.store); + validate_store(source_indirect_.store); + + if (!is_point_type(source_indirect_.store->type(), source_.store->dim())) { + throw std::invalid_argument("Source indirection store should contain " + + std::to_string(source_.store->dim()) + "-D points"); + } + if (!is_point_type(target_indirect_.store->type(), target_.store->dim())) { + throw std::invalid_argument("Target indirection store should contain " + + std::to_string(target_.store->dim()) + "-D points"); + } + + constraint_->validate(); +} + +void ScatterGather::launch(Strategy* p_strategy) +{ + auto& strategy = *p_strategy; + CopyLauncher launcher(machine_); + auto launch_domain = strategy.launch_domain(this); + + launcher.add_input(source_.store, create_projection_info(strategy, launch_domain, source_)); + launcher.add_source_indirect(source_indirect_.store, + create_projection_info(strategy, launch_domain, source_indirect_)); + launcher.add_inout(target_.store, create_projection_info(strategy, launch_domain, target_)); + launcher.add_target_indirect(target_indirect_.store, + create_projection_info(strategy, launch_domain, target_indirect_)); + launcher.set_target_indirect_out_of_range(target_indirect_out_of_range_); + launcher.set_source_indirect_out_of_range(source_indirect_out_of_range_); + + if (launch_domain != nullptr) { + return launcher.execute(*launch_domain); + } else { + return launcher.execute_single(); + } +} + +void ScatterGather::add_to_solver(ConstraintSolver& solver) +{ + solver.add_constraint(constraint_.get()); + solver.add_partition_symbol(target_.variable); + solver.add_partition_symbol(target_indirect_.variable); + solver.add_partition_symbol(source_.variable); + solver.add_partition_symbol(source_indirect_.variable); +} + +std::string ScatterGather::to_string() const +{ + return "ScatterGather:" + std::to_string(unique_id_); +} + +} // namespace legate::detail diff --git a/src/core/operation/detail/scatter_gather.h b/src/core/operation/detail/scatter_gather.h new file mode 100644 index 0000000000..0a9c0f5459 --- /dev/null +++ b/src/core/operation/detail/scatter_gather.h @@ -0,0 +1,62 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include + +#include "core/data/detail/logical_store.h" +#include "core/operation/detail/operation.h" +#include "core/partitioning/constraint.h" + +namespace legate::detail { + +class ConstraintSolver; + +class ScatterGather : public Operation { + public: + ScatterGather(std::shared_ptr target, + std::shared_ptr target_indirect, + std::shared_ptr source, + std::shared_ptr source_indirect, + int64_t unique_id, + mapping::MachineDesc&& machine); + + public: + void set_source_indirect_out_of_range(bool flag); + void set_target_indirect_out_of_range(bool flag); + + public: + void validate() override; + void launch(detail::Strategy* strategy) override; + + public: + void add_to_solver(detail::ConstraintSolver& solver) override; + + public: + std::string to_string() const override; + + private: + bool source_indirect_out_of_range_{true}; + bool target_indirect_out_of_range_{true}; + StoreArg target_; + StoreArg target_indirect_; + StoreArg source_; + StoreArg source_indirect_; + std::unique_ptr constraint_; +}; + +} // namespace legate::detail diff --git a/src/core/operation/detail/task.cc b/src/core/operation/detail/task.cc index 14f0a91466..f9c47ea07e 100644 --- a/src/core/operation/detail/task.cc +++ b/src/core/operation/detail/task.cc @@ -235,12 +235,12 @@ AutoTask::AutoTask(const LibraryContext* library, { } -void AutoTask::add_input(std::shared_ptr&& store, const Variable* partition_symbol) +void AutoTask::add_input(std::shared_ptr store, const Variable* partition_symbol) { add_store(inputs_, std::move(store), partition_symbol); } -void AutoTask::add_output(std::shared_ptr&& store, const Variable* partition_symbol) +void AutoTask::add_output(std::shared_ptr store, const Variable* partition_symbol) { if (store->has_scalar_storage()) scalar_outputs_.push_back(outputs_.size()); @@ -249,7 +249,7 @@ void AutoTask::add_output(std::shared_ptr&& store, const Variable* add_store(outputs_, std::move(store), partition_symbol); } -void AutoTask::add_reduction(std::shared_ptr&& store, +void AutoTask::add_reduction(std::shared_ptr store, Legion::ReductionOpID redop, const Variable* partition_symbol) { @@ -262,7 +262,7 @@ void AutoTask::add_store(std::vector& store_args, std::shared_ptr store, const Variable* partition_symbol) { - store_args.push_back(StoreArg(store.get(), partition_symbol)); + store_args.push_back(StoreArg{store.get(), partition_symbol}); record_partition(partition_symbol, std::move(store)); } @@ -279,6 +279,11 @@ void AutoTask::add_to_solver(detail::ConstraintSolver& solver) for (auto& [_, symb] : inputs_) solver.add_partition_symbol(symb); } +void AutoTask::validate() +{ + for (auto& constraint : constraints_) constraint->validate(); +} + //////////////////////////////////////////////////// // legate::ManualTask //////////////////////////////////////////////////// @@ -296,17 +301,17 @@ ManualTask::ManualTask(const LibraryContext* library, strategy_->set_launch_shape(this, launch_shape); } -void ManualTask::add_input(std::shared_ptr&& store) +void ManualTask::add_input(std::shared_ptr store) { add_store(inputs_, std::move(store), create_no_partition()); } -void ManualTask::add_input(std::shared_ptr&& store_partition) +void ManualTask::add_input(std::shared_ptr store_partition) { add_store(inputs_, store_partition->store(), store_partition->partition()); } -void ManualTask::add_output(std::shared_ptr&& store) +void ManualTask::add_output(std::shared_ptr store) { if (store->has_scalar_storage()) scalar_outputs_.push_back(outputs_.size()); @@ -315,7 +320,7 @@ void ManualTask::add_output(std::shared_ptr&& store) add_store(outputs_, std::move(store), create_no_partition()); } -void ManualTask::add_output(std::shared_ptr&& store_partition) +void ManualTask::add_output(std::shared_ptr store_partition) { #ifdef DEBUG_LEGATE // TODO: We need to raise an exception for the user error in this case @@ -325,14 +330,14 @@ void ManualTask::add_output(std::shared_ptr&& store_parti add_store(outputs_, store_partition->store(), store_partition->partition()); } -void ManualTask::add_reduction(std::shared_ptr&& store, Legion::ReductionOpID redop) +void ManualTask::add_reduction(std::shared_ptr store, Legion::ReductionOpID redop) { if (store->has_scalar_storage()) scalar_reductions_.push_back(reductions_.size()); add_store(reductions_, std::move(store), create_no_partition()); reduction_ops_.push_back(redop); } -void ManualTask::add_reduction(std::shared_ptr&& store_partition, +void ManualTask::add_reduction(std::shared_ptr store_partition, Legion::ReductionOpID redop) { if (store_partition->store()->has_scalar_storage()) @@ -346,7 +351,7 @@ void ManualTask::add_store(std::vector& store_args, std::shared_ptr partition) { auto partition_symbol = declare_partition(); - store_args.push_back(StoreArg(store.get(), partition_symbol)); + store_args.push_back(StoreArg{store.get(), partition_symbol}); if (store->unbound()) { auto field_space = detail::Runtime::get_runtime()->create_field_space(); strategy_->insert(partition_symbol, std::move(partition), field_space); @@ -355,6 +360,8 @@ void ManualTask::add_store(std::vector& store_args, all_stores_.insert(std::move(store)); } +void ManualTask::validate() {} + void ManualTask::launch(Strategy*) { Task::launch(strategy_.get()); } void ManualTask::add_to_solver(ConstraintSolver& solver) {} diff --git a/src/core/operation/detail/task.h b/src/core/operation/detail/task.h index 626fb21039..89d22db6cf 100644 --- a/src/core/operation/detail/task.h +++ b/src/core/operation/detail/task.h @@ -88,9 +88,9 @@ class AutoTask : public Task { ~AutoTask() {} public: - void add_input(std::shared_ptr&& store, const Variable* partition_symbol); - void add_output(std::shared_ptr&& store, const Variable* partition_symbol); - void add_reduction(std::shared_ptr&& store, + void add_input(std::shared_ptr store, const Variable* partition_symbol); + void add_output(std::shared_ptr store, const Variable* partition_symbol); + void add_reduction(std::shared_ptr store, Legion::ReductionOpID redop, const Variable* partition_symbol); @@ -103,6 +103,9 @@ class AutoTask : public Task { void add_constraint(std::unique_ptr constraint); void add_to_solver(ConstraintSolver& solver) override; + public: + void validate() override; + private: std::vector> constraints_{}; }; @@ -120,12 +123,12 @@ class ManualTask : public Task { ~ManualTask(); public: - void add_input(std::shared_ptr&& store); - void add_input(std::shared_ptr&& store_partition); - void add_output(std::shared_ptr&& store); - void add_output(std::shared_ptr&& store_partition); - void add_reduction(std::shared_ptr&& store, Legion::ReductionOpID redop); - void add_reduction(std::shared_ptr&& store_partition, + void add_input(std::shared_ptr store); + void add_input(std::shared_ptr store_partition); + void add_output(std::shared_ptr store); + void add_output(std::shared_ptr store_partition); + void add_reduction(std::shared_ptr store, Legion::ReductionOpID redop); + void add_reduction(std::shared_ptr store_partition, Legion::ReductionOpID redop); private: @@ -134,6 +137,7 @@ class ManualTask : public Task { std::shared_ptr partition); public: + void validate() override; void launch(Strategy* strategy) override; public: diff --git a/src/core/partitioning/constraint.cc b/src/core/partitioning/constraint.cc index 4d66d903b4..654c22f0c9 100644 --- a/src/core/partitioning/constraint.cc +++ b/src/core/partitioning/constraint.cc @@ -50,7 +50,7 @@ std::string Variable::to_string() const return std::move(ss).str(); } -Alignment::Alignment(std::unique_ptr&& lhs, std::unique_ptr&& rhs) +Alignment::Alignment(std::unique_ptr&& lhs, std::unique_ptr&& rhs) : lhs_(std::move(lhs)), rhs_(std::move(rhs)) { } @@ -61,6 +61,16 @@ void Alignment::find_partition_symbols(std::vector& partition_s rhs_->find_partition_symbols(partition_symbols); } +void Alignment::validate() const +{ + auto* lhs_store = lhs_->operation()->find_store(lhs_.get()); + auto* rhs_store = rhs_->operation()->find_store(rhs_.get()); + if (lhs_store->extents() != rhs_store->extents()) + throw std::invalid_argument("Alignment requires the stores to have the same shape, but found " + + lhs_store->extents().to_string() + " and " + + rhs_store->extents().to_string()); +} + std::string Alignment::to_string() const { std::stringstream ss; @@ -78,6 +88,16 @@ void Broadcast::find_partition_symbols(std::vector& partition_s partition_symbols.push_back(variable_.get()); } +void Broadcast::validate() const +{ + auto* store = variable_->operation()->find_store(variable_.get()); + for (auto axis : axes_.data()) { + if (axis < 0 || axis >= store->dim()) + throw std::invalid_argument("Invalid broadcasting dimension " + std::to_string(axis) + + " for a " + std::to_string(store->dim()) + "-D store"); + } +} + std::string Broadcast::to_string() const { std::stringstream ss; diff --git a/src/core/partitioning/constraint.h b/src/core/partitioning/constraint.h index 98280d3766..140da91f7c 100644 --- a/src/core/partitioning/constraint.h +++ b/src/core/partitioning/constraint.h @@ -176,6 +176,8 @@ struct Constraint { */ virtual Kind kind() const = 0; + virtual void validate() const = 0; + virtual const Alignment* as_alignment() const = 0; virtual const Broadcast* as_broadcast() const = 0; }; @@ -190,7 +192,7 @@ struct Constraint { */ class Alignment : public Constraint { public: - Alignment(std::unique_ptr&& lhs, std::unique_ptr&& rhs); + Alignment(std::unique_ptr&& lhs, std::unique_ptr&& rhs); public: Kind kind() const override { return Kind::ALIGNMENT; } @@ -198,6 +200,9 @@ class Alignment : public Constraint { public: void find_partition_symbols(std::vector& partition_symbols) const override; + public: + void validate() const; + public: std::string to_string() const override; @@ -209,19 +214,19 @@ class Alignment : public Constraint { /** * @brief Returns the LHS of the alignment constraint * - * @return Expression + * @return Variable */ - const Expr* lhs() const { return lhs_.get(); } + const Variable* lhs() const { return lhs_.get(); } /** * @brief Returns the RHS of the alignment constraint * - * @return Expression + * @return Variable */ - const Expr* rhs() const { return rhs_.get(); } + const Variable* rhs() const { return rhs_.get(); } private: - std::unique_ptr lhs_; - std::unique_ptr rhs_; + std::unique_ptr lhs_; + std::unique_ptr rhs_; }; /** @@ -243,6 +248,9 @@ class Broadcast : public Constraint { public: void find_partition_symbols(std::vector& partition_symbols) const override; + public: + void validate() const; + public: std::string to_string() const override; diff --git a/src/core/partitioning/partitioner.h b/src/core/partitioning/partitioner.h index 1f9c9873e1..6382160b1d 100644 --- a/src/core/partitioning/partitioner.h +++ b/src/core/partitioning/partitioner.h @@ -23,10 +23,8 @@ #include "core/data/shape.h" #include "core/partitioning/constraint.h" #include "core/partitioning/restriction.h" -#include "legion.h" namespace legate { -class Operation; class Partition; } // namespace legate @@ -35,7 +33,6 @@ namespace legate::detail { class ConstraintSolver; class LogicalStore; class Partitioner; -class ProjectionInfo; class Strategy { friend class Partitioner; diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index d87a22cc20..8877ffd6c7 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -23,6 +23,9 @@ #include "core/mapping/default_mapper.h" #include "core/operation/detail/copy.h" #include "core/operation/detail/fill.h" +#include "core/operation/detail/gather.h" +#include "core/operation/detail/scatter.h" +#include "core/operation/detail/scatter_gather.h" #include "core/operation/detail/task.h" #include "core/operation/detail/task_launcher.h" #include "core/partitioning/partitioner.h" @@ -181,11 +184,49 @@ std::unique_ptr Runtime::create_task(LibraryContext* library, return std::unique_ptr(task); } -std::unique_ptr Runtime::create_copy() +void Runtime::issue_copy(std::shared_ptr target, std::shared_ptr source) { auto machine = machine_manager_->get_machine(); - auto copy = new Copy(next_unique_id_++, std::move(machine)); - return std::unique_ptr(copy); + submit(std::make_unique( + std::move(target), std::move(source), next_unique_id_++, std::move(machine))); +} + +void Runtime::issue_gather(std::shared_ptr target, + std::shared_ptr source, + std::shared_ptr source_indirect) +{ + auto machine = machine_manager_->get_machine(); + submit(std::make_unique(std::move(target), + std::move(source), + std::move(source_indirect), + next_unique_id_++, + std::move(machine))); +} + +void Runtime::issue_scatter(std::shared_ptr target, + std::shared_ptr target_indirect, + std::shared_ptr source) +{ + auto machine = machine_manager_->get_machine(); + submit(std::make_unique(std::move(target), + std::move(target_indirect), + std::move(source), + next_unique_id_++, + std::move(machine))); +} + +void Runtime::issue_scatter_gather(std::shared_ptr target, + std::shared_ptr target_indirect, + std::shared_ptr source, + std::shared_ptr source_indirect) +{ + auto machine = machine_manager_->get_machine(); + submit(std::make_unique(std::move(target), + std::move(target_indirect), + std::move(source), + std::move(source_indirect), + next_unique_id_++, + std::move(machine))); } void Runtime::issue_fill(std::shared_ptr lhs, std::shared_ptr value) @@ -206,6 +247,7 @@ void Runtime::flush_scheduling_window() void Runtime::submit(std::unique_ptr op) { + op->validate(); operations_.push_back(std::move(op)); if (operations_.size() >= window_size_) { flush_scheduling_window(); } } diff --git a/src/core/runtime/detail/runtime.h b/src/core/runtime/detail/runtime.h index 6248c9a64a..f429f2956c 100644 --- a/src/core/runtime/detail/runtime.h +++ b/src/core/runtime/detail/runtime.h @@ -72,7 +72,17 @@ class Runtime { std::unique_ptr create_task(LibraryContext* library, int64_t task_id, const Shape& launch_shape); - std::unique_ptr create_copy(); + void issue_copy(std::shared_ptr target, std::shared_ptr source); + void issue_gather(std::shared_ptr target, + std::shared_ptr source, + std::shared_ptr source_indirect); + void issue_scatter(std::shared_ptr target, + std::shared_ptr target_indirect, + std::shared_ptr source); + void issue_scatter_gather(std::shared_ptr target, + std::shared_ptr target_indirect, + std::shared_ptr source, + std::shared_ptr source_indirect); void issue_fill(std::shared_ptr lhs, std::shared_ptr value); void flush_scheduling_window(); void submit(std::unique_ptr op); diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 460f259ae9..2921fdbb5e 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -163,7 +163,29 @@ ManualTask Runtime::create_task(LibraryContext* library, int64_t task_id, const return ManualTask(impl_->create_task(library, task_id, launch_shape)); } -Copy Runtime::create_copy() { return Copy(impl_->create_copy()); } +void Runtime::issue_copy(LogicalStore target, LogicalStore source) +{ + impl_->issue_copy(target.impl(), source.impl()); +} + +void Runtime::issue_gather(LogicalStore target, LogicalStore source, LogicalStore source_indirect) +{ + impl_->issue_gather(target.impl(), source.impl(), source_indirect.impl()); +} + +void Runtime::issue_scatter(LogicalStore target, LogicalStore target_indirect, LogicalStore source) +{ + impl_->issue_scatter(target.impl(), target_indirect.impl(), source.impl()); +} + +void Runtime::issue_scatter_gather(LogicalStore target, + LogicalStore target_indirect, + LogicalStore source, + LogicalStore source_indirect) +{ + impl_->issue_scatter_gather( + target.impl(), target_indirect.impl(), source.impl(), source_indirect.impl()); +} void Runtime::issue_fill(LogicalStore lhs, LogicalStore value) { @@ -179,8 +201,6 @@ void Runtime::submit(AutoTask&& task) { impl_->submit(std::move(task.impl_)); } void Runtime::submit(ManualTask&& task) { impl_->submit(std::move(task.impl_)); } -void Runtime::submit(Copy&& copy) { impl_->submit(std::move(copy.impl_)); } - LogicalStore Runtime::create_store(std::unique_ptr type, int32_t dim) { return LogicalStore(impl_->create_store(std::move(type), dim)); diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 0443301ebd..0e17b26ba7 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -22,7 +22,6 @@ #include "core/data/logical_store.h" #include "core/data/shape.h" #include "core/data/store.h" -#include "core/operation/copy.h" #include "core/operation/task.h" #include "core/runtime/resource.h" #include "core/task/exception.h" @@ -158,11 +157,49 @@ class Runtime { */ ManualTask create_task(LibraryContext* library, int64_t task_id, const Shape& launch_shape); /** - * @brief Creates a Copy + * @brief Issues a copy between stores. * - * @return Copy object + * The source and target stores must have the same shape. + * + * @param target Copy target + * @param source Copy source + */ + void issue_copy(LogicalStore target, LogicalStore source); + /** + * @brief Issues a gather copy between stores. + * + * The indirection store and the target store must have the same shape. + * + * @param target Copy target + * @param source Copy source + * @param source_indirect Store for source indirection + */ + void issue_gather(LogicalStore target, LogicalStore source, LogicalStore source_indirect); + /** + * @brief Issues a scatter copy between stores. + * + * The indirection store and the source store must have the same shape. + * + * @param target Copy target + * @param target_indirect Store for target indirection + * @param source Copy source + */ + void issue_scatter(LogicalStore target, LogicalStore target_indirect, LogicalStore source); + /** + * @brief Issues a scatter-gather copy between stores. + * + * The indirection stores must have the same shape. + * + * @param target Copy target + * @param target_indirect Store for target indirection + * @param source Copy source + * @param source_indirect Store for source indirection */ - Copy create_copy(); + void issue_scatter_gather(LogicalStore target, + LogicalStore target_indirect, + LogicalStore source, + LogicalStore source_indirect); + /** * @brief Fills a given store with a constant * @@ -195,15 +232,6 @@ class Runtime { * @param task A ManualTask to execute */ void submit(ManualTask&& task); - /** - * @brief Submits a copy for execution - * - * Each submitted operation goes through multiple pipeline steps to eventually get scheduled - * for execution. It's not guaranteed that the submitted operation starts executing immediately. - * - * @param copy A Copy to execute - */ - void submit(Copy&& copy); public: /** diff --git a/src/core/type/type_info.cc b/src/core/type/type_info.cc index bb87b25a84..47722e386b 100644 --- a/src/core/type/type_info.cc +++ b/src/core/type/type_info.cc @@ -314,18 +314,34 @@ constexpr int32_t RECT_UID_BASE = POINT_UID_BASE + LEGATE_MAX_DIM + 1; } // namespace -std::unique_ptr point_type(int32_t dim) +std::unique_ptr point_type(int32_t ndim) { - if (dim == 1) return int64(); - return std::make_unique(POINT_UID_BASE + dim, int64(), dim); + if (ndim == 1) return int64(); + return std::make_unique(POINT_UID_BASE + ndim, int64(), ndim); } -std::unique_ptr rect_type(int32_t dim) +std::unique_ptr rect_type(int32_t ndim) { std::vector> field_types; - field_types.push_back(point_type(dim)); - field_types.push_back(point_type(dim)); - return std::make_unique(RECT_UID_BASE + dim, std::move(field_types), true /*align*/); + field_types.push_back(point_type(ndim)); + field_types.push_back(point_type(ndim)); + return std::make_unique(RECT_UID_BASE + ndim, std::move(field_types), true /*align*/); +} + +bool is_point_type(const Type& type, int32_t ndim) +{ + switch (type.code) { + case Type::Code::INT64: { + return 1 == ndim; + } + case Type::Code::FIXED_ARRAY: { + return static_cast(type).num_elements() == ndim; + } + default: { + return false; + } + } + return false; } } // namespace legate diff --git a/src/core/type/type_info.h b/src/core/type/type_info.h index 30abacc7b4..6cdbb75b54 100644 --- a/src/core/type/type_info.h +++ b/src/core/type/type_info.h @@ -535,7 +535,7 @@ std::unique_ptr string(); * * @return Type object */ -std::unique_ptr point_type(int32_t dim); +std::unique_ptr point_type(int32_t ndim); /** * @ingroup types @@ -543,6 +543,15 @@ std::unique_ptr point_type(int32_t dim); * * @return Type object */ -std::unique_ptr rect_type(int32_t dim); +std::unique_ptr rect_type(int32_t ndim); + +/** + * @ingroup types + * @brief Checks if the type is a point type of the given dimensionality + * + * @return true If the `type` is a point type + * @return false Otherwise + */ +bool is_point_type(const Type& type, int32_t ndim); } // namespace legate diff --git a/src/legate.h b/src/legate.h index f3b1564918..2aa129372f 100644 --- a/src/legate.h +++ b/src/legate.h @@ -30,7 +30,6 @@ #include "core/data/store.h" #include "core/legate_c.h" #include "core/mapping/mapping.h" -#include "core/operation/copy.h" #include "core/operation/task.h" #include "core/partitioning/constraint.h" #include "core/partitioning/partition.h" diff --git a/tests/cpp/integration/alignment_constraints.cc b/tests/cpp/integration/alignment_constraints.cc index 5224e960c7..c35ddaeea6 100644 --- a/tests/cpp/integration/alignment_constraints.cc +++ b/tests/cpp/integration/alignment_constraints.cc @@ -201,15 +201,47 @@ void test_alignment_transformed() launch_tester(store1.delinearize(0, {10, 10}), store4); } +void test_invalid_alignment() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + auto store1 = runtime->create_store({10}, legate::int64()); + auto store2 = runtime->create_store({9}, legate::int64()); + auto task = runtime->create_task(context, TRANSFORMED_TESTER + store1.dim()); + + auto part1 = task.declare_partition(); + auto part2 = task.declare_partition(); + task.add_output(store1, part1); + task.add_output(store2, part2); + task.add_constraint(legate::align(part1, part2)); + EXPECT_THROW(runtime->submit(std::move(task)), std::invalid_argument); +} + } // namespace -TEST(Integration, AlignmentConstraints) +TEST(Alignment, Basic) { legate::Core::perform_registration(); - test_alignment(); +} + +TEST(Alignment, WithBroadcast) +{ + legate::Core::perform_registration(); test_alignment_and_broadcast(); +} + +TEST(Alignment, WithTransform) +{ + legate::Core::perform_registration(); test_alignment_transformed(); } +TEST(Alignment, Invalid) +{ + legate::Core::perform_registration(); + test_invalid_alignment(); +} + } // namespace alignment_constraints diff --git a/tests/cpp/integration/broadcast_constraints.cc b/tests/cpp/integration/broadcast_constraints.cc index dcd34a7c4b..73bbe43527 100644 --- a/tests/cpp/integration/broadcast_constraints.cc +++ b/tests/cpp/integration/broadcast_constraints.cc @@ -117,14 +117,37 @@ void test_promoted_store() launch_tester(1); } +void test_invalid_broadcast() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + auto task = runtime->create_task(context, INITIALIZER); + auto store = runtime->create_store({10}, legate::int64()); + auto part = task.declare_partition(); + task.add_output(store, part); + task.add_constraint(legate::broadcast(part, {1})); + EXPECT_THROW(runtime->submit(std::move(task)), std::invalid_argument); +} + } // namespace -TEST(Integration, BroadcastConstraints) +TEST(Broadcast, Basic) { legate::Core::perform_registration(); - test_normal_store(); +} + +TEST(Broadcast, WithPromotion) +{ + legate::Core::perform_registration(); test_promoted_store(); } +TEST(Broadcast, Invalid) +{ + legate::Core::perform_registration(); + test_invalid_broadcast(); +} + } // namespace broadcast_constraints diff --git a/tests/cpp/integration/copy_failure.cc b/tests/cpp/integration/copy_failure.cc index 0bf78ef54c..7f1b686652 100644 --- a/tests/cpp/integration/copy_failure.cc +++ b/tests/cpp/integration/copy_failure.cc @@ -20,93 +20,75 @@ namespace copy_failure { -void test_input_output_failure() +void test_invalid_stores() { - // inconsistent number of inputs and outputs auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library("legate.core"); - - std::vector extents = {100}; - auto in_store1 = runtime->create_store(extents, legate::int64()); - auto in_store2 = runtime->create_store(extents, legate::int64()); - auto out_store = runtime->create_store(extents, legate::int64()); - // fill input store with some values - auto copy = runtime->create_copy(); - copy.add_input(in_store1); - copy.add_input(in_store2); - copy.add_output(out_store); - EXPECT_THROW(runtime->submit(std::move(copy)), std::runtime_error); -} -void test_indirect_failure() -{ - // using indirect with several inputs/outputs - auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library("legate.core"); - - std::vector extents = {100}; - auto in_store1 = runtime->create_store(extents, legate::int64()); - auto in_store2 = runtime->create_store(extents, legate::int64()); - auto out_store1 = runtime->create_store(extents, legate::int64()); - auto out_store2 = runtime->create_store(extents, legate::int64()); - auto indirect_store = runtime->create_store(extents, legate::int64()); - - auto copy = runtime->create_copy(); - copy.add_input(in_store1); - copy.add_input(in_store2); - copy.add_output(out_store1); - copy.add_output(out_store2); - copy.add_target_indirect(indirect_store); - - EXPECT_THROW(runtime->submit(std::move(copy)), std::runtime_error); + auto store1 = runtime->create_store({10, 10}, legate::int64()); + auto store2 = runtime->create_store({1}, legate::int64(), true /*optimize_scalar*/); + auto store3 = runtime->create_store(legate::int64()); + auto store4 = runtime->create_store({10, 10}, legate::int64()).promote(2, 10); + + EXPECT_THROW(runtime->issue_copy(store2, store1), std::invalid_argument); + EXPECT_THROW(runtime->issue_copy(store3, store1), std::invalid_argument); + EXPECT_THROW(runtime->issue_copy(store4, store1), std::invalid_argument); + + EXPECT_THROW(runtime->issue_gather(store2, store3, store1), std::invalid_argument); + EXPECT_THROW(runtime->issue_gather(store3, store4, store1), std::invalid_argument); + EXPECT_THROW(runtime->issue_gather(store4, store2, store1), std::invalid_argument); + + EXPECT_THROW(runtime->issue_scatter(store2, store3, store1), std::invalid_argument); + EXPECT_THROW(runtime->issue_scatter(store3, store4, store1), std::invalid_argument); + EXPECT_THROW(runtime->issue_scatter(store4, store2, store1), std::invalid_argument); + + EXPECT_THROW(runtime->issue_scatter_gather(store2, store3, store4, store1), + std::invalid_argument); + EXPECT_THROW(runtime->issue_scatter_gather(store3, store4, store2, store1), + std::invalid_argument); + EXPECT_THROW(runtime->issue_scatter_gather(store4, store2, store3, store1), + std::invalid_argument); } void test_shape_check_failure() { auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library("legate.core"); auto store1 = runtime->create_store({10, 10}, legate::int64()); auto store2 = runtime->create_store({5, 20}, legate::int64()); auto store3 = runtime->create_store({20, 5}, legate::int64()); + auto store4 = runtime->create_store({5, 5}, legate::int64()); - { - auto copy = runtime->create_copy(); - copy.add_input(store1); - copy.add_output(store2); - EXPECT_THROW(runtime->submit(std::move(copy)), std::runtime_error); - } - - { - auto copy = runtime->create_copy(); - copy.add_input(store1); - copy.add_source_indirect(store2); - copy.add_output(store3); - EXPECT_THROW(runtime->submit(std::move(copy)), std::runtime_error); - } - - { - auto copy = runtime->create_copy(); - copy.add_input(store1); - copy.add_target_indirect(store2); - copy.add_output(store3); - EXPECT_THROW(runtime->submit(std::move(copy)), std::runtime_error); - } - - { - auto copy = runtime->create_copy(); - copy.add_input(store1); - copy.add_source_indirect(store2); - copy.add_target_indirect(store3); - copy.add_output(store1); - EXPECT_THROW(runtime->submit(std::move(copy)), std::runtime_error); - } + EXPECT_THROW(runtime->issue_copy(store2, store1), std::invalid_argument); + + EXPECT_THROW(runtime->issue_gather(store3, store2, store1), std::invalid_argument); + + EXPECT_THROW(runtime->issue_scatter(store3, store2, store1), std::invalid_argument); + + EXPECT_THROW(runtime->issue_scatter_gather(store4, store3, store2, store1), + std::invalid_argument); } -TEST(Copy, FailureInputOutputMismatch) { test_input_output_failure(); } +void test_non_point_types_failure() +{ + auto runtime = legate::Runtime::get_runtime(); + + auto store1 = runtime->create_store({10, 10}, legate::int32()); + auto store2 = runtime->create_store({10, 10}, legate::int32()); + auto store3 = runtime->create_store({10, 10}, legate::int32()); + auto store4 = runtime->create_store({10, 10}, legate::int32()); -TEST(Copy, FailureMultipleIndirectCopies) { test_indirect_failure(); } + EXPECT_THROW(runtime->issue_gather(store3, store2, store1), std::invalid_argument); + + EXPECT_THROW(runtime->issue_scatter(store3, store2, store1), std::invalid_argument); + + EXPECT_THROW(runtime->issue_scatter_gather(store4, store3, store2, store1), + std::invalid_argument); +} + +TEST(Copy, FailureInvalidStores) { test_invalid_stores(); } TEST(Copy, FailureDifferentShapes) { test_shape_check_failure(); } +TEST(Copy, FailureNonPointTypes) { test_non_point_types_failure(); } + } // namespace copy_failure diff --git a/tests/cpp/integration/copy_gather.cc b/tests/cpp/integration/copy_gather.cc index ca6489f5a1..c10e2c109d 100644 --- a/tests/cpp/integration/copy_gather.cc +++ b/tests/cpp/integration/copy_gather.cc @@ -144,11 +144,7 @@ void test_gather(const GatherSpec& spec) fill_input(context, src, spec.seed); fill_indirect(context, ind, src); - auto copy = runtime->create_copy(); - copy.add_input(src); - copy.add_output(tgt); - copy.add_source_indirect(ind); - runtime->submit(std::move(copy)); + runtime->issue_gather(tgt, src, ind); check_gather_output(context, src, tgt, ind); } diff --git a/tests/cpp/integration/copy_gather_scatter.cc b/tests/cpp/integration/copy_gather_scatter.cc index f5478a315a..efcd290cac 100644 --- a/tests/cpp/integration/copy_gather_scatter.cc +++ b/tests/cpp/integration/copy_gather_scatter.cc @@ -173,13 +173,7 @@ void test_gather_scatter(const GatherScatterSpec& spec) fill_indirect(context, src_ind, src); fill_indirect(context, tgt_ind, tgt); runtime->issue_fill(tgt, spec.init); - - auto copy = runtime->create_copy(); - copy.add_input(src); - copy.add_output(tgt); - copy.add_source_indirect(src_ind); - copy.add_target_indirect(tgt_ind); - runtime->submit(std::move(copy)); + runtime->issue_scatter_gather(tgt, tgt_ind, src, src_ind); check_gather_scatter_output(context, src, tgt, src_ind, tgt_ind, spec.init); } diff --git a/tests/cpp/integration/copy_normal.cc b/tests/cpp/integration/copy_normal.cc index 013f001ebd..a9c3fcd138 100644 --- a/tests/cpp/integration/copy_normal.cc +++ b/tests/cpp/integration/copy_normal.cc @@ -96,37 +96,21 @@ struct NormalCopySpec { legate::Scalar seed; }; -void test_normal_copies(const std::vector specs) +void test_normal_copy(const NormalCopySpec& spec) { auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(library_name); - std::vector inputs; - std::vector outputs; + auto& [shape, type, seed] = spec; - for (auto& [shape, type, seed] : specs) { - auto input = runtime->create_store(shape, type.clone()); - fill_input(context, input, seed); - inputs.push_back(std::move(input)); - outputs.push_back(runtime->create_store(shape, type.clone())); - } + auto input = runtime->create_store(shape, type.clone()); + auto output = runtime->create_store(shape, type.clone()); - auto copy = runtime->create_copy(); - for (auto& input : inputs) copy.add_input(input); - for (auto& output : outputs) copy.add_output(output); - runtime->submit(std::move(copy)); + fill_input(context, input, seed); + runtime->issue_copy(output, input); // check the result of copy - for (uint32_t idx = 0; idx < outputs.size(); ++idx) { - auto& output = outputs[idx]; - auto& input = inputs.at(idx); - check_output(context, input, output); - } -} - -void test_all_normal_copies() -{ - test_normal_copies({{{1000, 100}, *uint32, legate::Scalar(uint32_t(3))}}); + check_output(context, input, output); } TEST(Copy, Single) @@ -135,29 +119,8 @@ TEST(Copy, Single) // For some reason, clang-format gets tripped over by singleton initialization lists, // so factor out the definition here std::vector shape1d{9}; - test_normal_copies({{{4, 7}, *int64, legate::Scalar(int64_t(12))}}); - test_normal_copies({{{1000, 100}, *uint32, legate::Scalar(uint32_t(3))}}); -} - -TEST(Copy, Multi2) -{ - legate::Core::perform_registration(); - std::vector shape1d{13}; - test_normal_copies({ - {shape1d, *float64, legate::Scalar(double(5.0))}, - {{3, 7, 5}, *uint32, legate::Scalar(uint32_t(456))}, - }); -} - -TEST(Copy, Multi3) -{ - legate::Core::perform_registration(); - std::vector shape1d{11}; - test_normal_copies({ - {shape1d, *int64, legate::Scalar(int64_t(789))}, - {{5, 5}, *float64, legate::Scalar(double(10.0))}, - {{2, 4, 4}, *uint32, legate::Scalar(uint32_t(7))}, - }); + test_normal_copy({{4, 7}, *int64, legate::Scalar(int64_t(12))}); + test_normal_copy({{1000, 100}, *uint32, legate::Scalar(uint32_t(3))}); } } // namespace copy_normal diff --git a/tests/cpp/integration/copy_scatter.cc b/tests/cpp/integration/copy_scatter.cc index e1e2454354..9e64c68119 100644 --- a/tests/cpp/integration/copy_scatter.cc +++ b/tests/cpp/integration/copy_scatter.cc @@ -170,12 +170,7 @@ void test_scatter(const ScatterSpec& spec) fill_input(context, src, spec.seed); fill_indirect(context, ind, tgt); runtime->issue_fill(tgt, spec.init); - - auto copy = runtime->create_copy(); - copy.add_input(src); - copy.add_output(tgt); - copy.add_target_indirect(ind); - runtime->submit(std::move(copy)); + runtime->issue_scatter(tgt, ind, src); check_scatter_output(context, src, tgt, ind, spec.init); } From e430cf5666624d888f5c5a7a192c2acdd0df6d62 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 30 Jun 2023 23:15:27 -0700 Subject: [PATCH 0157/1425] More safety checks for string types in Scalar * More safety checks for string types in Scalar See merge request legate/legate.core.internal!71 --- src/core/data/scalar.h | 20 ++++++++++++++++---- src/core/data/scalar.inl | 23 +++++++++++++++++++++-- tests/cpp/unit/scalar.cc | 7 ++++--- 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/src/core/data/scalar.h b/src/core/data/scalar.h index 538663fd42..4d3b0c9a31 100644 --- a/src/core/data/scalar.h +++ b/src/core/data/scalar.h @@ -129,20 +129,32 @@ class Scalar { public: /** - * @brief Returns the value stored in the `Scalar`. The call does no type checking; - * i.e., passing a wrong type parameter will not be caught by the call. + * @brief Returns a copy of the value stored in this `Scalar`. * * @tparam VAL Type of the value to unwrap * - * @return The value stored in the `Scalar` + * @return A copy of the value stored in this `Scalar` + * + * @throw std::invalid_argument If one of the following cases is encountered: + * + * 1) size of the scalar does not match with size of `VAL`, + * 2) the scalar holds a string but `VAL` isn't `std:string` or `std:string_view`, or + * 3) the inverse; i.e., `VAL` is `std:string` or `std:string_view` but the scalar's type + * isn't string */ template VAL value() const; /** - * @brief Returns values stored in the `Scalar`. If the `Scalar` contains a scalar, + * @brief Returns values stored in the `Scalar`. If the `Scalar` does not have a fixed array type, * a unit span will be returned. * * @return Values stored in the `Scalar` + * + * @throw std::invalid_argument If one of the following cases is encountered: + * + * 1) the scalar has a fixed array type whose elemenet type has a different size from `VAL`, + * 2) the scalar holds a string and size of `VAL` isn't 1 byte, + * 3) the scalar's type isn't a fixed array type and the size is different from size of `VAL` */ template Span values() const; diff --git a/src/core/data/scalar.inl b/src/core/data/scalar.inl index a2c13364cb..eff6987820 100644 --- a/src/core/data/scalar.inl +++ b/src/core/data/scalar.inl @@ -16,6 +16,8 @@ #pragma once +#include + // Useful for IDEs #include "core/data/scalar.h" @@ -74,6 +76,8 @@ Scalar::Scalar(const Rect& rect) : own_(true), type_(rect_type(DIM)) template VAL Scalar::value() const { + if (type_->code == Type::Code::STRING) + throw std::invalid_argument("String cannot be casted to other types"); if (sizeof(VAL) != type_->size()) throw std::invalid_argument("Size of the scalar is " + std::to_string(type_->size()) + ", but the requested type has size " + std::to_string(sizeof(VAL))); @@ -85,14 +89,22 @@ inline std::string Scalar::value() const { if (type_->code != Type::Code::STRING) throw std::invalid_argument("Type of the scalar is not string"); - // Getting a span of a temporary scalar is illegal in general, - // but we know this is safe as the span's pointer is held by this object. auto len = *static_cast(data_); const auto* begin = static_cast(data_) + sizeof(uint32_t); const auto* end = begin + len; return std::string(begin, end); } +template <> +inline std::string_view Scalar::value() const +{ + if (type_->code != Type::Code::STRING) + throw std::invalid_argument("Type of the scalar is not string"); + auto len = *static_cast(data_); + const auto* begin = static_cast(data_) + sizeof(uint32_t); + return std::string_view(begin, len); +} + template Span Scalar::values() const { @@ -105,6 +117,13 @@ Span Scalar::values() const ", but the requested element type has size " + std::to_string(sizeof(VAL))); auto size = arr_type->num_elements(); return Span(reinterpret_cast(data_), size); + } else if (type_->code == Type::Code::STRING) { + if (sizeof(VAL) != 1) + throw std::invalid_argument( + "String scalar can only be converted into a span of a type whose size is 1 byte"); + auto len = *static_cast(data_); + const auto* begin = static_cast(data_) + sizeof(uint32_t); + return Span(reinterpret_cast(begin), len); } else { if (sizeof(VAL) != type_->size()) throw std::invalid_argument("Size of the scalar is " + std::to_string(type_->size()) + diff --git a/tests/cpp/unit/scalar.cc b/tests/cpp/unit/scalar.cc index 8ea5f43337..af49cdde30 100644 --- a/tests/cpp/unit/scalar.cc +++ b/tests/cpp/unit/scalar.cc @@ -165,9 +165,10 @@ TEST(ScalarUnit, CreateWithString) // Check values EXPECT_EQ(scalar.value(), inputString); - legate::Span actualValues(scalar.values()); - EXPECT_EQ(actualValues.size(), 1); - EXPECT_EQ(*actualValues.begin(), inputString.substr(0, 1)); + EXPECT_EQ(scalar.value(), inputString); + legate::Span actualValues(scalar.values()); + EXPECT_EQ(actualValues.size(), inputString.size()); + EXPECT_EQ(*actualValues.begin(), inputString[0]); // Invalid type EXPECT_THROW(scalar.value(), std::invalid_argument); From 4a97d533156561cf811e973ac21ca3661e288e8d Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Sat, 1 Jul 2023 01:25:27 -0700 Subject: [PATCH 0158/1425] Find_or_create method for library initialization * Add a find_or_create method for library initialization See merge request legate/legate.core.internal!72 --- src/core/operation/detail/task.cc | 2 +- src/core/runtime/context.cc | 36 ++++++++++++++------ src/core/runtime/context.h | 7 +++- src/core/runtime/detail/runtime.cc | 46 +++++++++++++++---------- src/core/runtime/detail/runtime.h | 10 ++++-- src/core/runtime/runtime.cc | 7 ++-- src/core/runtime/runtime.h | 39 +++++++++++++++------ tests/cpp/unit/library.cc | 54 ++++++++++++++++++++++++++++++ 8 files changed, 154 insertions(+), 47 deletions(-) create mode 100644 tests/cpp/unit/library.cc diff --git a/src/core/operation/detail/task.cc b/src/core/operation/detail/task.cc index f9c47ea07e..fdb0d68a87 100644 --- a/src/core/operation/detail/task.cc +++ b/src/core/operation/detail/task.cc @@ -219,7 +219,7 @@ void Task::demux_scalar_stores(const Legion::FutureMap& result, const Domain& la std::string Task::to_string() const { std::stringstream ss; - ss << library_->find_task(task_id_)->name() << ":" << unique_id_; + ss << library_->get_task_name(task_id_) << ":" << unique_id_; return std::move(ss).str(); } diff --git a/src/core/runtime/context.cc b/src/core/runtime/context.cc index 22f3e2a552..6d877a3b68 100644 --- a/src/core/runtime/context.cc +++ b/src/core/runtime/context.cc @@ -47,9 +47,7 @@ const char* InvalidTaskIdException::what() const throw() { return error_message. LibraryContext::LibraryContext(const std::string& library_name, const ResourceConfig& config, std::unique_ptr mapper) - : runtime_(Legion::Runtime::get_runtime()), - library_name_(library_name), - mapper_(std::move(mapper)) + : runtime_(Legion::Runtime::get_runtime()), library_name_(library_name) { task_scope_ = ResourceIdScope( runtime_->generate_library_task_ids(library_name.c_str(), config.max_tasks), config.max_tasks); @@ -62,14 +60,9 @@ LibraryContext::LibraryContext(const std::string& library_name, shard_scope_ = ResourceIdScope( runtime_->generate_library_sharding_ids(library_name.c_str(), config.max_shardings), config.max_shardings); - - auto base_mapper = new mapping::BaseMapper(mapper_.get(), runtime_->get_mapper_runtime(), this); - Legion::Mapping::Mapper* legion_mapper = base_mapper; - if (Core::log_mapping_decisions) - legion_mapper = new Legion::Mapping::LoggingWrapper(base_mapper, &base_mapper->logger); - mapper_id_ = runtime_->generate_library_mapper_ids(library_name.c_str(), 1); - runtime_->add_mapper(mapper_id_, legion_mapper); + + register_mapper(std::move(mapper)); } const std::string& LibraryContext::get_library_name() const { return library_name_; } @@ -150,6 +143,23 @@ bool LibraryContext::valid_sharding_id(Legion::ShardingID shard_id) const return shard_scope_.in_scope(shard_id); } +const std::string& LibraryContext::get_task_name(int64_t local_task_id) const +{ + return find_task(local_task_id)->name(); +} + +void LibraryContext::register_mapper(std::unique_ptr mapper) +{ + // Hold the pointer to the mapper to keep it alive + mapper_ = std::move(mapper); + + auto base_mapper = new mapping::BaseMapper(mapper_.get(), runtime_->get_mapper_runtime(), this); + Legion::Mapping::Mapper* legion_mapper = base_mapper; + if (Core::log_mapping_decisions) + legion_mapper = new Legion::Mapping::LoggingWrapper(base_mapper, &base_mapper->logger); + runtime_->add_mapper(mapper_id_, legion_mapper); +} + void LibraryContext::register_task(int64_t local_task_id, std::unique_ptr task_info) { auto task_id = get_task_id(local_task_id); @@ -167,7 +177,11 @@ void LibraryContext::register_task(int64_t local_task_id, std::unique_ptrsecond.get(); + if (tasks_.end() == finder) { + throw std::out_of_range("Library " + get_library_name() + " does not have task " + + std::to_string(local_task_id)); + } + return finder->second.get(); } void LibraryContext::perform_callback(Legion::RegistrationWithArgsCallbackFnptr callback, diff --git a/src/core/runtime/context.h b/src/core/runtime/context.h index 88f3507a9d..4f5cf29d2f 100644 --- a/src/core/runtime/context.h +++ b/src/core/runtime/context.h @@ -109,7 +109,6 @@ class LibraryContext { int64_t get_new_task_id() { return task_scope_.generate_id(); } public: - void record_task_name(int64_t local_task_id, const std::string& task_name); /** * @brief Returns the name of a task * @@ -174,6 +173,12 @@ class LibraryContext { */ template int32_t register_reduction_operator(int32_t redop_id); + /** + * @brief Registers a library mapper. Replaces the existing mapper if there already is one. + * + * @param mapper A Legate mapper to register for the library + */ + void register_mapper(std::unique_ptr mapper); public: void register_task(int64_t local_task_id, std::unique_ptr task_info); diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index 8877ffd6c7..91a65a64a6 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -62,17 +62,6 @@ Runtime::Runtime() Runtime::~Runtime() {} -LibraryContext* Runtime::find_library(const std::string& library_name, - bool can_fail /*=false*/) const -{ - auto finder = libraries_.find(library_name); - if (libraries_.end() == finder) { - if (!can_fail) throw std::out_of_range("Library " + library_name + " does not exist"); - return nullptr; - } - return finder->second; -} - LibraryContext* Runtime::create_library(const std::string& library_name, const ResourceConfig& config, std::unique_ptr mapper) @@ -87,6 +76,32 @@ LibraryContext* Runtime::create_library(const std::string& library_name, return context; } +LibraryContext* Runtime::find_library(const std::string& library_name, + bool can_fail /*=false*/) const +{ + auto finder = libraries_.find(library_name); + if (libraries_.end() == finder) { + if (!can_fail) throw std::out_of_range("Library " + library_name + " does not exist"); + return nullptr; + } + return finder->second; +} + +LibraryContext* Runtime::find_or_create_library(const std::string& library_name, + const ResourceConfig& config, + std::unique_ptr mapper, + bool* created) +{ + LibraryContext* result = find_library(library_name, true /*can_fail*/); + if (result != nullptr) { + if (created != nullptr) *created = false; + return result; + } + result = create_library(library_name, config, std::move(mapper)); + if (created != nullptr) *created = true; + return result; +} + uint32_t Runtime::get_type_uid() { return next_type_uid_++; } void Runtime::record_reduction_operator(int32_t type_uid, int32_t op_kind, int32_t legion_op_id) @@ -131,7 +146,7 @@ void Runtime::initialize(Legion::Context legion_context) if (initialized_) throw std::runtime_error("Legate runtime has already been initialized"); initialized_ = true; legion_context_ = legion_context; - core_context_ = find_library(core_library_name); + core_context_ = find_library(core_library_name, false /*can_fail*/); communicator_manager_ = new CommunicatorManager(); partition_manager_ = new PartitionManager(this, core_context_); machine_manager_ = new MachineManager(); @@ -143,12 +158,7 @@ void Runtime::initialize(Legion::Context legion_context) mapping::MachineDesc Runtime::slice_machine_for_task(LibraryContext* library, int64_t task_id) { - auto task_info = library->find_task(task_id); - if (nullptr == task_info) { - std::stringstream ss; - ss << "Library " << library->get_library_name() << " does not have task " << task_id; - throw std::invalid_argument(std::move(ss).str()); - } + auto* task_info = library->find_task(task_id); std::vector task_targets; auto& machine = machine_manager_->get_machine(); diff --git a/src/core/runtime/detail/runtime.h b/src/core/runtime/detail/runtime.h index f429f2956c..0ad8e6caf7 100644 --- a/src/core/runtime/detail/runtime.h +++ b/src/core/runtime/detail/runtime.h @@ -53,10 +53,14 @@ class Runtime { ~Runtime(); public: - LibraryContext* find_library(const std::string& library_name, bool can_fail = false) const; LibraryContext* create_library(const std::string& library_name, - const ResourceConfig& config = ResourceConfig{}, - std::unique_ptr mapper = nullptr); + const ResourceConfig& config, + std::unique_ptr mapper); + LibraryContext* find_library(const std::string& library_name, bool can_fail) const; + LibraryContext* find_or_create_library(const std::string& library_name, + const ResourceConfig& config, + std::unique_ptr mapper, + bool* created); public: uint32_t get_type_uid(); diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 2921fdbb5e..9e9097939d 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -147,9 +147,12 @@ LibraryContext* Runtime::create_library(const std::string& library_name, return impl_->create_library(library_name, config, std::move(mapper)); } -void Runtime::record_reduction_operator(int32_t type_uid, int32_t op_kind, int32_t legion_op_id) +LibraryContext* Runtime::find_or_create_library(const std::string& library_name, + const ResourceConfig& config, + std::unique_ptr mapper, + bool* created) { - impl_->record_reduction_operator(type_uid, op_kind, legion_op_id); + return impl_->find_or_create_library(library_name, config, std::move(mapper), created); } // This function should be moved to the library context diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 0e17b26ba7..0bea67d5dd 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -103,6 +103,25 @@ class Runtime; class Runtime { public: + /** + * @brief Creates a library + * + * A library is a collection of tasks and custom reduction operators. The maximum number of + * tasks and reduction operators can be optionally specified with a `ResourceConfig` object. + * Each library can optionally have a mapper that specifies mapping policies for its tasks. + * When no mapper is given, the default mapper is used. + * + * @param library_name Library name. Must be unique to this library + * @param config Optional configuration object + * @param mapper Optional mapper object + * + * @return Context object for the library + * + * @throw std::invalid_argument If a library already exists for a given name + */ + LibraryContext* create_library(const std::string& library_name, + const ResourceConfig& config = ResourceConfig{}, + std::unique_ptr mapper = nullptr); /** * @brief Finds a library * @@ -116,25 +135,23 @@ class Runtime { */ LibraryContext* find_library(const std::string& library_name, bool can_fail = false) const; /** - * @brief Creates a library + * @brief Finds or creates a library. + * + * The optional configuration and mapper objects are picked up only when the library is created. * - * A library is a collection of tasks and custom reduction operators. The maximum number of - * tasks and reduction operators can be optionally specified with a `ResourceConfig` object. - * Each library can optionally have a mapper that specifies mapping policies for its tasks. - * When no mapper is given, the default mapper is used. * * @param library_name Library name. Must be unique to this library * @param config Optional configuration object * @param mapper Optional mapper object - * - * @throw std::invalid_argument If a library already exists for a given name + * @param created Optional pointer to a boolean flag indicating whether the library has been + * created because of this call * * @return Context object for the library */ - LibraryContext* create_library(const std::string& library_name, - const ResourceConfig& config = ResourceConfig{}, - std::unique_ptr mapper = nullptr); - void record_reduction_operator(int32_t type_uid, int32_t op_kind, int32_t legion_op_id); + LibraryContext* find_or_create_library(const std::string& library_name, + const ResourceConfig& config = ResourceConfig{}, + std::unique_ptr mapper = nullptr, + bool* created = nullptr); public: /** diff --git a/tests/cpp/unit/library.cc b/tests/cpp/unit/library.cc new file mode 100644 index 0000000000..791687a07c --- /dev/null +++ b/tests/cpp/unit/library.cc @@ -0,0 +1,54 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "legate.h" + +TEST(Library, Create) +{ + auto* runtime = legate::Runtime::get_runtime(); + auto* lib = runtime->create_library("libA"); + EXPECT_EQ(lib, runtime->find_library("libA")); +} + +TEST(Library, FindOrCreate) +{ + auto* runtime = legate::Runtime::get_runtime(); + + legate::ResourceConfig config; + config.max_tasks = 1; + + bool created = false; + auto* p_lib1 = runtime->find_or_create_library("libA", config, nullptr, &created); + EXPECT_TRUE(created); + + config.max_tasks = 2; + auto* p_lib2 = runtime->find_or_create_library("libA", config, nullptr, &created); + EXPECT_FALSE(created); + EXPECT_EQ(p_lib1, p_lib2); + EXPECT_TRUE(p_lib2->valid_task_id(p_lib2->get_task_id(0))); + EXPECT_FALSE(p_lib2->valid_task_id(p_lib2->get_task_id(1))); +} + +TEST(Library, FindNonExistent) +{ + auto* runtime = legate::Runtime::get_runtime(); + + EXPECT_EQ(runtime->find_library("libB", true /*can_fail*/), nullptr); + + EXPECT_THROW(runtime->find_library("libB", false /*can_fail*/), std::out_of_range); +} From bc31d08b2e82a167fd40fc6d22ed85fc4f441688 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Mon, 3 Jul 2023 13:58:09 -0700 Subject: [PATCH 0159/1425] Python-side API to extract raw pointers to C++ type objects * Python-side API to extract raw pointers to C++ type objects See merge request legate/legate.core.internal!74 --- legate/core/_lib/types.pyx | 4 ++++ legate/core/types.pyi | 2 ++ 2 files changed, 6 insertions(+) diff --git a/legate/core/_lib/types.pyx b/legate/core/_lib/types.pyx index 2e321afa04..e945d5d2c8 100644 --- a/legate/core/_lib/types.pyx +++ b/legate/core/_lib/types.pyx @@ -217,6 +217,10 @@ cdef class Dtype: def serialize(self, buf) -> None: buf.pack_32bit_int(self.code) + @property + def raw_ptr(self) -> long: + return (self._type.get()) + cdef class FixedArrayDtype(Dtype): def num_elements(self) -> int: diff --git a/legate/core/types.pyi b/legate/core/types.pyi index 0836bc8743..77743a671d 100644 --- a/legate/core/types.pyi +++ b/legate/core/types.pyi @@ -47,6 +47,8 @@ class Dtype: def __repr__(self) -> str: ... def to_numpy_dtype(self) -> np.dtype[Any]: ... def serialize(self, buf: BufferBuilder) -> None: ... + @property + def raw_ptr(self) -> int: ... class FixedArrayDtype(Dtype): def num_elements(self) -> int: ... From c1acf02133e46b463398e6fe65fdb60a22547ccb Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Mon, 3 Jul 2023 19:37:26 -0700 Subject: [PATCH 0160/1425] Check duplicate task IDs * Check duplicate task IDs See merge request legate/legate.core.internal!75 --- src/core/runtime/context.cc | 23 +++++-------- src/core/runtime/context.h | 13 -------- tests/cpp/unit/registration.cc | 61 ++++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 27 deletions(-) create mode 100644 tests/cpp/unit/registration.cc diff --git a/src/core/runtime/context.cc b/src/core/runtime/context.cc index 6d877a3b68..d83ad9d2b4 100644 --- a/src/core/runtime/context.cc +++ b/src/core/runtime/context.cc @@ -32,18 +32,6 @@ namespace legate { -InvalidTaskIdException::InvalidTaskIdException(const std::string& library_name, - int64_t offending_task_id, - int64_t max_task_id) -{ - std::stringstream ss; - ss << "Task id " << offending_task_id << " is invalid for library '" << library_name - << "' (max local task id: " << max_task_id << ")"; - error_message = std::move(ss).str(); -} - -const char* InvalidTaskIdException::what() const throw() { return error_message.c_str(); } - LibraryContext::LibraryContext(const std::string& library_name, const ResourceConfig& config, std::unique_ptr mapper) @@ -163,13 +151,20 @@ void LibraryContext::register_mapper(std::unique_ptr mapper) void LibraryContext::register_task(int64_t local_task_id, std::unique_ptr task_info) { auto task_id = get_task_id(local_task_id); - if (!task_scope_.in_scope(task_id)) - throw InvalidTaskIdException(library_name_, local_task_id, task_scope_.size() - 1); + if (!task_scope_.in_scope(task_id)) { + std::stringstream ss; + ss << "Task " << local_task_id << " is invalid for library '" << library_name_ + << "' (max local task id: " << (task_scope_.size() - 1) << ")"; + throw std::out_of_range(std::move(ss).str()); + } #ifdef DEBUG_LEGATE log_legate.debug() << "[" << library_name_ << "] task " << local_task_id << " (global id: " << task_id << "), " << *task_info; #endif + if (tasks_.find(local_task_id) != tasks_.end()) + throw std::invalid_argument("Task " + std::to_string(local_task_id) + + " already exists in library " + library_name_); task_info->register_task(task_id); tasks_.emplace(std::make_pair(local_task_id, std::move(task_info))); } diff --git a/src/core/runtime/context.h b/src/core/runtime/context.h index 4f5cf29d2f..7880c530c5 100644 --- a/src/core/runtime/context.h +++ b/src/core/runtime/context.h @@ -50,19 +50,6 @@ class Scalar; extern Legion::Logger log_legate; -class InvalidTaskIdException : public std::exception { - public: - InvalidTaskIdException(const std::string& library_name, - int64_t offending_task_id, - int64_t max_task_id); - - public: - virtual const char* what() const throw(); - - private: - std::string error_message; -}; - /** * @ingroup runtime * @brief A library context that provides APIs for registering components diff --git a/tests/cpp/unit/registration.cc b/tests/cpp/unit/registration.cc new file mode 100644 index 0000000000..18a651380b --- /dev/null +++ b/tests/cpp/unit/registration.cc @@ -0,0 +1,61 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "legate.h" + +namespace test_registration { + +template +struct CPUVariantTask : public legate::LegateTask> { + static const int32_t TASK_ID = ID; + static void cpu_variant(legate::TaskContext& context) {} +}; + +template +struct GPUVariantTask : public legate::LegateTask> { + static const int32_t TASK_ID = ID; + static void gpu_variant(legate::TaskContext& context) {} +}; + +} // namespace test_registration + +void test_duplicates() +{ + auto* runtime = legate::Runtime::get_runtime(); + auto* library = runtime->create_library("libA"); + test_registration::CPUVariantTask<0>::register_variants(library); + EXPECT_THROW(test_registration::CPUVariantTask<0>::register_variants(library), + std::invalid_argument); +} + +void test_out_of_bounds_task_id() +{ + legate::ResourceConfig config; + config.max_tasks = 1; + auto* runtime = legate::Runtime::get_runtime(); + auto* library = runtime->create_library("libA", config); + + EXPECT_THROW(test_registration::CPUVariantTask<1>::register_variants(library), std::out_of_range); +} + +TEST(Registration, Duplicate) { legate::Core::perform_registration(); } + +TEST(Registration, TaskIDOutOfBounds) +{ + legate::Core::perform_registration(); +} From 7bd13e10e2892f5dac8158d5ca9d0c87ebca56c1 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Tue, 4 Jul 2023 00:42:03 -0700 Subject: [PATCH 0161/1425] Fix for store unmap and clean up for store error checking * Type check for reduction accessors and use exceptions on errors * Clean up and extend inline mapping tests * Fix for removal of a non-existent value See merge request legate/legate.core.internal!77 --- src/core/data/store.cc | 43 +++++++++-------- src/core/data/store.h | 56 ++++++++++++++++++--- src/core/data/store.inl | 75 +++++++++++------------------ src/core/utilities/multi_set.inl | 2 +- tests/cpp/integration/inline_map.cc | 39 ++++++++++----- 5 files changed, 128 insertions(+), 87 deletions(-) diff --git a/src/core/data/store.cc b/src/core/data/store.cc index ba825cb231..2791c4cfb7 100644 --- a/src/core/data/store.cc +++ b/src/core/data/store.cc @@ -330,9 +330,8 @@ bool Store::valid() const { return is_future_ || is_unbound_store_ || region_fie Domain Store::domain() const { -#ifdef DEBUG_LEGATE - assert(!is_unbound_store_); -#endif + if (is_unbound_store_) + throw std::invalid_argument("Invalid to retrieve the domain of an unbound store"); auto result = is_future_ ? future_.domain() : region_field_.domain(); if (!transform_->identity()) result = transform_->transform(result); #ifdef DEBUG_LEGATE @@ -349,9 +348,7 @@ void Store::unmap() void Store::bind_empty_data() { -#ifdef DEBUG_LEGATE - check_valid_return(); -#endif + check_valid_binding(); unbound_field_.bind_empty_data(dim_); } @@ -363,33 +360,37 @@ void Store::remove_transform() dim_ = transform_->pop()->target_ndim(dim_); } -void Store::check_valid_return() const +void Store::check_accessor_dimension(const int32_t dim) const { - if (!is_unbound_store_) { - log_legate.error("Invalid to return a buffer to a bound store"); - LEGATE_ABORT; - } - if (unbound_field_.bound()) { - log_legate.error("Invalid to return more than one buffer to an unbound store"); - LEGATE_ABORT; + if (!(dim == dim_ || (dim_ == 0 && dim == 1))) { + throw std::invalid_argument("Dimension mismatch: invalid to create a " + std::to_string(dim) + + "-D accessor to a " + std::to_string(dim_) + "-D store"); } } void Store::check_buffer_dimension(const int32_t dim) const { if (dim != dim_) { - log_legate.error( - "Dimension mismatch: invalid to bind a %d-D buffer to a %d-D store", dim, dim_); - LEGATE_ABORT; + throw std::invalid_argument("Dimension mismatch: invalid to bind a " + std::to_string(dim) + + "-D buffer to a " + std::to_string(dim_) + "-D store"); } } -void Store::check_accessor_dimension(const int32_t dim) const +void Store::check_shape_dimension(const int32_t dim) const { if (!(dim == dim_ || (dim_ == 0 && dim == 1))) { - log_legate.error( - "Dimension mismatch: invalid to create a %d-D accessor to a %d-D store", dim, dim_); - LEGATE_ABORT; + throw std::invalid_argument("Dimension mismatch: invalid to retrieve a " + std::to_string(dim) + + "-D rect from a " + std::to_string(dim_) + "-D store"); + } +} + +void Store::check_valid_binding() const +{ + if (!is_unbound_store_) { + throw std::invalid_argument("Buffer can be bound only to an unbound store"); + } + if (unbound_field_.bound()) { + throw std::invalid_argument("A buffer has already been bound to the store"); } } diff --git a/src/core/data/store.h b/src/core/data/store.h index 1d075683bd..be45fd725e 100644 --- a/src/core/data/store.h +++ b/src/core/data/store.h @@ -352,8 +352,13 @@ class Store { public: /** - * @brief Returns a read-only accessor to the store for the entire domain. Validates type and - * dimension by default, disable with VALIDATE_TYPE = false. + * @brief Returns a read-only accessor to the store for the entire domain. + * + * @tparam T Element type + * + * @tparam DIM Number of dimensions + * + * @tparam VALIDATE_TYPE If `true` (default), validates type and number of dimensions * * @return A read-only accessor to the store */ @@ -363,6 +368,12 @@ class Store { * @brief Returns a write-only accessor to the store for the entire domain. Validates type and * dimension by default, disable with VALIDATE_TYPE = false. * + * @tparam T Element type + * + * @tparam DIM Number of dimensions + * + * @tparam VALIDATE_TYPE If `true` (default), validates type and number of dimensions + * * @return A write-only accessor to the store */ template @@ -371,6 +382,12 @@ class Store { * @brief Returns a read-write accessor to the store for the entire domain. Validates type and * dimension by default, disable with VALIDATE_TYPE = false. * + * @tparam T Element type + * + * @tparam DIM Number of dimensions + * + * @tparam VALIDATE_TYPE If `true` (default), validates type and number of dimensions + * * @return A read-write accessor to the store */ template @@ -384,9 +401,13 @@ class Store { * @tparam EXCLUSIVE Indicates whether reductions can be performed in exclusive mode. If * `EXCLUSIVE` is `false`, every reduction via the acecssor is performed atomically. * + * @tparam DIM Number of dimensions + * + * @tparam VALIDATE_TYPE If `true` (default), validates type and number of dimensions + * * @return A reduction accessor to the store */ - template + template AccessorRD reduce_accessor() const; public: @@ -394,6 +415,12 @@ class Store { * @brief Returns a read-only accessor to the store for specific bounds. Validates type and * dimension by default, disable with VALIDATE_TYPE = false. * + * @tparam T Element type + * + * @tparam DIM Number of dimensions + * + * @tparam VALIDATE_TYPE If `true` (default), validates type and number of dimensions + * * @param bounds Domain within which accesses should be allowed. * The actual bounds for valid access are determined by an intersection between * the store's domain and the bounds. @@ -406,6 +433,12 @@ class Store { * @brief Returns a write-only accessor to the store for the entire domain. Validates type and * dimension by default, disable with VALIDATE_TYPE = false. * + * @tparam T Element type + * + * @tparam DIM Number of dimensions + * + * @tparam VALIDATE_TYPE If `true` (default), validates type and number of dimensions + * * @param bounds Domain within which accesses should be allowed. * The actual bounds for valid access are determined by an intersection between * the store's domain and the bounds. @@ -418,6 +451,12 @@ class Store { * @brief Returns a read-write accessor to the store for the entire domain. Validates type and * dimension by default, disable with VALIDATE_TYPE = false. * + * @tparam T Element type + * + * @tparam DIM Number of dimensions + * + * @tparam VALIDATE_TYPE If `true` (default), validates type and number of dimensions + * * @param bounds Domain within which accesses should be allowed. * The actual bounds for valid access are determined by an intersection between * the store's domain and the bounds. @@ -439,9 +478,13 @@ class Store { * @tparam EXCLUSIVE Indicates whether reductions can be performed in exclusive mode. If * `EXCLUSIVE` is `false`, every reduction via the acecssor is performed atomically. * + * @tparam DIM Number of dimensions + * + * @tparam VALIDATE_TYPE If `true` (default), validates type and number of dimensions + * * @return A reduction accessor to the store */ - template + template AccessorRD reduce_accessor(const Rect& bounds) const; public: @@ -568,9 +611,10 @@ class Store { void remove_transform(); private: - void check_valid_return() const; - void check_buffer_dimension(const int32_t dim) const; void check_accessor_dimension(const int32_t dim) const; + void check_buffer_dimension(const int32_t dim) const; + void check_shape_dimension(const int32_t dim) const; + void check_valid_binding() const; template void check_accessor_type() const; diff --git a/src/core/data/store.inl b/src/core/data/store.inl index 4f5dce7943..50bc3b2753 100644 --- a/src/core/data/store.inl +++ b/src/core/data/store.inl @@ -281,22 +281,17 @@ void Store::check_accessor_type() const if (in_type == this->code()) return; // Test exact match for primitive types if (in_type != Type::Code::INVALID) { - log_legate.error( - "Type mismatch: %s accessor to a %s store. Disable type checking via " - "accessor template parameter if this is intended.", - PrimitiveType(in_type).to_string().c_str(), - this->type().to_string().c_str()); - LEGATE_ABORT; + throw std::invalid_argument( + "Type mismatch: " + PrimitiveType(in_type).to_string() + " accessor to a " + + type().to_string() + + " store. Disable type checking via accessor template parameter if this is intended."); } // Test size matches for other types - if (sizeof(T) != this->type().size()) { - log_legate.error( - "Type size mismatch: store type %s has size %d, requested type has size %d. Disable type " - "checking via accessor template parameter if this is intended.", - this->type().to_string().c_str(), - this->type().size(), - sizeof(T)); - LEGATE_ABORT; + if (sizeof(T) != type().size()) { + throw std::invalid_argument( + "Type size mismatch: store type " + type().to_string() + " has size " + + std::to_string(type().size()) + ", requested type has size " + std::to_string(sizeof(T)) + + ". Disable type checking via accessor template parameter if this is intended."); } } @@ -351,12 +346,14 @@ AccessorRW Store::read_write_accessor() const return region_field_.read_write_accessor(shape()); } -template +template AccessorRD Store::reduce_accessor() const { -#ifdef DEBUG_LEGATE - check_accessor_dimension(DIM); -#endif + using T = typename OP::LHS; + if constexpr (VALIDATE_TYPE) { + check_accessor_dimension(DIM); + check_accessor_type(); + } if (is_future_) return future_.reduce_accessor(redop_id_, shape()); @@ -392,10 +389,6 @@ AccessorWO Store::write_accessor(const Rect& bounds) const check_accessor_type(); } -#ifdef DEBUG_LEGATE - check_accessor_dimension(DIM); -#endif - if (is_future_) return future_.write_accessor(bounds); if (!transform_->identity()) { @@ -422,12 +415,14 @@ AccessorRW Store::read_write_accessor(const Rect& bounds) const return region_field_.read_write_accessor(bounds); } -template +template AccessorRD Store::reduce_accessor(const Rect& bounds) const { -#ifdef DEBUG_LEGATE - check_accessor_dimension(DIM); -#endif + using T = typename OP::LHS; + if constexpr (VALIDATE_TYPE) { + check_accessor_dimension(DIM); + check_accessor_type(); + } if (is_future_) return future_.reduce_accessor(redop_id_, bounds); @@ -441,28 +436,18 @@ AccessorRD Store::reduce_accessor(const Rect& bounds) c template Buffer Store::create_output_buffer(const Point& extents, bool bind_buffer /*= false*/) { -#ifdef DEBUG_LEGATE - check_valid_return(); + check_valid_binding(); check_buffer_dimension(DIM); -#endif return unbound_field_.create_output_buffer(extents, bind_buffer); } template Rect Store::shape() const { -#ifdef DEBUG_LEGATE - if (!(DIM == dim_ || (dim_ == 0 && DIM == 1))) { - log_legate.error( - "Dimension mismatch: invalid to retrieve a %d-D shape from a %d-D store", DIM, dim_); - LEGATE_ABORT; - } -#endif - - auto dom = domain(); - if (dom.dim > 0) - return dom.bounds(); - else { + check_shape_dimension(DIM); + if (dim_ > 0) { + return domain().bounds(); + } else { auto p = Point::ZEROES(); return Rect(p, p); } @@ -471,19 +456,15 @@ Rect Store::shape() const template VAL Store::scalar() const { -#ifdef DEBUG_LEGATE - assert(is_future_); -#endif + if (!is_future_) throw std::invalid_argument("Scalars can be retrieved only from scalar stores"); return future_.scalar(); } template void Store::bind_data(Buffer& buffer, const Point& extents) { -#ifdef DEBUG_LEGATE - check_valid_return(); + check_valid_binding(); check_buffer_dimension(DIM); -#endif unbound_field_.bind_data(buffer, extents); } diff --git a/src/core/utilities/multi_set.inl b/src/core/utilities/multi_set.inl index b6cb7a99e6..f14b0e21d0 100644 --- a/src/core/utilities/multi_set.inl +++ b/src/core/utilities/multi_set.inl @@ -35,7 +35,7 @@ template bool MultiSet::remove(const T& value) { auto finder = map_.find(value); - assert(finder != map_.end()); + if (map_.end() == finder) return false; finder->second--; if (finder->second == 0) { map_.erase(finder); diff --git a/tests/cpp/integration/inline_map.cc b/tests/cpp/integration/inline_map.cc index d8dcb527ad..900d16f8e8 100644 --- a/tests/cpp/integration/inline_map.cc +++ b/tests/cpp/integration/inline_map.cc @@ -45,8 +45,9 @@ void register_tasks() AdderTask::register_variants(context); } -void test_mapped_regions_leak(legate::Runtime* runtime, legate::LibraryContext* context) +void test_mapped_regions_leak() { + auto runtime = legate::Runtime::get_runtime(); { auto l_store = runtime->create_store({5}, legate::int64()); auto p_store = l_store.get_physical_store(); @@ -56,15 +57,17 @@ void test_mapped_regions_leak(legate::Runtime* runtime, legate::LibraryContext* EXPECT_EQ(runtime->impl()->num_inline_mapped(), 0); } -void test_inline_map_future(legate::Runtime* runtime, legate::LibraryContext* context) +void test_inline_map_future() { + auto runtime = legate::Runtime::get_runtime(); auto l_store = runtime->create_store({1}, legate::int64(), true /*optimize_scalar*/); auto p_store = l_store.get_physical_store(); EXPECT_TRUE(p_store->is_future()); } -void test_inline_map_region_and_slice(legate::Runtime* runtime, legate::LibraryContext* context) +void test_inline_map_region_and_slice() { + auto runtime = legate::Runtime::get_runtime(); auto root_ls = runtime->create_store({5}, legate::int64()); auto root_ps = root_ls.get_physical_store(); EXPECT_FALSE(root_ps->is_future()); @@ -77,8 +80,10 @@ void test_inline_map_region_and_slice(legate::Runtime* runtime, legate::LibraryC EXPECT_EQ(slice_acc[1], 42); } -void test_inline_map_and_task(legate::Runtime* runtime, legate::LibraryContext* context) +void test_inline_map_and_task() { + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); auto l_store = runtime->create_store({5}, legate::int64()); { auto p_store = l_store.get_physical_store(); @@ -94,16 +99,26 @@ void test_inline_map_and_task(legate::Runtime* runtime, legate::LibraryContext* EXPECT_EQ(acc[2], 43); } -TEST(Integration, InlineMap) +void test_inline_map_unmap() { - auto runtime = legate::Runtime::get_runtime(); - legate::Core::perform_registration(); - auto context = runtime->find_library(library_name); + auto runtime = legate::Runtime::get_runtime(); + auto logical_store = runtime->create_store({1, 3}, legate::int64()); + auto store = logical_store.get_physical_store(); + store->unmap(); +} + +TEST(InlineMap, MappedRegionsLeak) { test_mapped_regions_leak(); } + +TEST(InlineMap, Future) { test_inline_map_future(); } + +TEST(InlineMap, RegionAndSlice) { test_inline_map_region_and_slice(); } - test_mapped_regions_leak(runtime, context); - test_inline_map_future(runtime, context); - test_inline_map_region_and_slice(runtime, context); - test_inline_map_and_task(runtime, context); +TEST(InlineMap, WithTask) +{ + legate::Core::perform_registration(); + test_inline_map_and_task(); } +TEST(InlineMap, Unmap) { test_inline_map_unmap(); } + } // namespace inline_map From b13b7772080378f19736ed5c917ff34add42e51f Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Tue, 11 Jul 2023 17:19:30 -0700 Subject: [PATCH 0162/1425] Type checks on value types in copies * Type checks on value types in copies See merge request legate/legate.core.internal!80 --- src/core/operation/detail/copy.cc | 3 ++ src/core/operation/detail/gather.cc | 3 ++ src/core/operation/detail/scatter.cc | 3 ++ src/core/operation/detail/scatter_gather.cc | 3 ++ tests/cpp/integration/copy_failure.cc | 40 +++++++++++++++++++++ 5 files changed, 52 insertions(+) diff --git a/src/core/operation/detail/copy.cc b/src/core/operation/detail/copy.cc index f41e123887..794d31fa0a 100644 --- a/src/core/operation/detail/copy.cc +++ b/src/core/operation/detail/copy.cc @@ -39,6 +39,9 @@ Copy::Copy(std::shared_ptr target, void Copy::validate() { + if (source_.store->type() != target_.store->type()) { + throw std::invalid_argument("Source and targets must have the same type"); + } auto validate_store = [](auto* store) { if (store->unbound() || store->has_scalar_storage() || store->transformed()) { throw std::invalid_argument("Copy accepts only normal, untransformed, region-backed stores"); diff --git a/src/core/operation/detail/gather.cc b/src/core/operation/detail/gather.cc index 217f34847d..4c306f92e0 100644 --- a/src/core/operation/detail/gather.cc +++ b/src/core/operation/detail/gather.cc @@ -42,6 +42,9 @@ Gather::Gather(std::shared_ptr target, void Gather::validate() { + if (source_.store->type() != target_.store->type()) { + throw std::invalid_argument("Source and targets must have the same type"); + } auto validate_store = [](auto* store) { if (store->unbound() || store->has_scalar_storage() || store->transformed()) { throw std::invalid_argument( diff --git a/src/core/operation/detail/scatter.cc b/src/core/operation/detail/scatter.cc index 2bc6db7fc9..99aeb5491d 100644 --- a/src/core/operation/detail/scatter.cc +++ b/src/core/operation/detail/scatter.cc @@ -42,6 +42,9 @@ Scatter::Scatter(std::shared_ptr target, void Scatter::validate() { + if (source_.store->type() != target_.store->type()) { + throw std::invalid_argument("Source and targets must have the same type"); + } auto validate_store = [](auto* store) { if (store->unbound() || store->has_scalar_storage() || store->transformed()) { throw std::invalid_argument( diff --git a/src/core/operation/detail/scatter_gather.cc b/src/core/operation/detail/scatter_gather.cc index 26e1eeb4ed..2c3cfdcf1f 100644 --- a/src/core/operation/detail/scatter_gather.cc +++ b/src/core/operation/detail/scatter_gather.cc @@ -55,6 +55,9 @@ void ScatterGather::set_target_indirect_out_of_range(bool flag) void ScatterGather::validate() { + if (source_.store->type() != target_.store->type()) { + throw std::invalid_argument("Source and targets must have the same type"); + } auto validate_store = [](auto* store) { if (store->unbound() || store->has_scalar_storage() || store->transformed()) { throw std::invalid_argument( diff --git a/tests/cpp/integration/copy_failure.cc b/tests/cpp/integration/copy_failure.cc index 7f1b686652..f4d0913cba 100644 --- a/tests/cpp/integration/copy_failure.cc +++ b/tests/cpp/integration/copy_failure.cc @@ -49,6 +49,25 @@ void test_invalid_stores() std::invalid_argument); } +void test_type_check_failure() +{ + auto runtime = legate::Runtime::get_runtime(); + + auto source = runtime->create_store({10, 10}, legate::int64()); + auto target = runtime->create_store({10, 10}, legate::float64()); + auto source_indirect = runtime->create_store({10, 10}, legate::point_type(2)); + auto target_indirect = runtime->create_store({10, 10}, legate::point_type(2)); + + EXPECT_THROW(runtime->issue_copy(target, source), std::invalid_argument); + + EXPECT_THROW(runtime->issue_gather(target, source, source_indirect), std::invalid_argument); + + EXPECT_THROW(runtime->issue_scatter(target, target_indirect, source), std::invalid_argument); + + EXPECT_THROW(runtime->issue_scatter_gather(target, target_indirect, source, source_indirect), + std::invalid_argument); +} + void test_shape_check_failure() { auto runtime = legate::Runtime::get_runtime(); @@ -85,10 +104,31 @@ void test_non_point_types_failure() std::invalid_argument); } +void test_dimension_mismatch_failure() +{ + auto runtime = legate::Runtime::get_runtime(); + + auto source = runtime->create_store({10, 10}, legate::int64()); + auto target = runtime->create_store({10, 10}, legate::int64()); + auto source_indirect = runtime->create_store({10, 10}, legate::point_type(3)); + auto target_indirect = runtime->create_store({10, 10}, legate::point_type(3)); + + EXPECT_THROW(runtime->issue_gather(target, source, source_indirect), std::invalid_argument); + + EXPECT_THROW(runtime->issue_scatter(target, target_indirect, source), std::invalid_argument); + + EXPECT_THROW(runtime->issue_scatter_gather(target, target_indirect, source, source_indirect), + std::invalid_argument); +} + TEST(Copy, FailureInvalidStores) { test_invalid_stores(); } +TEST(Copy, FailureDifferentTypes) { test_type_check_failure(); } + TEST(Copy, FailureDifferentShapes) { test_shape_check_failure(); } TEST(Copy, FailureNonPointTypes) { test_non_point_types_failure(); } +TEST(Copy, FailureDimensionMismatch) { test_dimension_mismatch_failure(); } + } // namespace copy_failure From fd8094974247cc820ab2ba99a17e054b6f70c044 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Tue, 11 Jul 2023 23:30:13 -0700 Subject: [PATCH 0163/1425] Basic support for image partitioning * Validity checks for image constraints * Perform inclusion checks in the image constraint tests * Support for partition by image range * Remove the obsolete comment * Make the Image class concrete by implementing a missing function * Basic machinery for image partitioning. Currently doesn't work See merge request legate/legate.core.internal!81 --- src/core/operation/detail/fill.cc | 2 +- src/core/operation/detail/operation.cc | 4 +- src/core/operation/detail/operation.h | 4 +- src/core/partitioning/constraint.cc | 52 +++- src/core/partitioning/constraint.h | 62 ++++- src/core/partitioning/constraint_solver.cc | 49 +++- src/core/partitioning/constraint_solver.h | 7 + src/core/partitioning/partition.cc | 80 ++++++ src/core/partitioning/partition.h | 45 ++++ src/core/partitioning/partitioner.cc | 11 +- src/core/runtime/detail/runtime.cc | 30 +++ src/core/runtime/detail/runtime.h | 6 + src/core/type/type_info.cc | 25 +- src/core/type/type_info.h | 41 +++ tests/cpp/integration/image_constraints.cc | 299 +++++++++++++++++++++ 15 files changed, 700 insertions(+), 17 deletions(-) create mode 100644 tests/cpp/integration/image_constraints.cc diff --git a/src/core/operation/detail/fill.cc b/src/core/operation/detail/fill.cc index 06b1c74cd4..43cae2156b 100644 --- a/src/core/operation/detail/fill.cc +++ b/src/core/operation/detail/fill.cc @@ -33,7 +33,7 @@ Fill::Fill(std::shared_ptr&& lhs, lhs_(std::move(lhs)), value_(std::move(value)) { - store_mappings_[*lhs_var_] = lhs_.get(); + store_mappings_[*lhs_var_] = lhs_; if (lhs_->unbound() || lhs_->has_scalar_storage()) throw std::runtime_error("Fill lhs must be a normal, region-backed store"); diff --git a/src/core/operation/detail/operation.cc b/src/core/operation/detail/operation.cc index 5b65ccddbe..b68432d4f8 100644 --- a/src/core/operation/detail/operation.cc +++ b/src/core/operation/detail/operation.cc @@ -44,7 +44,7 @@ const Variable* Operation::declare_partition() return partition_symbols_.back().get(); } -LogicalStore* Operation::find_store(const Variable* part_symb) const +std::shared_ptr Operation::find_store(const Variable* part_symb) const { return store_mappings_.at(*part_symb); } @@ -59,7 +59,7 @@ void Operation::record_partition(const Variable* variable, std::shared_ptr store); const Variable* declare_partition(); - LogicalStore* find_store(const Variable* variable) const; + std::shared_ptr find_store(const Variable* variable) const; public: const mapping::MachineDesc& machine() const { return machine_; } @@ -77,7 +77,7 @@ class Operation { protected: uint32_t next_part_id_{0}; std::vector> partition_symbols_{}; - std::map store_mappings_{}; + std::map> store_mappings_{}; std::map, const Variable*> part_mappings_{}; std::string provenance_; mapping::MachineDesc machine_; diff --git a/src/core/partitioning/constraint.cc b/src/core/partitioning/constraint.cc index 654c22f0c9..fc7fd08deb 100644 --- a/src/core/partitioning/constraint.cc +++ b/src/core/partitioning/constraint.cc @@ -20,6 +20,7 @@ #include "core/operation/detail/operation.h" #include "core/partitioning/constraint.h" #include "core/partitioning/partition.h" +#include "core/partitioning/partitioner.h" namespace legate { @@ -63,8 +64,8 @@ void Alignment::find_partition_symbols(std::vector& partition_s void Alignment::validate() const { - auto* lhs_store = lhs_->operation()->find_store(lhs_.get()); - auto* rhs_store = rhs_->operation()->find_store(rhs_.get()); + auto lhs_store = lhs_->operation()->find_store(lhs_.get()); + auto rhs_store = rhs_->operation()->find_store(rhs_.get()); if (lhs_store->extents() != rhs_store->extents()) throw std::invalid_argument("Alignment requires the stores to have the same shape, but found " + lhs_store->extents().to_string() + " and " + @@ -90,7 +91,7 @@ void Broadcast::find_partition_symbols(std::vector& partition_s void Broadcast::validate() const { - auto* store = variable_->operation()->find_store(variable_.get()); + auto store = variable_->operation()->find_store(variable_.get()); for (auto axis : axes_.data()) { if (axis < 0 || axis >= store->dim()) throw std::invalid_argument("Invalid broadcasting dimension " + std::to_string(axis) + @@ -105,6 +106,45 @@ std::string Broadcast::to_string() const return std::move(ss).str(); } +ImageConstraint::ImageConstraint(std::unique_ptr var_function, + std::unique_ptr var_range) + : var_function_(std::move(var_function)), var_range_(std::move(var_range)) +{ +} + +void ImageConstraint::find_partition_symbols(std::vector& partition_symbols) const +{ + partition_symbols.push_back(var_function_.get()); + partition_symbols.push_back(var_range_.get()); +} + +void ImageConstraint::validate() const +{ + auto func = var_function_->operation()->find_store(var_function_.get()); + auto range = var_range_->operation()->find_store(var_range_.get()); + + if (!(is_point_type(func->type(), range->dim()) || is_rect_type(func->type(), range->dim()))) + throw std::invalid_argument("Store from which the image partition is derived should have " + + std::to_string(range->dim()) + "-D points or rects"); +} + +std::string ImageConstraint::to_string() const +{ + std::stringstream ss; + ss << "ImageConstraint(" << var_function_->to_string() << ", " << var_range_->to_string() << ")"; + return std::move(ss).str(); +} + +std::unique_ptr ImageConstraint::resolve(const detail::Strategy& strategy) const +{ + const auto* src = var_function(); + auto src_part = strategy[src]; + if (src_part->has_launch_domain()) + return create_image(src->operation()->find_store(src), src_part); + else + return create_no_partition(); +} + std::unique_ptr align(const Variable* lhs, const Variable* rhs) { if (*lhs == *rhs) throw std::invalid_argument("Alignment needs two distinct variables"); @@ -123,4 +163,10 @@ std::unique_ptr broadcast(const Variable* variable, tuple&& return std::make_unique(std::make_unique(*variable), std::move(axes)); } +std::unique_ptr image(const Variable* var_function, const Variable* var_range) +{ + return std::make_unique(std::make_unique(*var_function), + std::make_unique(*var_range)); +} + } // namespace legate diff --git a/src/core/partitioning/constraint.h b/src/core/partitioning/constraint.h index 140da91f7c..3d682a922d 100644 --- a/src/core/partitioning/constraint.h +++ b/src/core/partitioning/constraint.h @@ -32,6 +32,7 @@ namespace legate::detail { class Operation; +class Strategy; } // namespace legate::detail namespace legate { @@ -39,6 +40,7 @@ namespace legate { class Alignment; class Broadcast; class Constraint; +class ImageConstraint; class Literal; class Partition; class Variable; @@ -156,6 +158,7 @@ struct Constraint { enum class Kind : int32_t { ALIGNMENT = 0, BROADCAST = 1, + IMAGE = 2, }; virtual ~Constraint() {} @@ -178,8 +181,9 @@ struct Constraint { virtual void validate() const = 0; - virtual const Alignment* as_alignment() const = 0; - virtual const Broadcast* as_broadcast() const = 0; + virtual const Alignment* as_alignment() const = 0; + virtual const Broadcast* as_broadcast() const = 0; + virtual const ImageConstraint* as_image_constraint() const = 0; }; /** @@ -209,6 +213,7 @@ class Alignment : public Constraint { public: const Alignment* as_alignment() const override { return this; } const Broadcast* as_broadcast() const override { return nullptr; } + const ImageConstraint* as_image_constraint() const override { return nullptr; } public: /** @@ -257,6 +262,7 @@ class Broadcast : public Constraint { public: const Alignment* as_alignment() const override { return nullptr; } const Broadcast* as_broadcast() const override { return this; } + const ImageConstraint* as_image_constraint() const override { return nullptr; } public: /** @@ -277,6 +283,47 @@ class Broadcast : public Constraint { tuple axes_; }; +/** + * @ingroup partitioning + * @brief A class for image constraints + * + * This constraint tells the partitioner that `var_range_` should be derived by collecting the image + * of a "function", a store that contains either points or rects. `var_function_` is the partition + * of the function that the partitioner should use in the image partitioning. + */ +class ImageConstraint : public Constraint { + public: + ImageConstraint(std::unique_ptr var_function, std::unique_ptr var_range); + + public: + Kind kind() const override { return Kind::IMAGE; } + + public: + void find_partition_symbols(std::vector& partition_symbols) const override; + + public: + void validate() const; + + public: + std::string to_string() const override; + + public: + const Alignment* as_alignment() const override { return nullptr; } + const Broadcast* as_broadcast() const override { return nullptr; } + const ImageConstraint* as_image_constraint() const override { return this; } + + public: + const Variable* var_function() const { return var_function_.get(); } + const Variable* var_range() const { return var_range_.get(); } + + public: + std::unique_ptr resolve(const detail::Strategy& strategy) const; + + private: + std::unique_ptr var_function_; + std::unique_ptr var_range_; +}; + /** * @ingroup partitioning * @brief Creates an alignment constraint on two variables @@ -310,4 +357,15 @@ std::unique_ptr broadcast(const Variable* variable, const tuple broadcast(const Variable* variable, tuple&& axes); +/** + * @ingroup partitioning + * @brief Creates an image constraint between partitions. + * + * @param var_function Partition symbol for the function store + * @param var_range Partition symbol of the store whose partition should be derived from the image + * + * @return Broadcast constraint + */ +std::unique_ptr image(const Variable* var_function, const Variable* var_range); + } // namespace legate diff --git a/src/core/partitioning/constraint_solver.cc b/src/core/partitioning/constraint_solver.cc index f7641e8464..b26b37b434 100644 --- a/src/core/partitioning/constraint_solver.cc +++ b/src/core/partitioning/constraint_solver.cc @@ -22,6 +22,7 @@ #include "core/operation/detail/operation.h" #include "core/partitioning/constraint.h" #include "core/partitioning/constraint_solver.h" +#include "core/partitioning/partitioner.h" namespace legate { extern Legion::Logger log_legate; @@ -87,6 +88,7 @@ void ConstraintSolver::add_partition_symbol(const Variable* partition_symbol, bo { partition_symbols_.insert(partition_symbol); is_output_.insert({*partition_symbol, is_output}); + is_dependent_.insert({*partition_symbol, false}); } void ConstraintSolver::add_constraint(const Constraint* constraint) @@ -107,8 +109,8 @@ void ConstraintSolver::solve_constraints() for (auto& symb : all_symbols) { // TODO: partition symbols can be independent of any stores of the operation // (e.g., when a symbol subsumes a union of two other symbols) - auto* store = symb->operation()->find_store(symb); - entries.emplace_back(symb, store); + auto store = symb->operation()->find_store(symb); + entries.emplace_back(symb, store.get()); table.insert({*symb, &entries.back()}); } }; @@ -157,8 +159,14 @@ void ConstraintSolver::solve_constraints() } }; + // Here we only mark dependent partition symbols + auto handle_image_constraint = [&](const ImageConstraint* image_constraint) { + is_dependent_[*image_constraint->var_range()] = true; + }; + // Reflect each constraint to the solver state - for (auto& constraint : constraints_) switch (constraint->kind()) { + for (auto& constraint : constraints_) { + switch (constraint->kind()) { case Constraint::Kind::ALIGNMENT: { handle_alignment(constraint->as_alignment()); break; @@ -167,7 +175,12 @@ void ConstraintSolver::solve_constraints() handle_broadcast(constraint->as_broadcast()); break; } + case Constraint::Kind::IMAGE: { + handle_image_constraint(constraint->as_image_constraint()); + break; + } } + } // Combine states of each union of equivalence classes into one std::unordered_set distinct_entries; @@ -180,6 +193,26 @@ void ConstraintSolver::solve_constraints() } } +void ConstraintSolver::solve_dependent_constraints(Strategy& strategy) +{ + auto solve_image_constraint = [&strategy](const ImageConstraint* image_constraint) { + auto image = image_constraint->resolve(strategy); + strategy.insert(image_constraint->var_range(), std::move(image)); + }; + + for (auto& constraint : constraints_) { + switch (constraint->kind()) { + case Constraint::Kind::IMAGE: { + solve_image_constraint(constraint->as_image_constraint()); + break; + } + default: { + continue; + } + } + } +} + const std::vector& ConstraintSolver::find_equivalence_class( const Variable* partition_symbol) const { @@ -196,9 +229,19 @@ bool ConstraintSolver::is_output(const Variable& partition_symbol) const return is_output_.at(partition_symbol); } +bool ConstraintSolver::is_dependent(const Variable& partition_symbol) const +{ + return is_dependent_.at(partition_symbol); +} + void ConstraintSolver::dump() { log_legate.debug("===== Constraint Graph ====="); + log_legate.debug() << "Stores:"; + for (auto& symbol : partition_symbols_.elements()) { + auto store = symbol->operation()->find_store(symbol); + log_legate.debug() << " " << symbol->to_string() << " ~> " << store->to_string(); + } log_legate.debug() << "Variables:"; for (auto& symbol : partition_symbols_.elements()) log_legate.debug() << " " << symbol->to_string(); diff --git a/src/core/partitioning/constraint_solver.h b/src/core/partitioning/constraint_solver.h index 195410cd35..95175bb313 100644 --- a/src/core/partitioning/constraint_solver.h +++ b/src/core/partitioning/constraint_solver.h @@ -26,6 +26,8 @@ namespace legate::detail { +class Strategy; + struct ConstraintSolver { public: ConstraintSolver(); @@ -43,10 +45,12 @@ struct ConstraintSolver { public: void solve_constraints(); + void solve_dependent_constraints(Strategy& strategy); const std::vector& find_equivalence_class( const Variable* partition_symbol) const; const Restrictions& find_restrictions(const Variable* partition_symbol) const; bool is_output(const Variable& partition_symbol) const; + bool is_dependent(const Variable& partition_symbol) const; private: ordered_set partition_symbols_{}; @@ -57,6 +61,9 @@ struct ConstraintSolver { class EquivClass; std::map equiv_class_map_{}; std::vector equiv_classes_{}; + + private: + std::map is_dependent_{}; }; } // namespace legate::detail diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index 77e9e8b7bb..29feabefca 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -257,6 +257,80 @@ std::string Weighted::to_string() const return std::move(ss).str(); } +Image::Image(std::shared_ptr func, std::shared_ptr func_partition) + : func_(std::move(func)), func_partition_(std::move(func_partition)) +{ +} + +bool Image::operator==(const Image& other) const +{ + // FIXME: This needs to be implemented to cache image partitions + return false; +} + +bool Image::operator<(const Image& other) const +{ + // FIXME: This needs to be implemented to cache image partitions + return false; +} + +bool Image::is_complete_for(const detail::Storage* storage) const +{ + // Completeness check for image partitions is expensive, so we give a sound answer + return false; +} + +bool Image::is_disjoint_for(const Domain* launch_domain) const +{ + // Disjointness check for image partitions is expensive, so we give a sound answer; + return false; +} + +bool Image::satisfies_restrictions(const Restrictions& restrictions) const +{ + static auto satisfies_restriction = [](Restriction r, size_t ext) { + return r != Restriction::FORBID || ext == 1; + }; + return apply(satisfies_restriction, restrictions, color_shape()).all(); +} + +Legion::LogicalPartition Image::construct(Legion::LogicalRegion region, bool complete) const +{ + auto* func_rf = func_->get_region_field(); + auto func_region = func_rf->region(); + auto func_partition = + func_partition_->construct(func_region, func_partition_->is_complete_for(func_->get_storage())); + + auto runtime = detail::Runtime::get_runtime(); + bool is_range = func_->type().code == Type::Code::STRUCT; + auto color_space = runtime->find_or_create_index_space(to_domain(color_shape())); + + auto index_partition = runtime->create_image_partition(region.get_index_space(), + color_space, + func_region, + func_partition, + func_rf->field_id(), + is_range); + + return runtime->create_logical_partition(region, index_partition); +} + +bool Image::has_launch_domain() const { return true; } + +Domain Image::launch_domain() const { return to_domain(color_shape()); } + +std::unique_ptr Image::clone() const { return std::make_unique(*this); } + +std::string Image::to_string() const +{ + std::stringstream ss; + ss << "Image(func: " << func_->to_string() << ", partition: " << func_partition_->to_string() + << ")"; + return std::move(ss).str(); +} + +const Shape& Image::color_shape() const { return func_partition_->color_shape(); } + std::unique_ptr create_no_partition() { return std::make_unique(); } std::unique_ptr create_tiling(Shape&& tile_shape, @@ -273,6 +347,12 @@ std::unique_ptr create_weighted(const Legion::FutureMap& weights, return std::make_unique(weights, color_domain); } +std::unique_ptr create_image(std::shared_ptr func, + std::shared_ptr func_partition) +{ + return std::make_unique(std::move(func), std::move(func_partition)); +} + std::ostream& operator<<(std::ostream& out, const Partition& partition) { out << partition.to_string(); diff --git a/src/core/partitioning/partition.h b/src/core/partitioning/partition.h index 939232ce86..fed9ed4712 100644 --- a/src/core/partitioning/partition.h +++ b/src/core/partitioning/partition.h @@ -37,6 +37,7 @@ struct Partition { NO_PARTITION = 0, TILING = 1, WEIGHTED = 2, + IMAGE = 3, }; public: @@ -194,6 +195,47 @@ class Weighted : public Partition { Shape color_shape_; }; +class Image : public Partition { + public: + Image(std::shared_ptr func, std::shared_ptr func_partition); + + public: + Image(const Image&) = default; + + public: + bool operator==(const Image& other) const; + bool operator<(const Image& other) const; + + public: + Kind kind() const override { return Kind::IMAGE; } + + public: + bool is_complete_for(const detail::Storage* storage) const override; + bool is_disjoint_for(const Domain* launch_domain) const override; + bool satisfies_restrictions(const Restrictions& restrictions) const override; + bool is_convertible() const override { return false; } + + public: + Legion::LogicalPartition construct(Legion::LogicalRegion region, bool complete) const override; + + public: + bool has_launch_domain() const override; + Domain launch_domain() const override; + + public: + std::unique_ptr clone() const override; + + public: + std::string to_string() const override; + + public: + const Shape& color_shape() const override; + + private: + std::shared_ptr func_; + std::shared_ptr func_partition_; +}; + std::unique_ptr create_no_partition(); std::unique_ptr create_tiling(Shape&& tile_shape, @@ -203,6 +245,9 @@ std::unique_ptr create_tiling(Shape&& tile_shape, std::unique_ptr create_weighted(const Legion::FutureMap& weights, const Domain& color_domain); +std::unique_ptr create_image(std::shared_ptr func, + std::shared_ptr func_partition); + std::ostream& operator<<(std::ostream& out, const Partition& partition); } // namespace legate diff --git a/src/core/partitioning/partitioner.cc b/src/core/partitioning/partitioner.cc index 597f378bfb..0c9940c6b2 100644 --- a/src/core/partitioning/partitioner.cc +++ b/src/core/partitioning/partitioner.cc @@ -238,8 +238,8 @@ std::unique_ptr Partitioner::partition_stores() auto remaining_symbols = handle_unbound_stores(strategy.get(), partition_symbols, solver); auto comparison_key = [&solver](const auto& part_symb) { - auto* op = part_symb->operation(); - auto* store = op->find_store(part_symb); + auto* op = part_symb->operation(); + auto store = op->find_store(part_symb); auto has_key_part = store->has_key_partition(op->machine(), solver.find_restrictions(part_symb)); #ifdef DEBUG_LEGATE @@ -255,7 +255,10 @@ std::unique_ptr Partitioner::partition_stores() }); for (auto& part_symb : remaining_symbols) { - if (strategy->has_assignment(part_symb)) continue; + if (strategy->has_assignment(part_symb)) + continue; + else if (solver.is_dependent(*part_symb)) + continue; const auto& equiv_class = solver.find_equivalence_class(part_symb); const auto& restrictions = solver.find_restrictions(part_symb); @@ -271,6 +274,8 @@ std::unique_ptr Partitioner::partition_stores() for (auto symb : equiv_class) strategy->insert(symb, partition); } + solver.solve_dependent_constraints(*strategy); + strategy->compute_launch_domains(solver); #ifdef DEBUG_LEGATE diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index c05544e362..ae0799994a 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -468,6 +468,36 @@ Legion::IndexPartition Runtime::create_weighted_partition(const Legion::IndexSpa legion_context_, index_space, weights, color_space); } +Legion::IndexPartition Runtime::create_image_partition( + const Legion::IndexSpace& index_space, + const Legion::IndexSpace& color_space, + const Legion::LogicalRegion& func_region, + const Legion::LogicalPartition& func_partition, + Legion::FieldID func_field_id, + bool is_range) +{ + if (is_range) + return legion_runtime_->create_partition_by_image_range(legion_context_, + index_space, + func_partition, + func_region, + func_field_id, + color_space, + LEGION_COMPUTE_KIND, + LEGION_AUTO_GENERATE_ID, + core_context_->get_mapper_id()); + else + return legion_runtime_->create_partition_by_image(legion_context_, + index_space, + func_partition, + func_region, + func_field_id, + color_space, + LEGION_COMPUTE_KIND, + LEGION_AUTO_GENERATE_ID, + core_context_->get_mapper_id()); +} + Legion::FieldSpace Runtime::create_field_space() { assert(nullptr != legion_context_); diff --git a/src/core/runtime/detail/runtime.h b/src/core/runtime/detail/runtime.h index 0ad8e6caf7..f4371a3a5c 100644 --- a/src/core/runtime/detail/runtime.h +++ b/src/core/runtime/detail/runtime.h @@ -137,6 +137,12 @@ class Runtime { Legion::IndexPartition create_weighted_partition(const Legion::IndexSpace& index_space, const Legion::IndexSpace& color_space, const Legion::FutureMap& weights); + Legion::IndexPartition create_image_partition(const Legion::IndexSpace& index_space, + const Legion::IndexSpace& color_space, + const Legion::LogicalRegion& func_region, + const Legion::LogicalPartition& func_partition, + Legion::FieldID func_field_id, + bool is_range); Legion::FieldSpace create_field_space(); Legion::LogicalRegion create_region(const Legion::IndexSpace& index_space, const Legion::FieldSpace& field_space); diff --git a/src/core/type/type_info.cc b/src/core/type/type_info.cc index 739b1385b4..e815324ec2 100644 --- a/src/core/type/type_info.cc +++ b/src/core/type/type_info.cc @@ -67,6 +67,18 @@ const char* _VARIABLE_SIZE_ERROR_MESSAGE = "Variable-size element type cannot be Type::Type(Code c) : code(c) {} +const FixedArrayType& Type::as_fixed_array_type() const +{ + throw std::invalid_argument("Type is not a fixed array type"); + return *static_cast(nullptr); +} + +const StructType& Type::as_struct_type() const +{ + throw std::invalid_argument("Type is not a struct type"); + return *static_cast(nullptr); +} + void Type::record_reduction_operator(int32_t op_kind, int32_t global_op_id) const { detail::Runtime::get_runtime()->record_reduction_operator(uid(), op_kind, global_op_id); @@ -132,6 +144,8 @@ void FixedArrayType::pack(BufferBuilder& buffer) const element_type_->pack(buffer); } +const FixedArrayType& FixedArrayType::as_fixed_array_type() const { return *this; } + bool FixedArrayType::equal(const Type& other) const { if (code != other.code) return false; @@ -208,6 +222,8 @@ void StructType::pack(BufferBuilder& buffer) const buffer.pack(aligned_); } +const StructType& StructType::as_struct_type() const { return *this; } + bool StructType::equal(const Type& other) const { if (code != other.code) return false; @@ -326,7 +342,6 @@ constexpr int32_t RECT_UID_BASE = POINT_UID_BASE + LEGATE_MAX_DIM + 1; std::unique_ptr point_type(int32_t ndim) { - if (ndim == 1) return int64(); return std::make_unique(POINT_UID_BASE + ndim, int64(), ndim); } @@ -354,4 +369,12 @@ bool is_point_type(const Type& type, int32_t ndim) return false; } +bool is_rect_type(const Type& type, int32_t ndim) +{ + if (type.code != Type::Code::STRUCT) return false; + auto& st_type = static_cast(type); + return st_type.num_fields() == 2 && is_point_type(st_type.field_type(0), ndim) && + is_point_type(st_type.field_type(1), ndim); +} + } // namespace legate diff --git a/src/core/type/type_info.h b/src/core/type/type_info.h index a8d2e7490d..abb6baea46 100644 --- a/src/core/type/type_info.h +++ b/src/core/type/type_info.h @@ -27,6 +27,8 @@ namespace legate { class BufferBuilder; +class FixedArrayType; +class StructType; /** * @ingroup types @@ -140,6 +142,24 @@ class Type { */ virtual void pack(BufferBuilder& buffer) const = 0; + /** + * @brief Dynamically casts the type into a fixed size array type. + * + * If the type is not a fixed size array type, an exception will be raised. + * + * @return Type object + */ + virtual const FixedArrayType& as_fixed_array_type() const; + + /** + * @brief Dynamically casts the type into a struct type. + * + * If the type is not a struct type, an exception will be raised. + * + * @return Type object + */ + virtual const StructType& as_struct_type() const; + /** * @brief Records a reduction operator. * @@ -272,6 +292,7 @@ class FixedArrayType : public ExtensionType { std::unique_ptr clone() const override; std::string to_string() const override; void pack(BufferBuilder& buffer) const override; + const FixedArrayType& as_fixed_array_type() const override; /** * @brief Returns the number of elements @@ -318,6 +339,7 @@ class StructType : public ExtensionType { std::unique_ptr clone() const override; std::string to_string() const override; void pack(BufferBuilder& buffer) const override; + const StructType& as_struct_type() const override; /** * @brief Returns the number of fields @@ -543,6 +565,8 @@ std::unique_ptr string(); * @ingroup types * @brief Creates a point type * + * @param ndim Number of dimensions + * * @return Type object */ std::unique_ptr point_type(int32_t ndim); @@ -551,6 +575,8 @@ std::unique_ptr point_type(int32_t ndim); * @ingroup types * @brief Creates a rect type * + * @param ndim Number of dimensions + * * @return Type object */ std::unique_ptr rect_type(int32_t ndim); @@ -559,9 +585,24 @@ std::unique_ptr rect_type(int32_t ndim); * @ingroup types * @brief Checks if the type is a point type of the given dimensionality * + * @param type Type to check + * @param ndim Number of dimensions the point type should have + * * @return true If the `type` is a point type * @return false Otherwise */ bool is_point_type(const Type& type, int32_t ndim); +/** + * @ingroup types + * @brief Checks if the type is a rect type of the given dimensionality + * + * @param type Type to check + * @param ndim Number of dimensions the rect type should have + * + * @return true If the `type` is a rect type + * @return false Otherwise + */ +bool is_rect_type(const Type& type, int32_t ndim); + } // namespace legate diff --git a/tests/cpp/integration/image_constraints.cc b/tests/cpp/integration/image_constraints.cc new file mode 100644 index 0000000000..f1921a1c7d --- /dev/null +++ b/tests/cpp/integration/image_constraints.cc @@ -0,0 +1,299 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "core/data/detail/logical_store.h" +#include "legate.h" + +namespace image_constraints { + +static const char* library_name = "test_image_constraints"; + +static const int32_t TEST_MAX_DIM = 3; + +static legate::Logger logger(library_name); + +enum TaskIDs { + INIT_FUNC = 0, + IMAGE_TESTER = INIT_FUNC + TEST_MAX_DIM * 2, +}; + +template +legate::Point delinearize(size_t index, const legate::Point& extents) +{ + legate::Point result; + for (int32_t dim = 0; dim < DIM; ++dim) { + result[dim] = index % extents[dim]; + index = index / extents[dim]; + } + return result; +} + +template +struct InitializeFunction : public legate::LegateTask> { + struct InitializeRects { + template + void operator()(legate::Store& output, const legate::Scalar& extents_scalar) + { + auto shape = output.shape(); + auto extents = extents_scalar.value>(); + auto acc = output.write_accessor, DIM>(); + + size_t vol = shape.volume(); + size_t tgt_vol = 1; + for (int32_t dim = 0; dim < TGT_DIM; ++dim) tgt_vol *= extents[dim]; + + auto in_bounds = [&](const auto& point) { + for (int32_t dim = 0; dim < TGT_DIM; ++dim) + if (point[dim] >= extents[dim]) return false; + return true; + }; + size_t idx = 0; + for (legate::PointInRectIterator it(shape); it.valid(); ++it) { + auto lo = delinearize(idx++ * tgt_vol / vol, extents); + auto hi = lo + legate::Point::ONES(); + if (in_bounds(hi)) + acc[*it] = legate::Rect(lo, hi); + else + acc[*it] = legate::Rect(lo, lo); + } + } + }; + + struct InitializePoints { + template + void operator()(legate::Store& output, const legate::Scalar& extents_scalar) + { + auto shape = output.shape(); + auto extents = extents_scalar.value>(); + auto acc = output.write_accessor, DIM>(); + + size_t vol = shape.volume(); + size_t tgt_vol = 1; + for (int32_t dim = 0; dim < TGT_DIM; ++dim) tgt_vol *= extents[dim]; + + size_t idx = 0; + for (legate::PointInRectIterator it(shape); it.valid(); ++it) { + auto p = delinearize(idx++ * tgt_vol / vol, extents); + acc[*it] = p; + } + } + }; + + static const int32_t TASK_ID = INIT_FUNC + static_cast(RECT) * TEST_MAX_DIM + DIM; + static void cpu_variant(legate::TaskContext& context) + { + auto& output = context.outputs().at(0); + auto& extents = context.scalars().at(0); + if constexpr (RECT) { + const auto& rect_type = output.type().as_struct_type(); + const auto& point_type = rect_type.field_type(0).as_fixed_array_type(); + legate::dim_dispatch(point_type.num_elements(), InitializeRects{}, output, extents); + } else { + const auto& point_type = output.type().as_fixed_array_type(); + legate::dim_dispatch(point_type.num_elements(), InitializePoints{}, output, extents); + } + } +}; + +template +struct ImageTester : public legate::LegateTask> { + static const int32_t TASK_ID = IMAGE_TESTER + static_cast(RECT) * TEST_MAX_DIM + DIM; + struct CheckRects { + template + void operator()(legate::Store& func, const legate::Domain& range) + { + auto shape = func.shape(); + if (shape.empty()) return; + + auto acc = func.read_accessor, DIM>(); + for (legate::PointInRectIterator it(shape); it.valid(); ++it) { + auto rect = acc[*it]; + for (legate::PointInRectIterator rit(rect); rit.valid(); ++rit) { + EXPECT_TRUE(range.contains(*rit)); + } + } + } + }; + struct CheckPoints { + template + void operator()(legate::Store& func, const legate::Domain& range) + { + auto shape = func.shape(); + if (shape.empty()) return; + + auto acc = func.read_accessor, DIM>(); + for (legate::PointInRectIterator it(shape); it.valid(); ++it) { + auto p = acc[*it]; + EXPECT_TRUE(range.contains(p)); + } + } + }; + static void cpu_variant(legate::TaskContext& context) + { + auto& func = context.inputs().at(0); + + auto range = context.inputs().at(1).domain(); + EXPECT_FALSE(range.get_volume() > 1 && range.dense()); + + if constexpr (RECT) { + const auto& rect_type = func.type().as_struct_type(); + const auto& point_type = rect_type.field_type(0).as_fixed_array_type(); + legate::dim_dispatch(point_type.num_elements(), CheckRects{}, func, range); + } else { + const auto& point_type = func.type().as_fixed_array_type(); + legate::dim_dispatch(point_type.num_elements(), CheckPoints{}, func, range); + } + } +}; + +void prepare() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + InitializeFunction<1, true>::register_variants(context); + InitializeFunction<2, true>::register_variants(context); + InitializeFunction<3, true>::register_variants(context); + InitializeFunction<1, false>::register_variants(context); + InitializeFunction<2, false>::register_variants(context); + InitializeFunction<3, false>::register_variants(context); + ImageTester<1, true>::register_variants(context); + ImageTester<2, true>::register_variants(context); + ImageTester<3, true>::register_variants(context); + ImageTester<1, false>::register_variants(context); + ImageTester<2, false>::register_variants(context); + ImageTester<3, false>::register_variants(context); +} + +void initialize_function(legate::LogicalStore func, const std::vector range_extents) +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + auto is_rect = func.type().code == legate::Type::Code::STRUCT; + auto task = runtime->create_task( + context, INIT_FUNC + static_cast(is_rect) * TEST_MAX_DIM + func.dim()); + auto part = task.declare_partition(); + task.add_output(func, part); + task.add_scalar_arg(range_extents); + task.add_constraint(legate::broadcast(part, legate::from_range(func.dim()))); + runtime->submit(std::move(task)); + + func.impl()->reset_key_partition(); +} + +void check_image(legate::LogicalStore func, legate::LogicalStore range) +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + auto is_rect = func.type().code == legate::Type::Code::STRUCT; + auto task = runtime->create_task( + context, IMAGE_TESTER + static_cast(is_rect) * TEST_MAX_DIM + func.dim()); + auto part_domain = task.declare_partition(); + auto part_range = task.declare_partition(); + + task.add_input(func, part_domain); + task.add_input(range, part_range); + task.add_constraint(legate::image(part_domain, part_range)); + + runtime->submit(std::move(task)); +} + +struct ImageTestSpec { + std::vector domain_extents; + std::vector range_extents; + bool is_rect; +}; + +void test_image(const ImageTestSpec& spec) +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + int32_t tgt_dim = spec.range_extents.size(); + auto image_type = spec.is_rect ? legate::rect_type(tgt_dim) : legate::point_type(tgt_dim); + auto func = runtime->create_store(spec.domain_extents, std::move(image_type)); + auto range = runtime->create_store(spec.range_extents, legate::int64()); + + initialize_function(func, spec.range_extents); + runtime->issue_fill(range, legate::Scalar(int64_t(1234))); + check_image(func, range); +} + +void test_invalid() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + auto func = runtime->create_store({10, 10}, legate::int32()); + auto range = runtime->create_store({10, 10}, legate::int64()); + + auto task = runtime->create_task(context, IMAGE_TESTER + 1); + auto part_domain = task.declare_partition(); + auto part_range = task.declare_partition(); + + task.add_input(func, part_domain); + task.add_input(range, part_range); + task.add_constraint(legate::image(part_domain, part_range)); + + EXPECT_THROW(runtime->submit(std::move(task)), std::invalid_argument); +} + +TEST(ImageConstraint, Point1D) +{ + legate::Core::perform_registration(); + test_image({{9}, {100}, false}); +} + +TEST(ImageConstraint, Point2D) +{ + legate::Core::perform_registration(); + test_image({{4, 4}, {10, 10}, false}); +} + +TEST(ImageConstraint, Point3D) +{ + legate::Core::perform_registration(); + test_image({{2, 3, 4}, {5, 5, 5}, false}); +} + +TEST(ImageConstraint, Rect1D) +{ + legate::Core::perform_registration(); + test_image({{9}, {100}, true}); +} + +TEST(ImageConstraint, Rect2D) +{ + legate::Core::perform_registration(); + test_image({{4, 4}, {10, 10}, true}); +} + +TEST(ImageConstraint, Rect3D) +{ + legate::Core::perform_registration(); + test_image({{2, 3, 4}, {5, 5, 5}, true}); +} + +TEST(ImageConstraint, Invalid) +{ + legate::Core::perform_registration(); + test_invalid(); +} + +} // namespace image_constraints From b98430bc4f7dd512fd7679b80bc22c91b2e9a3a7 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Tue, 18 Jul 2023 16:22:38 -0700 Subject: [PATCH 0164/1425] Reduction fix and lenient alignments * Allow alignments to be set on the same variable * Make the example and test consistent with the new behavior * Move the reduction operator translation inside the runtime See merge request legate/legate.core.internal!86 --- examples/cpp/hello/hello.cc | 3 +- src/core/operation/detail/task.cc | 15 ++++---- src/core/operation/task.cc | 21 +++++++++-- src/core/operation/task.h | 41 +++++++++++++++++----- src/core/partitioning/constraint.cc | 3 +- src/core/partitioning/constraint.h | 3 ++ src/core/partitioning/constraint_solver.cc | 3 +- tests/cpp/integration/multi_scalar_out.cc | 12 +++---- 8 files changed, 71 insertions(+), 30 deletions(-) diff --git a/examples/cpp/hello/hello.cc b/examples/cpp/hello/hello.cc index a6be30d3d2..9a8fefd390 100644 --- a/examples/cpp/hello/hello.cc +++ b/examples/cpp/hello/hello.cc @@ -54,13 +54,12 @@ legate::LogicalStore sum(legate::LibraryContext* context, auto task = runtime->create_task(context, task::hello::SUM); auto output = runtime->create_store(legate::Scalar(legate::float32(), bytearray)); - auto redop = input.type().find_reduction_operator(legate::ReductionOpKind::ADD); auto part1 = task.declare_partition(); auto part2 = task.declare_partition(); task.add_input(input, part1); - task.add_reduction(output, redop, part2); + task.add_reduction(output, legate::ReductionOpKind::ADD, part2); runtime->submit(std::move(task)); return output; } diff --git a/src/core/operation/detail/task.cc b/src/core/operation/detail/task.cc index fdb0d68a87..da68a6dbff 100644 --- a/src/core/operation/detail/task.cc +++ b/src/core/operation/detail/task.cc @@ -250,12 +250,13 @@ void AutoTask::add_output(std::shared_ptr store, const Variable* p } void AutoTask::add_reduction(std::shared_ptr store, - Legion::ReductionOpID redop, + int32_t redop, const Variable* partition_symbol) { + auto legion_redop_id = store->type().find_reduction_operator(redop); if (store->has_scalar_storage()) scalar_reductions_.push_back(reductions_.size()); add_store(reductions_, std::move(store), partition_symbol); - reduction_ops_.push_back(redop); + reduction_ops_.push_back(legion_redop_id); } void AutoTask::add_store(std::vector& store_args, @@ -330,20 +331,22 @@ void ManualTask::add_output(std::shared_ptr store_partiti add_store(outputs_, store_partition->store(), store_partition->partition()); } -void ManualTask::add_reduction(std::shared_ptr store, Legion::ReductionOpID redop) +void ManualTask::add_reduction(std::shared_ptr store, int32_t redop) { + auto legion_redop_id = store->type().find_reduction_operator(redop); if (store->has_scalar_storage()) scalar_reductions_.push_back(reductions_.size()); add_store(reductions_, std::move(store), create_no_partition()); - reduction_ops_.push_back(redop); + reduction_ops_.push_back(legion_redop_id); } void ManualTask::add_reduction(std::shared_ptr store_partition, - Legion::ReductionOpID redop) + int32_t redop) { + auto legion_redop_id = store_partition->store()->type().find_reduction_operator(redop); if (store_partition->store()->has_scalar_storage()) scalar_reductions_.push_back(reductions_.size()); add_store(reductions_, store_partition->store(), store_partition->partition()); - reduction_ops_.push_back(redop); + reduction_ops_.push_back(legion_redop_id); } void ManualTask::add_store(std::vector& store_args, diff --git a/src/core/operation/task.cc b/src/core/operation/task.cc index e4ac878afa..d9d0d76f75 100644 --- a/src/core/operation/task.cc +++ b/src/core/operation/task.cc @@ -35,8 +35,13 @@ void AutoTask::add_output(LogicalStore store, const Variable* partition_symbol) } void AutoTask::add_reduction(LogicalStore store, - Legion::ReductionOpID redop, + ReductionOpKind redop, const Variable* partition_symbol) +{ + impl_->add_reduction(store.impl(), static_cast(redop), partition_symbol); +} + +void AutoTask::add_reduction(LogicalStore store, int32_t redop, const Variable* partition_symbol) { impl_->add_reduction(store.impl(), redop, partition_symbol); } @@ -94,12 +99,22 @@ void ManualTask::add_output(LogicalStorePartition store_partition) impl_->add_output(store_partition.impl()); } -void ManualTask::add_reduction(LogicalStore store, Legion::ReductionOpID redop) +void ManualTask::add_reduction(LogicalStore store, ReductionOpKind redop) +{ + impl_->add_reduction(store.impl(), static_cast(redop)); +} + +void ManualTask::add_reduction(LogicalStore store, int32_t redop) { impl_->add_reduction(store.impl(), redop); } -void ManualTask::add_reduction(LogicalStorePartition store_partition, Legion::ReductionOpID redop) +void ManualTask::add_reduction(LogicalStorePartition store_partition, ReductionOpKind redop) +{ + impl_->add_reduction(store_partition.impl(), static_cast(redop)); +} + +void ManualTask::add_reduction(LogicalStorePartition store_partition, int32_t redop) { impl_->add_reduction(store_partition.impl(), redop); } diff --git a/src/core/operation/task.h b/src/core/operation/task.h index b8d52d7b9a..102ac4e83b 100644 --- a/src/core/operation/task.h +++ b/src/core/operation/task.h @@ -70,12 +70,21 @@ class AutoTask { * associated with the store * * @param store A store to add to the task for reductions - * @param redop ID of the reduction operator to use + * @param redop ID of the reduction operator to use. The store's type must support the operator. * @param partition_symbol A partition symbol for the store */ - void add_reduction(LogicalStore store, - Legion::ReductionOpID redop, - const Variable* partition_symbol); + void add_reduction(LogicalStore store, ReductionOpKind redop, const Variable* partition_symbol); + /** + * @brief Adds a store to the task for reductions + * + * Partitioning of the store is controlled by constraints on the partition symbol + * associated with the store + * + * @param store A store to add to the task for reductions + * @param redop ID of the reduction operator to use. The store's type must support the operator. + * @param partition_symbol A partition symbol for the store + */ + void add_reduction(LogicalStore store, int32_t redop, const Variable* partition_symbol); /** * @brief Adds a by-value scalar argument to the task * @@ -210,16 +219,32 @@ class ManualTask { * The store will be unpartitioned but broadcasted to all the tasks * * @param store A store to add to the task for reductions - * @param redop ID of the reduction operator + * @param redop ID of the reduction operator to use. The store's type must support the operator. + */ + void add_reduction(LogicalStore store, ReductionOpKind redop); + /** + * @brief Adds a store to the task for reductions + * + * The store will be unpartitioned but broadcasted to all the tasks + * + * @param store A store to add to the task for reductions + * @param redop ID of the reduction operator to use. The store's type must support the operator. + */ + void add_reduction(LogicalStore store, int32_t redop); + /** + * @brief Adds a store partition to the task for reductions + * + * @param store_partition A store partition to add to the task for reductions + * @param redop ID of the reduction operator to use. The store's type must support the operator. */ - void add_reduction(LogicalStore store, Legion::ReductionOpID redop); + void add_reduction(LogicalStorePartition store_partition, ReductionOpKind redop); /** * @brief Adds a store partition to the task for reductions * * @param store_partition A store partition to add to the task for reductions - * @param redop ID of the reduction operator + * @param redop ID of the reduction operator to use. The store's type must support the operator. */ - void add_reduction(LogicalStorePartition store_partition, Legion::ReductionOpID redop); + void add_reduction(LogicalStorePartition store_partition, int32_t redop); /** * @brief Adds a by-value scalar argument to the task * diff --git a/src/core/partitioning/constraint.cc b/src/core/partitioning/constraint.cc index fc7fd08deb..53a9b899a8 100644 --- a/src/core/partitioning/constraint.cc +++ b/src/core/partitioning/constraint.cc @@ -64,6 +64,7 @@ void Alignment::find_partition_symbols(std::vector& partition_s void Alignment::validate() const { + if (*lhs_ == *rhs_) return; auto lhs_store = lhs_->operation()->find_store(lhs_.get()); auto rhs_store = rhs_->operation()->find_store(rhs_.get()); if (lhs_store->extents() != rhs_store->extents()) @@ -147,8 +148,6 @@ std::unique_ptr ImageConstraint::resolve(const detail::Strategy& stra std::unique_ptr align(const Variable* lhs, const Variable* rhs) { - if (*lhs == *rhs) throw std::invalid_argument("Alignment needs two distinct variables"); - // Since an Alignment object owns child nodes, inputs need to be copied return std::make_unique(std::make_unique(*lhs), std::make_unique(*rhs)); } diff --git a/src/core/partitioning/constraint.h b/src/core/partitioning/constraint.h index 3d682a922d..b5b0df3948 100644 --- a/src/core/partitioning/constraint.h +++ b/src/core/partitioning/constraint.h @@ -229,6 +229,9 @@ class Alignment : public Constraint { */ const Variable* rhs() const { return rhs_.get(); } + public: + bool is_trivial() const { return *lhs_ == *rhs_; } + private: std::unique_ptr lhs_; std::unique_ptr rhs_; diff --git a/src/core/partitioning/constraint_solver.cc b/src/core/partitioning/constraint_solver.cc index b26b37b434..8ce81d33b9 100644 --- a/src/core/partitioning/constraint_solver.cc +++ b/src/core/partitioning/constraint_solver.cc @@ -168,7 +168,8 @@ void ConstraintSolver::solve_constraints() for (auto& constraint : constraints_) { switch (constraint->kind()) { case Constraint::Kind::ALIGNMENT: { - handle_alignment(constraint->as_alignment()); + auto* alignment = constraint->as_alignment(); + if (!alignment->is_trivial()) handle_alignment(alignment); break; } case Constraint::Kind::BROADCAST: { diff --git a/tests/cpp/integration/multi_scalar_out.cc b/tests/cpp/integration/multi_scalar_out.cc index 3a97140f37..29fddeadf6 100644 --- a/tests/cpp/integration/multi_scalar_out.cc +++ b/tests/cpp/integration/multi_scalar_out.cc @@ -44,10 +44,8 @@ void test_reducer_auto(legate::LibraryContext* context, auto part1 = task.declare_partition(); auto part2 = task.declare_partition(); auto part3 = task.declare_partition(); - auto redop1 = scalar1.type().find_reduction_operator(legate::ReductionOpKind::ADD); - auto redop2 = scalar2.type().find_reduction_operator(legate::ReductionOpKind::MUL); - task.add_reduction(scalar1, redop1, part1); - task.add_reduction(scalar2, redop2, part2); + task.add_reduction(scalar1, legate::ReductionOpKind::ADD, part1); + task.add_reduction(scalar2, legate::ReductionOpKind::MUL, part2); task.add_output(store, part3); runtime->submit(std::move(task)); } @@ -58,10 +56,8 @@ void test_reducer_manual(legate::LibraryContext* context, { auto runtime = legate::Runtime::get_runtime(); auto task = runtime->create_task(context, task::simple::REDUCER, legate::Shape({2})); - auto redop1 = scalar1.type().find_reduction_operator(legate::ReductionOpKind::ADD); - auto redop2 = scalar2.type().find_reduction_operator(legate::ReductionOpKind::MUL); - task.add_reduction(scalar1, redop1); - task.add_reduction(scalar2, redop2); + task.add_reduction(scalar1, legate::ReductionOpKind::ADD); + task.add_reduction(scalar2, legate::ReductionOpKind::MUL); runtime->submit(std::move(task)); } From 92319e60cb42895a35695ba4d15ce1ca67810890 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Tue, 18 Jul 2023 16:24:51 -0700 Subject: [PATCH 0165/1425] Update to pass pre-commit hooks --- tests/cpp/unit/span.cc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/cpp/unit/span.cc b/tests/cpp/unit/span.cc index 4f4376bbb1..f60d708e58 100644 --- a/tests/cpp/unit/span.cc +++ b/tests/cpp/unit/span.cc @@ -19,7 +19,6 @@ namespace span_test { - constexpr bool BOOL_VALUE = true; constexpr int8_t INT8_VALUE = 10; constexpr int16_t INT16_VALUE = -1000; @@ -42,7 +41,6 @@ const complex COMPLEX_DOUBLE_VALUE1{6, 7}; const complex COMPLEX_DOUBLE_VALUE2{8, 9}; const complex COMPLEX_DOUBLE_VALUE3{10, 11}; - constexpr uint32_t DATA_SIZE = 3; template @@ -69,9 +67,12 @@ TEST(SpanUnit, Create) create(INT32_VALUE, static_cast(INT32_VALUE + 1), static_cast(INT32_VALUE + 2)); create(INT64_VALUE, static_cast(INT64_VALUE + 1), static_cast(INT64_VALUE + 2)); create(UINT8_VALUE, static_cast(UINT8_VALUE + 1), static_cast(UINT8_VALUE + 2)); - create(UINT16_VALUE, static_cast(UINT16_VALUE + 1), static_cast(UINT16_VALUE + 2)); - create(UINT32_VALUE, static_cast(UINT32_VALUE + 1), static_cast(UINT32_VALUE + 2)); - create(UINT64_VALUE, static_cast(UINT64_VALUE + 1), static_cast(UINT64_VALUE + 2)); + create( + UINT16_VALUE, static_cast(UINT16_VALUE + 1), static_cast(UINT16_VALUE + 2)); + create( + UINT32_VALUE, static_cast(UINT32_VALUE + 1), static_cast(UINT32_VALUE + 2)); + create( + UINT64_VALUE, static_cast(UINT64_VALUE + 1), static_cast(UINT64_VALUE + 2)); create(FLOAT_VALUE, FLOAT_VALUE + 1.0f, FLOAT_VALUE + 2.0f); create(DOUBLE_VALUE, DOUBLE_VALUE + 1.0d, DOUBLE_VALUE + 2.0d); create(STRING_VALUE, STRING_VALUE + "1", STRING_VALUE + "2"); From b86527f4a298c50d8cf57d350c31e584698eb783 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Tue, 18 Jul 2023 23:32:37 -0700 Subject: [PATCH 0166/1425] Move the move constructors to .cc file * Move the move constructors to .cc file See merge request legate/legate.core.internal!87 --- src/core/operation/task.cc | 8 ++++++++ src/core/operation/task.h | 12 ++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/core/operation/task.cc b/src/core/operation/task.cc index d9d0d76f75..b3a5931f21 100644 --- a/src/core/operation/task.cc +++ b/src/core/operation/task.cc @@ -77,6 +77,10 @@ void AutoTask::throws_exception(bool can_throw_exception) void AutoTask::add_communicator(const std::string& name) { impl_->add_communicator(name); } +AutoTask::AutoTask(AutoTask&&) = default; + +AutoTask& AutoTask::operator=(AutoTask&&) = default; + AutoTask::~AutoTask() {} AutoTask::AutoTask(std::unique_ptr impl) : impl_(std::move(impl)) {} @@ -138,6 +142,10 @@ void ManualTask::throws_exception(bool can_throw_exception) void ManualTask::add_communicator(const std::string& name) { impl_->add_communicator(name); } +ManualTask::ManualTask(ManualTask&&) = default; + +ManualTask& ManualTask::operator=(ManualTask&&) = default; + ManualTask::~ManualTask() {} ManualTask::ManualTask(std::unique_ptr impl) : impl_(std::move(impl)) {} diff --git a/src/core/operation/task.h b/src/core/operation/task.h index 102ac4e83b..fde6df8a8c 100644 --- a/src/core/operation/task.h +++ b/src/core/operation/task.h @@ -165,10 +165,12 @@ class AutoTask { void add_communicator(const std::string& name); public: + AutoTask(AutoTask&&); + AutoTask& operator=(AutoTask&&); + + private: AutoTask(const AutoTask&) = delete; - AutoTask(AutoTask&&) = default; AutoTask& operator=(const AutoTask&) = delete; - AutoTask& operator=(AutoTask&&) = default; public: ~AutoTask(); @@ -305,10 +307,12 @@ class ManualTask { void add_communicator(const std::string& name); public: + ManualTask(ManualTask&&); + ManualTask& operator=(ManualTask&&); + + private: ManualTask(const ManualTask&) = delete; - ManualTask(ManualTask&&) = default; ManualTask& operator=(const ManualTask&) = delete; - ManualTask& operator=(ManualTask&&) = default; public: ~ManualTask(); From a665b2124f251c2eb2be3cf8775e7f54be9ae4ee Mon Sep 17 00:00:00 2001 From: Mark Vaz Date: Fri, 21 Jul 2023 12:27:52 -0700 Subject: [PATCH 0167/1425] pin cuda-version (#798) --- conda/conda-build/meta.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conda/conda-build/meta.yaml b/conda/conda-build/meta.yaml index e384affcf7..547a5672b4 100644 --- a/conda/conda-build/meta.yaml +++ b/conda/conda-build/meta.yaml @@ -121,7 +121,8 @@ requirements: - numpy {{ numpy_version }} - typing_extensions {% if gpu_enabled_bool %} - - cuda-cudart >={{ cuda_version }},<{{ cuda_major+1 }} + - cuda-cudart >={{ cuda_version }},<{{ cuda_major+1 }} + - cuda-version >={{ cuda_version }},<{{ cuda_major+1 }} - nccl {% endif %} From f1419c41eda8fd77f94a58e3493145a2589f665d Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Tue, 25 Jul 2023 01:10:20 -0700 Subject: [PATCH 0168/1425] Pimpl idiom for all user-facing abstractions * Pimpl for task info and registrar * Fix a compile issue in release build * Pimpl for ScopedAllocator * Minor refactoring for processor ranges * Pimpl for mapping::Machine * Pimpl for constraints and rearrange detail classes * Pimpl for the mapping policy abstractions * Stop exposing references to internal data structures in legate::mapping * Pimpl for legate::mapping::Task and legate::mapping::Store * Pimpl for legate::TaskContext and make return.h an internal header * Pimpl for legate::Store * More clean-up for applying pimpl to legate::Store * Start Pimpl for legate::Store * Propogate exceptions from find_library to python and clean up examples * Move TaskContext to the task sub-directory * Make buffer builder internal * Rearrange LibraryContext and TaskContext * Fix tests and examples * Pimpl for legate::Scalar * Deserializer headers don't need to be installed * No need to use reference wrappers * Pimpl for types See merge request legate/legate.core.internal!90 --- examples/cpp/hello/hello.cc | 28 +- examples/cpp/hello/hello_print.cc | 14 +- examples/cpp/hello/task_hello.cc | 10 +- examples/cpp/io/io.cc | 69 +-- examples/cpp/io/task_io.cc | 14 +- examples/io/legateio/legateio.py | 4 +- examples/io/src/util.h | 2 +- legate/core/_lib/context.pyx | 17 +- legate/core/_lib/types.pyx | 45 +- legate/core/types.pyi | 3 + legate_core_cpp.cmake | 56 +- src/core/comm/coll.cc | 7 +- src/core/comm/comm.cc | 17 +- src/core/comm/comm.h | 14 +- src/core/comm/comm_cpu.cc | 38 +- src/core/comm/comm_cpu.h | 12 +- src/core/comm/comm_nccl.cu | 38 +- src/core/comm/comm_nccl.h | 12 +- src/core/comm/local_comm.cc | 3 +- src/core/comm/mpi_comm.cc | 2 +- src/core/data/allocator.cc | 61 +- src/core/data/allocator.h | 25 +- src/core/data/detail/logical_store.cc | 39 +- src/core/data/detail/logical_store.h | 34 +- src/core/data/detail/scalar.cc | 103 ++++ src/core/data/detail/scalar.h | 71 +++ src/core/data/detail/store.cc | 453 +++++++++++++++ src/core/data/detail/store.h | 256 +++++++++ src/core/data/{ => detail}/transform.cc | 9 +- src/core/data/{ => detail}/transform.h | 10 +- src/core/data/logical_store.cc | 18 +- src/core/data/logical_store.h | 20 +- src/core/data/scalar.cc | 70 +-- src/core/data/scalar.h | 42 +- src/core/data/scalar.inl | 78 ++- src/core/data/store.cc | 375 ++----------- src/core/data/store.h | 526 ++++-------------- src/core/data/store.inl | 464 +++++---------- src/core/mapping/{ => detail}/base_mapper.cc | 211 ++++--- src/core/mapping/{ => detail}/base_mapper.h | 28 +- src/core/mapping/{ => detail}/core_mapper.cc | 19 +- src/core/mapping/{ => detail}/core_mapper.h | 4 +- .../mapping/{ => detail}/default_mapper.cc | 13 +- .../mapping/{ => detail}/default_mapper.h | 11 +- .../mapping/{ => detail}/instance_manager.cc | 6 +- .../mapping/{ => detail}/instance_manager.h | 4 +- src/core/mapping/detail/machine.cc | 346 ++++++++++++ src/core/mapping/detail/machine.h | 152 +++++ src/core/mapping/detail/mapping.cc | 194 +++++++ src/core/mapping/detail/mapping.h | 96 ++++ src/core/mapping/detail/operation.cc | 90 +++ src/core/mapping/detail/operation.h | 146 +++++ src/core/mapping/{ => detail}/operation.inl | 0 src/core/mapping/detail/store.cc | 146 +++++ src/core/mapping/detail/store.h | 163 ++++++ src/core/mapping/machine.cc | 455 +++------------ src/core/mapping/machine.h | 250 +++------ src/core/mapping/mapping.cc | 349 +++++++----- src/core/mapping/mapping.h | 287 +++++++--- src/core/mapping/operation.cc | 95 ++-- src/core/mapping/operation.h | 122 ++-- src/core/mapping/store.cc | 124 +---- src/core/mapping/store.h | 152 +---- src/core/operation/detail/copy.cc | 10 +- src/core/operation/detail/copy.h | 2 +- src/core/operation/detail/copy_launcher.cc | 14 +- src/core/operation/detail/copy_launcher.h | 11 +- src/core/operation/detail/fill.cc | 6 +- src/core/operation/detail/fill.h | 2 +- src/core/operation/detail/fill_launcher.cc | 11 +- src/core/operation/detail/fill_launcher.h | 11 +- src/core/operation/detail/gather.cc | 10 +- src/core/operation/detail/gather.h | 2 +- src/core/operation/detail/launcher_arg.cc | 4 +- src/core/operation/detail/launcher_arg.h | 5 +- src/core/operation/detail/operation.cc | 6 +- src/core/operation/detail/operation.h | 13 +- src/core/operation/detail/scatter.cc | 10 +- src/core/operation/detail/scatter.h | 2 +- src/core/operation/detail/scatter_gather.cc | 10 +- src/core/operation/detail/scatter_gather.h | 2 +- src/core/operation/detail/task.cc | 31 +- src/core/operation/detail/task.h | 17 +- src/core/operation/detail/task_launcher.cc | 24 +- src/core/operation/detail/task_launcher.h | 24 +- src/core/operation/task.cc | 45 +- src/core/operation/task.h | 32 +- src/core/partitioning/constraint.cc | 152 +---- src/core/partitioning/constraint.h | 307 +--------- src/core/partitioning/detail/constraint.cc | 162 ++++++ src/core/partitioning/detail/constraint.h | 231 ++++++++ .../{ => detail}/constraint_solver.cc | 8 +- .../{ => detail}/constraint_solver.h | 2 +- .../partitioning/{ => detail}/partitioner.cc | 6 +- .../partitioning/{ => detail}/partitioner.h | 2 +- src/core/partitioning/partition.cc | 3 +- src/core/runtime/context.cc | 315 ----------- .../runtime/detail/communicator_manager.h | 14 +- src/core/runtime/detail/library.cc | 173 ++++++ src/core/runtime/detail/library.h | 135 +++++ src/core/runtime/detail/machine_manager.cc | 13 +- src/core/runtime/detail/machine_manager.h | 15 +- src/core/runtime/detail/partition_manager.cc | 10 +- src/core/runtime/detail/partition_manager.h | 11 +- src/core/runtime/{ => detail}/projection.cc | 17 +- src/core/runtime/{ => detail}/projection.h | 8 +- src/core/runtime/detail/runtime.cc | 116 ++-- src/core/runtime/detail/runtime.h | 45 +- src/core/runtime/{ => detail}/shard.cc | 52 +- src/core/runtime/{ => detail}/shard.h | 8 +- src/core/runtime/library.cc | 114 ++++ src/core/runtime/{context.h => library.h} | 165 +----- src/core/runtime/{context.inl => library.inl} | 5 +- src/core/runtime/resource.h | 37 +- src/core/runtime/runtime.cc | 73 +-- src/core/runtime/runtime.h | 70 +-- src/core/runtime/tracker.cc | 11 +- src/core/runtime/tracker.h | 6 +- src/core/task/{ => detail}/return.cc | 23 +- src/core/task/{ => detail}/return.h | 6 +- src/core/task/detail/task_context.cc | 145 +++++ src/core/task/detail/task_context.h | 74 +++ src/core/task/exception.h | 2 + src/core/task/registrar.cc | 21 +- src/core/task/registrar.h | 25 +- src/core/task/task.cc | 12 +- src/core/task/task.h | 17 +- src/core/task/task.inl | 10 +- src/core/task/task_context.cc | 53 ++ src/core/task/task_context.h | 132 +++++ src/core/task/task_info.cc | 75 ++- src/core/task/task_info.h | 16 +- src/core/type/detail/type_info.cc | 289 ++++++++++ src/core/type/detail/type_info.h | 162 ++++++ src/core/type/type_info.cc | 371 +++++------- src/core/type/type_info.h | 256 +++------ src/core/utilities/deserializer.cc | 72 +-- src/core/utilities/deserializer.h | 40 +- src/core/utilities/deserializer.inl | 48 +- .../utilities/{ => detail}/buffer_builder.cc | 6 +- .../utilities/{ => detail}/buffer_builder.h | 6 +- .../utilities/{ => detail}/buffer_builder.inl | 8 +- src/core/utilities/nvtx_help.h | 2 +- src/core/utilities/typedefs.h | 2 +- src/env_defaults.h | 4 +- src/legate.h | 4 +- tests/cpp/integration/copy_gather.cc | 22 +- tests/cpp/integration/copy_gather_scatter.cc | 62 +-- tests/cpp/integration/copy_normal.cc | 40 +- tests/cpp/integration/copy_scatter.cc | 68 +-- tests/cpp/integration/copy_util.inl | 10 +- tests/cpp/integration/image_constraints.cc | 4 +- tests/cpp/integration/inline_map.cc | 18 +- tests/cpp/integration/machine_scope.cc | 30 +- tests/cpp/integration/manual_simple.cc | 18 +- tests/cpp/integration/multi_scalar_out.cc | 24 +- tests/cpp/integration/nccl.cu | 4 +- tests/cpp/integration/provenance.cc | 56 +- tests/cpp/integration/weighted.cc | 20 +- tests/cpp/unit/library.cc | 15 +- tests/cpp/unit/machine.cc | 107 ++-- tests/cpp/unit/registration.cc | 4 +- tests/cpp/unit/scalar.cc | 33 +- tests/cpp/unit/store.cc | 4 +- .../scoping/examples/test_scoping.py | 4 +- .../legate/core/test_invalid_scalar_arg.py | 6 +- 166 files changed, 6559 insertions(+), 4983 deletions(-) create mode 100644 src/core/data/detail/scalar.cc create mode 100644 src/core/data/detail/scalar.h create mode 100644 src/core/data/detail/store.cc create mode 100644 src/core/data/detail/store.h rename src/core/data/{ => detail}/transform.cc (99%) rename src/core/data/{ => detail}/transform.h (98%) rename src/core/mapping/{ => detail}/base_mapper.cc (89%) rename src/core/mapping/{ => detail}/base_mapper.h (96%) rename src/core/mapping/{ => detail}/core_mapper.cc (92%) rename src/core/mapping/{ => detail}/core_mapper.h (90%) rename src/core/mapping/{ => detail}/default_mapper.cc (68%) rename src/core/mapping/{ => detail}/default_mapper.h (71%) rename src/core/mapping/{ => detail}/instance_manager.cc (99%) rename src/core/mapping/{ => detail}/instance_manager.h (98%) create mode 100644 src/core/mapping/detail/machine.cc create mode 100644 src/core/mapping/detail/machine.h create mode 100644 src/core/mapping/detail/mapping.cc create mode 100644 src/core/mapping/detail/mapping.h create mode 100644 src/core/mapping/detail/operation.cc create mode 100644 src/core/mapping/detail/operation.h rename src/core/mapping/{ => detail}/operation.inl (100%) create mode 100644 src/core/mapping/detail/store.cc create mode 100644 src/core/mapping/detail/store.h create mode 100644 src/core/partitioning/detail/constraint.cc create mode 100644 src/core/partitioning/detail/constraint.h rename src/core/partitioning/{ => detail}/constraint_solver.cc (97%) rename src/core/partitioning/{ => detail}/constraint_solver.h (97%) rename src/core/partitioning/{ => detail}/partitioner.cc (98%) rename src/core/partitioning/{ => detail}/partitioner.h (98%) delete mode 100644 src/core/runtime/context.cc create mode 100644 src/core/runtime/detail/library.cc create mode 100644 src/core/runtime/detail/library.h rename src/core/runtime/{ => detail}/projection.cc (96%) rename src/core/runtime/{ => detail}/projection.h (94%) rename src/core/runtime/{ => detail}/shard.cc (79%) rename src/core/runtime/{ => detail}/shard.h (83%) create mode 100644 src/core/runtime/library.cc rename src/core/runtime/{context.h => library.h} (51%) rename src/core/runtime/{context.inl => library.inl} (95%) rename src/core/task/{ => detail}/return.cc (96%) rename src/core/task/{ => detail}/return.h (96%) create mode 100644 src/core/task/detail/task_context.cc create mode 100644 src/core/task/detail/task_context.h create mode 100644 src/core/task/task_context.cc create mode 100644 src/core/task/task_context.h create mode 100644 src/core/type/detail/type_info.cc create mode 100644 src/core/type/detail/type_info.h rename src/core/utilities/{ => detail}/buffer_builder.cc (90%) rename src/core/utilities/{ => detail}/buffer_builder.h (94%) rename src/core/utilities/{ => detail}/buffer_builder.inl (87%) diff --git a/examples/cpp/hello/hello.cc b/examples/cpp/hello/hello.cc index 9a8fefd390..ce613974e0 100644 --- a/examples/cpp/hello/hello.cc +++ b/examples/cpp/hello/hello.cc @@ -21,10 +21,10 @@ namespace hello { -legate::LogicalStore iota(legate::LibraryContext* context, size_t size) +legate::LogicalStore iota(legate::Library library, size_t size) { auto runtime = legate::Runtime::get_runtime(); - auto task = runtime->create_task(context, task::hello::IOTA); + auto task = runtime->create_task(library, task::hello::IOTA); auto output = runtime->create_store({size}, legate::float32(), true); auto part = task.declare_partition(); task.add_output(output, part); @@ -32,10 +32,10 @@ legate::LogicalStore iota(legate::LibraryContext* context, size_t size) return output; } -legate::LogicalStore square(legate::LibraryContext* context, legate::LogicalStore input) +legate::LogicalStore square(legate::Library library, legate::LogicalStore input) { auto runtime = legate::Runtime::get_runtime(); - auto task = runtime->create_task(context, task::hello::SQUARE); + auto task = runtime->create_task(library, task::hello::SQUARE); auto output = runtime->create_store(input.extents().data(), legate::float32(), true); auto part1 = task.declare_partition(); auto part2 = task.declare_partition(); @@ -46,12 +46,10 @@ legate::LogicalStore square(legate::LibraryContext* context, legate::LogicalStor return output; } -legate::LogicalStore sum(legate::LibraryContext* context, - legate::LogicalStore input, - const void* bytearray) +legate::LogicalStore sum(legate::Library library, legate::LogicalStore input, const void* bytearray) { auto runtime = legate::Runtime::get_runtime(); - auto task = runtime->create_task(context, task::hello::SUM); + auto task = runtime->create_task(library, task::hello::SUM); auto output = runtime->create_store(legate::Scalar(legate::float32(), bytearray)); @@ -64,11 +62,11 @@ legate::LogicalStore sum(legate::LibraryContext* context, return output; } -float to_scalar(legate::LibraryContext* context, legate::LogicalStore scalar) +float to_scalar(legate::Library library, legate::LogicalStore scalar) { auto runtime = legate::Runtime::get_runtime(); auto p_scalar = scalar.get_physical_store(); - auto acc = p_scalar->read_accessor(); + auto acc = p_scalar.read_accessor(); float output = static_cast(acc[{0}]); return output; } @@ -78,14 +76,14 @@ TEST(Example, Hello) legate::Core::perform_registration(); auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library(task::hello::library_name); + auto library = runtime->find_library(task::hello::library_name); float bytearray = 0; - auto store = iota(context, 5); - auto storeSquared = square(context, store); - auto storeSummed = sum(context, storeSquared, &bytearray); - float scalar = to_scalar(context, storeSummed); + auto store = iota(library, 5); + auto storeSquared = square(library, store); + auto storeSummed = sum(library, storeSquared, &bytearray); + float scalar = to_scalar(library, storeSummed); ASSERT_EQ(scalar, 55); } diff --git a/examples/cpp/hello/hello_print.cc b/examples/cpp/hello/hello_print.cc index 6516bea9b9..dd730e2c21 100644 --- a/examples/cpp/hello/hello_print.cc +++ b/examples/cpp/hello/hello_print.cc @@ -21,19 +21,19 @@ namespace helloprint { -void test_print_hello(legate::LibraryContext* context, std::string str) +void test_print_hello(legate::Library library, std::string str) { auto runtime = legate::Runtime::get_runtime(); - auto task = runtime->create_task(context, task::hello::HELLO_WORLD); + auto task = runtime->create_task(library, task::hello::HELLO_WORLD); task.add_scalar_arg(legate::Scalar(str)); runtime->submit(std::move(task)); } -void test_print_hellos(legate::LibraryContext* context, std::string str, size_t count) +void test_print_hellos(legate::Library library, std::string str, size_t count) { auto runtime = legate::Runtime::get_runtime(); - auto task = runtime->create_task(context, task::hello::HELLO_WORLD, legate::Shape({count})); + auto task = runtime->create_task(library, task::hello::HELLO_WORLD, legate::Shape({count})); task.add_scalar_arg(legate::Scalar(str)); runtime->submit(std::move(task)); @@ -44,10 +44,10 @@ TEST(Example, HelloPrint) legate::Core::perform_registration(); auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library(task::hello::library_name); + auto library = runtime->find_library(task::hello::library_name); - test_print_hello(context, "Hello World!"); - test_print_hellos(context, "Hello World! x3", 3); + test_print_hello(library, "Hello World!"); + test_print_hellos(library, "Hello World! x3", 3); } } // namespace helloprint diff --git a/examples/cpp/hello/task_hello.cc b/examples/cpp/hello/task_hello.cc index 117c4fc4e1..269579a9d8 100644 --- a/examples/cpp/hello/task_hello.cc +++ b/examples/cpp/hello/task_hello.cc @@ -25,11 +25,11 @@ Legion::Logger logger(library_name); void register_tasks() { auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->create_library(library_name); - HelloWorldTask::register_variants(context); - IotaTask::register_variants(context); - SquareTask::register_variants(context); - SumTask::register_variants(context); + auto library = runtime->create_library(library_name); + HelloWorldTask::register_variants(library); + IotaTask::register_variants(library); + SquareTask::register_variants(library); + SumTask::register_variants(library); } /*static*/ void HelloWorldTask::cpu_variant(legate::TaskContext& context) diff --git a/examples/cpp/io/io.cc b/examples/cpp/io/io.cc index 8dc2726f56..170cdf810c 100644 --- a/examples/cpp/io/io.cc +++ b/examples/cpp/io/io.cc @@ -25,15 +25,12 @@ namespace legateio { class Array { public: - Array(legate::LibraryContext* context, legate::LogicalStore store) - : context_(context), store_(store) - { - } + Array(legate::Library library, legate::LogicalStore store) : library_(library), store_(store) {} legate::LogicalStore store() { return store_; } protected: - legate::LibraryContext* context_; + legate::Library library_; legate::LogicalStore store_; }; @@ -44,7 +41,7 @@ class IOArray : public Array { void to_file(std::string filename) { auto runtime = legate::Runtime::get_runtime(); - auto task = runtime->create_task(context_, task::legateio::WRITE_FILE); + auto task = runtime->create_task(library_, task::legateio::WRITE_FILE); auto part = task.declare_partition(); task.add_scalar_arg(legate::Scalar(filename)); @@ -68,7 +65,7 @@ class IOArray : public Array { } auto runtime = legate::Runtime::get_runtime(); - auto task = runtime->create_task(context_, task::legateio::WRITE_UNEVEN_TILES); + auto task = runtime->create_task(library_, task::legateio::WRITE_UNEVEN_TILES); auto part = task.declare_partition(); task.add_scalar_arg(legate::Scalar(path)); @@ -87,9 +84,9 @@ class IOArray : public Array { auto runtime = legate::Runtime::get_runtime(); auto store_partition = store_.partition_by_tiling({tile_shape, tile_shape}); - auto launch_shape = store_partition.partition()->color_shape(); + auto launch_shape = store_partition.color_shape(); - auto task = runtime->create_task(context_, task::legateio::WRITE_EVEN_TILES, launch_shape); + auto task = runtime->create_task(library_, task::legateio::WRITE_EVEN_TILES, launch_shape); task.add_input(store_partition); task.add_scalar_arg(legate::Scalar(path)); @@ -104,30 +101,28 @@ class IOArray : public Array { static IOArray from_store(legate::LogicalStore store) { auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library(task::legateio::library_name); - return IOArray(context, store); + auto library = runtime->find_library(task::legateio::library_name); + return IOArray(library, store); } }; -IOArray read_file(legate::LibraryContext* context, - std::string filename, - std::unique_ptr dtype) +IOArray read_file(legate::Library library, std::string filename, legate::Type dtype) { auto runtime = legate::Runtime::get_runtime(); auto output = runtime->create_store(std::move(dtype), 1); - auto task = runtime->create_task(context, task::legateio::READ_FILE); + auto task = runtime->create_task(library, task::legateio::READ_FILE); auto part = task.declare_partition(); task.add_scalar_arg(legate::Scalar(filename)); task.add_output(output, part); runtime->submit(std::move(task)); - return IOArray(context, output); + return IOArray(library, output); } -IOArray read_file_parallel(legate::LibraryContext* context, +IOArray read_file_parallel(legate::Library library, std::string filename, - std::unique_ptr dtype, + legate::Type dtype, uint parallelism = 0) { auto runtime = legate::Runtime::get_runtime(); @@ -137,13 +132,13 @@ IOArray read_file_parallel(legate::LibraryContext* context, auto output = runtime->create_store(std::move(dtype), 1); auto task = - runtime->create_task(context, task::legateio::READ_FILE, legate::Shape({parallelism})); + runtime->create_task(library, task::legateio::READ_FILE, legate::Shape({parallelism})); task.add_scalar_arg(legate::Scalar(filename)); task.add_output(output); runtime->submit(std::move(task)); - return IOArray(context, output); + return IOArray(library, output); } void _read_header_uneven(std::string path, std::vector& color_shape) @@ -166,7 +161,7 @@ void _read_header_uneven(std::string path, std::vector& color_shape) in.close(); } -IOArray read_uneven_tiles(legate::LibraryContext* context, std::string path) +IOArray read_uneven_tiles(legate::Library library, std::string path) { // Read the dataset's header to find the type code and the color shape of // the partition, which are laid out in the header in the following way: @@ -200,13 +195,13 @@ IOArray read_uneven_tiles(legate::LibraryContext* context, std::string path) auto runtime = legate::Runtime::get_runtime(); auto output = runtime->create_store(legate::int8(), color_shape.size()); auto task = - runtime->create_task(context, task::legateio::READ_UNEVEN_TILES, legate::Shape(color_shape)); + runtime->create_task(library, task::legateio::READ_UNEVEN_TILES, legate::Shape(color_shape)); task.add_output(output); task.add_scalar_arg(legate::Scalar(path)); runtime->submit(std::move(task)); - return IOArray(context, output); + return IOArray(library, output); } void _read_header_even(std::string path, @@ -235,7 +230,7 @@ void _read_header_even(std::string path, in.close(); } -IOArray read_even_tiles(legate::LibraryContext* context, std::string path) +IOArray read_even_tiles(legate::Library library, std::string path) { // Read the dataset's header to find the type code, the array's shape, and // the tile shape. The following shows the header's format: @@ -253,18 +248,14 @@ IOArray read_even_tiles(legate::LibraryContext* context, std::string path) auto runtime = legate::Runtime::get_runtime(); auto output = runtime->create_store(shape, legate::int8()); auto output_partition = output.partition_by_tiling(tile_shape); - auto launch_shape = output_partition.partition()->color_shape(); - auto task = runtime->create_task(context, task::legateio::READ_EVEN_TILES, launch_shape); + auto launch_shape = output_partition.color_shape(); + auto task = runtime->create_task(library, task::legateio::READ_EVEN_TILES, launch_shape); task.add_output(output_partition); task.add_scalar_arg(legate::Scalar(path)); runtime->submit(std::move(task)); - // Unlike AutoTask, manually parallelized tasks don't update the "key" - // partition for their outputs. - output.set_key_partition(runtime->get_machine(), output_partition.partition().get()); - - return IOArray(context, output); + return IOArray(library, output); } TEST(Example, SingleFileIO) @@ -272,7 +263,7 @@ TEST(Example, SingleFileIO) legate::Core::perform_registration(); auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library(task::legateio::library_name); + auto library = runtime->find_library(task::legateio::library_name); uint32_t n = 10; std::string filename = "test.dat"; @@ -288,16 +279,16 @@ TEST(Example, SingleFileIO) runtime->issue_execution_fence(); // Read the file into a IOArray - IOArray c2 = read_file(context, filename, legate::int8()); + IOArray c2 = read_file(library, filename, legate::int8()); EXPECT_TRUE(cunumeric::array_equal(src, cunumeric::as_array(c2.store()))); // Read the file into a IOArray with a fixed degree of parallelism - IOArray c3 = read_file_parallel(context, filename, legate::int8(), 2); + IOArray c3 = read_file_parallel(library, filename, legate::int8(), 2); EXPECT_TRUE(cunumeric::array_equal(src, cunumeric::as_array(c3.store()))); // Read the file into a IOArray with the library-chosen degree of // parallelism - IOArray c4 = read_file_parallel(context, filename, legate::int8()); + IOArray c4 = read_file_parallel(library, filename, legate::int8()); EXPECT_TRUE(cunumeric::array_equal(src, cunumeric::as_array(c4.store()))); } @@ -306,7 +297,7 @@ TEST(Example, EvenTilesIO) legate::Core::perform_registration(); auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library(task::legateio::library_name); + auto library = runtime->find_library(task::legateio::library_name); std::string dataset_name = "even_datafiles"; uint32_t store_shape = 8; @@ -324,7 +315,7 @@ TEST(Example, EvenTilesIO) runtime->issue_execution_fence(true); // Read the dataset into an IOArray - IOArray c2 = read_even_tiles(context, dataset_name); + IOArray c2 = read_even_tiles(library, dataset_name); // Convert the IOArray into a cuNumeric ndarray and perform a binary // operation, just to confirm in the profile that the partition from the @@ -341,7 +332,7 @@ TEST(Example, UnevenTilesIO) legate::Core::perform_registration(); auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library(task::legateio::library_name); + auto library = runtime->find_library(task::legateio::library_name); std::string dataset_name = "uneven_datafiles"; uint32_t store_shape = 8; @@ -358,7 +349,7 @@ TEST(Example, UnevenTilesIO) runtime->issue_execution_fence(true); // Read the dataset into an IOArray - IOArray c2 = read_uneven_tiles(context, dataset_name); + IOArray c2 = read_uneven_tiles(library, dataset_name); EXPECT_TRUE(cunumeric::array_equal(src, cunumeric::as_array(c2.store()))); } diff --git a/examples/cpp/io/task_io.cc b/examples/cpp/io/task_io.cc index 7cf4446a01..897425a4ab 100644 --- a/examples/cpp/io/task_io.cc +++ b/examples/cpp/io/task_io.cc @@ -30,13 +30,13 @@ Legion::Logger logger(library_name); void register_tasks() { auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->create_library(library_name); - ReadEvenTilesTask::register_variants(context); - ReadFileTask::register_variants(context); - ReadUnevenTilesTask::register_variants(context); - WriteEvenTilesTask::register_variants(context); - WriteFileTask::register_variants(context); - WriteUnevenTilesTask::register_variants(context); + auto library = runtime->create_library(library_name); + ReadEvenTilesTask::register_variants(library); + ReadFileTask::register_variants(library); + ReadUnevenTilesTask::register_variants(library); + WriteEvenTilesTask::register_variants(library); + WriteFileTask::register_variants(library); + WriteUnevenTilesTask::register_variants(library); } namespace utils { diff --git a/examples/io/legateio/legateio.py b/examples/io/legateio/legateio.py index 5847237ef1..ea2a33efb4 100644 --- a/examples/io/legateio/legateio.py +++ b/examples/io/legateio/legateio.py @@ -19,7 +19,7 @@ from typing import Any, Optional import legate.core.types as ty -from legate.core import Array, Field, Rect, Store, get_legate_runtime +from legate.core import Array, Field, Rect, Store, get_machine from .library import user_context as context, user_lib @@ -245,7 +245,7 @@ def read_file_parallel( if parallelism is None: # the num_procs property returns the number of processors that # Legate will favor to launch tasks - parallelism = get_legate_runtime().num_procs + parallelism = len(get_machine()) # Create an 1D unbound store # diff --git a/examples/io/src/util.h b/examples/io/src/util.h index fa4f23eb87..86e6e1fa93 100644 --- a/examples/io/src/util.h +++ b/examples/io/src/util.h @@ -19,7 +19,7 @@ #include #include "core/data/store.h" -#include "core/runtime/context.h" +#include "core/task/task_context.h" namespace legateio { diff --git a/legate/core/_lib/context.pyx b/legate/core/_lib/context.pyx index 7e396c014e..cb16a1ef6c 100644 --- a/legate/core/_lib/context.pyx +++ b/legate/core/_lib/context.pyx @@ -27,20 +27,20 @@ cdef extern from "core/task/task_info.h" namespace "legate" nogil: bool has_variant(int) string name() -cdef extern from "core/runtime/context.h" namespace "legate" nogil: - cdef cppclass LibraryContext: +cdef extern from "core/runtime/detail/library.h" namespace "legate::detail" nogil: + cdef cppclass Library: unsigned int get_task_id(long long) unsigned int get_mapper_id() int get_reduction_op_id(long long) unsigned int get_projection_id(long long) unsigned int get_sharding_id(long long) - TaskInfo* find_task(long long) + const TaskInfo* find_task(long long) except+ -cdef extern from "core/runtime/runtime.h" namespace "legate" nogil: +cdef extern from "core/runtime/detail/runtime.h" namespace "legate::detail" nogil: cdef cppclass Runtime: @staticmethod Runtime* get_runtime() - LibraryContext* find_library(string, bool) + Library* find_library(string, bool) cdef class CppTaskInfo: @@ -67,7 +67,7 @@ cdef class CppTaskInfo: cdef class Context: - cdef LibraryContext* _context + cdef Library* _context def __cinit__(self, str library_name, bool can_fail=False): self._context = Runtime.get_runtime().find_library(library_name.encode(), can_fail) @@ -88,4 +88,7 @@ cdef class Context: return self._context.get_sharding_id(local_shard_id) def find_task(self, long long local_task_id) -> CppTaskInfo: - return CppTaskInfo.from_ptr(self._context.find_task(local_task_id)) + try: + return CppTaskInfo.from_ptr(self._context.find_task(local_task_id)) + except IndexError: + return CppTaskInfo.from_ptr(NULL) diff --git a/legate/core/_lib/types.pyx b/legate/core/_lib/types.pyx index f92805c894..2ea705963f 100644 --- a/legate/core/_lib/types.pyx +++ b/legate/core/_lib/types.pyx @@ -15,7 +15,7 @@ from libcpp cimport bool -from libcpp.memory cimport unique_ptr +from libcpp.memory cimport shared_ptr from libcpp.string cimport string from libcpp.utility cimport move from libcpp.vector cimport vector @@ -104,7 +104,7 @@ _NUMPY_DTYPES = { } -cdef extern from "core/type/type_info.h" namespace "legate" nogil: +cdef extern from "core/type/detail/type_info.h" namespace "legate::detail" nogil: cdef cppclass Type: ctypedef enum Code: pass @@ -113,34 +113,34 @@ cdef extern from "core/type/type_info.h" namespace "legate" nogil: unsigned int alignment() int uid() bool variable_size() - unique_ptr[Type] clone() string to_string() bool is_primitive() + void record_reduction_operator(int, int) except+ int find_reduction_operator(int) except+ cdef cppclass FixedArrayType(Type): unsigned int num_elements() - const Type& element_type() + shared_ptr[Type] element_type() cdef cppclass StructType(Type): unsigned int num_fields() - const Type& field_type(unsigned int) + shared_ptr[Type] field_type(unsigned int) bool aligned() - cdef unique_ptr[Type] primitive_type(int code) + cdef shared_ptr[Type] primitive_type(int code) - cdef unique_ptr[Type] string_type() + cdef shared_ptr[Type] string_type() - cdef unique_ptr[Type] fixed_array_type( - unique_ptr[Type] element_type, unsigned int N + cdef shared_ptr[Type] fixed_array_type( + shared_ptr[Type] element_type, unsigned int N ) except+ - cdef unique_ptr[Type] struct_type( - vector[unique_ptr[Type]] field_types, bool + cdef shared_ptr[Type] struct_type( + vector[shared_ptr[Type]] field_types, bool ) except+ -cdef Dtype from_ptr(unique_ptr[Type] ty): +cdef Dtype from_ptr(shared_ptr[Type] ty): cdef Dtype dtype if ty.get().code == FIXED_ARRAY: dtype = FixedArrayDtype.__new__(FixedArrayDtype) @@ -153,33 +153,33 @@ cdef Dtype from_ptr(unique_ptr[Type] ty): cdef class Dtype: - cdef unique_ptr[Type] _type + cdef shared_ptr[Type] _type @staticmethod def primitive_type(int code) -> Dtype: - return from_ptr(move(primitive_type( code))) + return from_ptr(primitive_type( code)) @staticmethod def string_type() -> Dtype: - return from_ptr(move(string_type())) + return from_ptr(string_type()) @staticmethod def fixed_array_type( Dtype element_type, unsigned N ) -> FixedArrayDtype: return from_ptr( - move(fixed_array_type(element_type._type.get().clone(), N)) + fixed_array_type(element_type._type, N) ) @staticmethod def struct_type(list field_types, bool align) -> StructDtype: - cdef vector[unique_ptr[Type]] types + cdef vector[shared_ptr[Type]] types for field_type in field_types: types.push_back( - move(( field_type)._type.get().clone()) + ( field_type)._type ) return from_ptr( - move(struct_type(move(types), align)) + struct_type(move(types), align) ) @property @@ -206,6 +206,9 @@ cdef class Dtype: def is_primitive(self) -> bool: return self._type.get().is_primitive() + def record_reduction_op(self, int op_kind, int reduction_op_id) -> None: + self._type.get().record_reduction_operator(op_kind, reduction_op_id) + def reduction_op_id(self, int op_kind) -> int: return self._type.get().find_reduction_operator(op_kind) @@ -235,7 +238,7 @@ cdef class FixedArrayDtype(Dtype): @property def element_type(self) -> Dtype: cdef FixedArrayType* ty = self._type.get() - return from_ptr(move(ty.element_type().clone())) + return from_ptr(ty.element_type()) def to_numpy_dtype(self): arr_type = ( @@ -259,7 +262,7 @@ cdef class StructDtype(Dtype): def field_type(self, int field_idx) -> Dtype: cdef StructType* ty = self._type.get() - return from_ptr(move(ty.field_type(field_idx).clone())) + return from_ptr(ty.field_type(field_idx)) def aligned(self) -> bool: cdef StructType* ty = self._type.get() diff --git a/legate/core/types.pyi b/legate/core/types.pyi index cf0b37535c..9a44c9b366 100644 --- a/legate/core/types.pyi +++ b/legate/core/types.pyi @@ -45,6 +45,9 @@ class Dtype: def variable_size(self) -> bool: ... @property def is_primitive(self) -> bool: ... + def record_reduction_op( + self, op_kind: int, reduction_op_id: int + ) -> None: ... def reduction_op_id(self, op_kind: int) -> int: ... def __repr__(self) -> str: ... def to_numpy_dtype(self) -> np.dtype[Any]: ... diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index 0a8b063adf..46871264a9 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -207,17 +207,23 @@ list(APPEND legate_core_SOURCES src/core/data/scalar.cc src/core/data/shape.cc src/core/data/store.cc - src/core/data/transform.cc src/core/data/detail/logical_region_field.cc src/core/data/detail/logical_store.cc - src/core/mapping/base_mapper.cc - src/core/mapping/core_mapper.cc - src/core/mapping/default_mapper.cc - src/core/mapping/instance_manager.cc + src/core/data/detail/scalar.cc + src/core/data/detail/store.cc + src/core/data/detail/transform.cc src/core/mapping/machine.cc src/core/mapping/mapping.cc src/core/mapping/operation.cc src/core/mapping/store.cc + src/core/mapping/detail/base_mapper.cc + src/core/mapping/detail/core_mapper.cc + src/core/mapping/detail/default_mapper.cc + src/core/mapping/detail/instance_manager.cc + src/core/mapping/detail/machine.cc + src/core/mapping/detail/mapping.cc + src/core/mapping/detail/operation.cc + src/core/mapping/detail/store.cc src/core/operation/task.cc src/core/operation/detail/copy.cc src/core/operation/detail/copy_launcher.cc @@ -233,33 +239,38 @@ list(APPEND legate_core_SOURCES src/core/operation/detail/task.cc src/core/operation/detail/task_launcher.cc src/core/partitioning/constraint.cc - src/core/partitioning/constraint_solver.cc src/core/partitioning/partition.cc - src/core/partitioning/partitioner.cc src/core/partitioning/restriction.cc - src/core/runtime/context.cc - src/core/runtime/projection.cc + src/core/partitioning/detail/constraint.cc + src/core/partitioning/detail/constraint_solver.cc + src/core/partitioning/detail/partitioner.cc + src/core/runtime/library.cc src/core/runtime/runtime.cc src/core/runtime/tracker.cc - src/core/runtime/shard.cc src/core/runtime/detail/communicator_manager.cc src/core/runtime/detail/field_manager.cc + src/core/runtime/detail/library.cc src/core/runtime/detail/machine_manager.cc src/core/runtime/detail/partition_manager.cc + src/core/runtime/detail/projection.cc src/core/runtime/detail/provenance_manager.cc src/core/runtime/detail/region_manager.cc src/core/runtime/detail/runtime.cc + src/core/runtime/detail/shard.cc src/core/task/registrar.cc - src/core/task/return.cc src/core/task/task.cc + src/core/task/task_context.cc src/core/task/task_info.cc src/core/task/variant_options.cc + src/core/task/detail/return.cc + src/core/task/detail/task_context.cc src/core/type/type_info.cc - src/core/utilities/buffer_builder.cc + src/core/type/detail/type_info.cc src/core/utilities/debug.cc src/core/utilities/deserializer.cc src/core/utilities/machine.cc src/core/utilities/linearize.cc + src/core/utilities/detail/buffer_builder.cc ) if(Legion_NETWORKS) @@ -365,6 +376,7 @@ if (legate_core_BUILD_DOCS) src/core/type/type_traits.h # task src/core/task/task.h + src/core/task/task_context.h src/core/task/registrar.h src/core/task/variant_options.h src/core/task/exception.h @@ -377,9 +389,9 @@ if (legate_core_BUILD_DOCS) src/core/data/allocator.h src/core/data/logical_store.h # runtime + src/core/runtime/library.h src/core/runtime/runtime.h src/core/runtime/runtime.inl - src/core/runtime/context.h # operation src/core/operation/task.h # partitioning @@ -388,6 +400,7 @@ if (legate_core_BUILD_DOCS) src/core/mapping/machine.h src/core/mapping/mapping.h src/core/mapping/operation.h + src/core/mapping/store.h # aliases src/core/utilities/typedefs.h # utilities @@ -459,14 +472,12 @@ install( src/core/data/slice.h src/core/data/store.h src/core/data/store.inl - src/core/data/transform.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/legate/core/data) install( FILES src/core/mapping/machine.h src/core/mapping/mapping.h src/core/mapping/operation.h - src/core/mapping/operation.inl src/core/mapping/store.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/legate/core/mapping) @@ -476,15 +487,12 @@ install( install( FILES src/core/partitioning/constraint.h - src/core/partitioning/partition.h - src/core/partitioning/restriction.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/legate/core/partitioning) install( - FILES src/core/runtime/context.h - src/core/runtime/context.inl + FILES src/core/runtime/library.h + src/core/runtime/library.inl src/core/runtime/resource.h - src/core/runtime/projection.h src/core/runtime/runtime.h src/core/runtime/runtime.inl src/core/runtime/tracker.h @@ -493,9 +501,9 @@ install( install( FILES src/core/task/exception.h src/core/task/registrar.h - src/core/task/return.h src/core/task/task.h src/core/task/task.inl + src/core/task/task_context.h src/core/task/task_info.h src/core/task/variant_helper.h src/core/task/variant_options.h @@ -506,11 +514,7 @@ install( src/core/type/type_traits.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/legate/core/type) install( - FILES src/core/utilities/buffer_builder.h - src/core/utilities/buffer_builder.inl - src/core/utilities/debug.h - src/core/utilities/deserializer.h - src/core/utilities/deserializer.inl + FILES src/core/utilities/debug.h src/core/utilities/dispatch.h src/core/utilities/machine.h src/core/utilities/nvtx_help.h diff --git a/src/core/comm/coll.cc b/src/core/comm/coll.cc index 9021426822..035ba70a7d 100644 --- a/src/core/comm/coll.cc +++ b/src/core/comm/coll.cc @@ -23,13 +23,8 @@ #include #include -#ifndef LEGATE_USE_NETWORK -#include -#endif - #include "coll.h" -#include "legate.h" -#include "legion.h" +#include "core/utilities/typedefs.h" namespace legate::comm::coll { diff --git a/src/core/comm/comm.cc b/src/core/comm/comm.cc index c34ba3a49a..9507d8afe6 100644 --- a/src/core/comm/comm.cc +++ b/src/core/comm/comm.cc @@ -19,30 +19,27 @@ #include "core/comm/comm_nccl.h" #endif #include "core/comm/comm_cpu.h" +#include "core/runtime/runtime.h" #include "env_defaults.h" -#include "core/runtime/detail/communicator_manager.h" - namespace legate::comm { -void register_tasks(Legion::Machine machine, - Legion::Runtime* runtime, - const LibraryContext* context) +void register_tasks(Legion::Runtime* runtime, const detail::Library* library) { #ifdef LEGATE_USE_CUDA - nccl::register_tasks(machine, runtime, context); + nccl::register_tasks(runtime, library); #endif bool disable_mpi = static_cast(extract_env("LEGATE_DISABLE_MPI", DISABLE_MPI_DEFAULT, DISABLE_MPI_TEST)); - if (!disable_mpi) { cpu::register_tasks(machine, runtime, context); } + if (!disable_mpi) { cpu::register_tasks(runtime, library); } } -void register_builtin_communicator_factories(const LibraryContext* context) +void register_builtin_communicator_factories(const detail::Library* library) { #ifdef LEGATE_USE_CUDA - nccl::register_factory(context); + nccl::register_factory(library); #endif - cpu::register_factory(context); + cpu::register_factory(library); } } // namespace legate::comm diff --git a/src/core/comm/comm.h b/src/core/comm/comm.h index fe5c9e6dce..9d5317db54 100644 --- a/src/core/comm/comm.h +++ b/src/core/comm/comm.h @@ -16,18 +16,16 @@ #pragma once -#include "legate.h" +#include "legion.h" -namespace legate { -class LibraryContext; -} // namespace legate +namespace legate::detail { +class Library; +} // namespace legate::detail namespace legate::comm { -void register_tasks(Legion::Machine machine, - Legion::Runtime* runtime, - const LibraryContext* context); +void register_tasks(Legion::Runtime* runtime, const detail::Library* library); -void register_builtin_communicator_factories(const LibraryContext* context); +void register_builtin_communicator_factories(const detail::Library* library); } // namespace legate::comm diff --git a/src/core/comm/comm_cpu.cc b/src/core/comm/comm_cpu.cc index ee822e6b61..d7860e9104 100644 --- a/src/core/comm/comm_cpu.cc +++ b/src/core/comm/comm_cpu.cc @@ -17,8 +17,9 @@ #include "core/comm/comm_cpu.h" #include "core/operation/detail/task_launcher.h" #include "core/runtime/detail/communicator_manager.h" +#include "core/runtime/detail/library.h" #include "core/runtime/detail/runtime.h" -#include "legate.h" +#include "core/runtime/runtime.h" #include "core/comm/coll.h" @@ -26,30 +27,31 @@ namespace legate::comm::cpu { class Factory : public detail::CommunicatorFactory { public: - Factory(const LibraryContext* core_context); + Factory(const detail::Library* core_library); public: bool needs_barrier() const override { return false; } bool is_supported_target(mapping::TaskTarget target) const override; protected: - Legion::FutureMap initialize(const mapping::MachineDesc& machine, uint32_t num_tasks) override; - void finalize(const mapping::MachineDesc& machine, + Legion::FutureMap initialize(const mapping::detail::Machine& machine, + uint32_t num_tasks) override; + void finalize(const mapping::detail::Machine& machine, uint32_t num_tasks, const Legion::FutureMap& communicator) override; private: - const LibraryContext* core_context_; + const detail::Library* core_library_; }; -Factory::Factory(const LibraryContext* core_context) : core_context_(core_context) {} +Factory::Factory(const detail::Library* core_library) : core_library_(core_library) {} bool Factory::is_supported_target(mapping::TaskTarget target) const { return target == mapping::TaskTarget::OMP || target == mapping::TaskTarget::CPU; } -Legion::FutureMap Factory::initialize(const mapping::MachineDesc& machine, uint32_t num_tasks) +Legion::FutureMap Factory::initialize(const mapping::detail::Machine& machine, uint32_t num_tasks) { Domain launch_domain(Rect<1>(Point<1>(0), Point<1>(static_cast(num_tasks) - 1))); auto tag = @@ -60,13 +62,13 @@ Legion::FutureMap Factory::initialize(const mapping::MachineDesc& machine, uint3 // Find a mapping of all participants detail::TaskLauncher init_cpucoll_mapping_launcher( - core_context_, machine, LEGATE_CORE_INIT_CPUCOLL_MAPPING_TASK_ID, tag); + core_library_, machine, LEGATE_CORE_INIT_CPUCOLL_MAPPING_TASK_ID, tag); init_cpucoll_mapping_launcher.add_future(comm_id); auto mapping = init_cpucoll_mapping_launcher.execute(launch_domain); // Then create communicators on participating processors detail::TaskLauncher init_cpucoll_launcher( - core_context_, machine, LEGATE_CORE_INIT_CPUCOLL_TASK_ID, tag); + core_library_, machine, LEGATE_CORE_INIT_CPUCOLL_TASK_ID, tag); init_cpucoll_launcher.add_future(comm_id); init_cpucoll_launcher.set_concurrent(true); @@ -76,14 +78,14 @@ Legion::FutureMap Factory::initialize(const mapping::MachineDesc& machine, uint3 return init_cpucoll_launcher.execute(launch_domain); } -void Factory::finalize(const mapping::MachineDesc& machine, +void Factory::finalize(const mapping::detail::Machine& machine, uint32_t num_tasks, const Legion::FutureMap& communicator) { auto tag = machine.preferred_target == mapping::TaskTarget::OMP ? LEGATE_OMP_VARIANT : LEGATE_CPU_VARIANT; Domain launch_domain(Rect<1>(Point<1>(0), Point<1>(static_cast(num_tasks) - 1))); - detail::TaskLauncher launcher(core_context_, machine, LEGATE_CORE_FINALIZE_CPUCOLL_TASK_ID, tag); + detail::TaskLauncher launcher(core_library_, machine, LEGATE_CORE_FINALIZE_CPUCOLL_TASK_ID, tag); launcher.set_concurrent(true); launcher.add_future_map(communicator); launcher.execute(launch_domain); @@ -155,27 +157,25 @@ static void finalize_cpucoll(const Legion::Task* task, comm = nullptr; } -void register_tasks(Legion::Machine machine, - Legion::Runtime* runtime, - const LibraryContext* context) +void register_tasks(Legion::Runtime* runtime, const detail::Library* core_library) { const auto& command_args = Legion::Runtime::get_input_args(); coll::collInit(command_args.argc, command_args.argv); auto init_cpucoll_mapping_task_id = - context->get_task_id(LEGATE_CORE_INIT_CPUCOLL_MAPPING_TASK_ID); + core_library->get_task_id(LEGATE_CORE_INIT_CPUCOLL_MAPPING_TASK_ID); const char* init_cpucoll_mapping_task_name = "core::comm::cpu::init_mapping"; runtime->attach_name(init_cpucoll_mapping_task_id, init_cpucoll_mapping_task_name, false /*mutable*/, true /*local only*/); - auto init_cpucoll_task_id = context->get_task_id(LEGATE_CORE_INIT_CPUCOLL_TASK_ID); + auto init_cpucoll_task_id = core_library->get_task_id(LEGATE_CORE_INIT_CPUCOLL_TASK_ID); const char* init_cpucoll_task_name = "core::comm::cpu::init"; runtime->attach_name( init_cpucoll_task_id, init_cpucoll_task_name, false /*mutable*/, true /*local only*/); - auto finalize_cpucoll_task_id = context->get_task_id(LEGATE_CORE_FINALIZE_CPUCOLL_TASK_ID); + auto finalize_cpucoll_task_id = core_library->get_task_id(LEGATE_CORE_FINALIZE_CPUCOLL_TASK_ID); const char* finalize_cpucoll_task_name = "core::comm::cpu::finalize"; runtime->attach_name( finalize_cpucoll_task_id, finalize_cpucoll_task_name, false /*mutable*/, true /*local only*/); @@ -221,10 +221,10 @@ void register_tasks(Legion::Machine machine, } } -void register_factory(const LibraryContext* context) +void register_factory(const detail::Library* library) { auto* comm_mgr = detail::Runtime::get_runtime()->communicator_manager(); - comm_mgr->register_factory("cpu", std::make_unique(context)); + comm_mgr->register_factory("cpu", std::make_unique(library)); } } // namespace legate::comm::cpu diff --git a/src/core/comm/comm_cpu.h b/src/core/comm/comm_cpu.h index ee2e733d02..b80265eef3 100644 --- a/src/core/comm/comm_cpu.h +++ b/src/core/comm/comm_cpu.h @@ -18,16 +18,14 @@ #include "legion.h" -namespace legate { -class LibraryContext; -} // namespace legate +namespace legate::detail { +class Library; +} // namespace legate::detail namespace legate::comm::cpu { -void register_tasks(Legion::Machine machine, - Legion::Runtime* runtime, - const LibraryContext* context); +void register_tasks(Legion::Runtime* runtime, const detail::Library* library); -void register_factory(const LibraryContext* context); +void register_factory(const detail::Library* library); } // namespace legate::comm::cpu diff --git a/src/core/comm/comm_nccl.cu b/src/core/comm/comm_nccl.cu index ccc4179754..50fe7fadfc 100644 --- a/src/core/comm/comm_nccl.cu +++ b/src/core/comm/comm_nccl.cu @@ -20,10 +20,11 @@ #include "core/data/buffer.h" #include "core/operation/detail/task_launcher.h" #include "core/runtime/detail/communicator_manager.h" +#include "core/runtime/detail/library.h" #include "core/runtime/detail/runtime.h" +#include "core/runtime/runtime.h" #include "core/utilities/nvtx_help.h" #include "core/utilities/typedefs.h" -#include "legate.h" #include #include @@ -55,23 +56,24 @@ inline void check_nccl(ncclResult_t error, const char* file, int line) class Factory : public detail::CommunicatorFactory { public: - Factory(const LibraryContext* core_context); + Factory(const detail::Library* core_library); public: bool needs_barrier() const override; bool is_supported_target(mapping::TaskTarget target) const override; protected: - Legion::FutureMap initialize(const mapping::MachineDesc& machine, uint32_t num_tasks) override; - void finalize(const mapping::MachineDesc& machine, + Legion::FutureMap initialize(const mapping::detail::Machine& machine, + uint32_t num_tasks) override; + void finalize(const mapping::detail::Machine& machine, uint32_t num_tasks, const Legion::FutureMap& communicator) override; private: - const LibraryContext* core_context_; + const detail::Library* core_library_; }; -Factory::Factory(const LibraryContext* core_context) : core_context_(core_context) {} +Factory::Factory(const detail::Library* core_library) : core_library_(core_library) {} bool Factory::needs_barrier() const { return legate::comm::nccl::needs_barrier(); } @@ -80,32 +82,32 @@ bool Factory::is_supported_target(mapping::TaskTarget target) const return target == mapping::TaskTarget::GPU; } -Legion::FutureMap Factory::initialize(const mapping::MachineDesc& machine, uint32_t num_tasks) +Legion::FutureMap Factory::initialize(const mapping::detail::Machine& machine, uint32_t num_tasks) { Domain launch_domain(Rect<1>(Point<1>(0), Point<1>(static_cast(num_tasks) - 1))); // Create a communicator ID detail::TaskLauncher init_nccl_id_launcher( - core_context_, machine, LEGATE_CORE_INIT_NCCL_ID_TASK_ID, LEGATE_GPU_VARIANT); + core_library_, machine, LEGATE_CORE_INIT_NCCL_ID_TASK_ID, LEGATE_GPU_VARIANT); init_nccl_id_launcher.set_side_effect(true); auto nccl_id = init_nccl_id_launcher.execute_single(); // Then create the communicators on participating GPUs detail::TaskLauncher init_nccl_launcher( - core_context_, machine, LEGATE_CORE_INIT_NCCL_TASK_ID, LEGATE_GPU_VARIANT); + core_library_, machine, LEGATE_CORE_INIT_NCCL_TASK_ID, LEGATE_GPU_VARIANT); init_nccl_launcher.add_future(nccl_id); init_nccl_launcher.set_concurrent(true); return init_nccl_launcher.execute(launch_domain); } -void Factory::finalize(const mapping::MachineDesc& machine, +void Factory::finalize(const mapping::detail::Machine& machine, uint32_t num_tasks, const Legion::FutureMap& communicator) { Domain launch_domain(Rect<1>(Point<1>(0), Point<1>(static_cast(num_tasks) - 1))); detail::TaskLauncher launcher( - core_context_, machine, LEGATE_CORE_FINALIZE_NCCL_TASK_ID, LEGATE_GPU_VARIANT); + core_library_, machine, LEGATE_CORE_FINALIZE_NCCL_TASK_ID, LEGATE_GPU_VARIANT); launcher.set_concurrent(true); launcher.add_future_map(communicator); launcher.execute(launch_domain); @@ -181,21 +183,19 @@ static void finalize_nccl(const Legion::Task* task, delete comm; } -void register_tasks(Legion::Machine machine, - Legion::Runtime* runtime, - const LibraryContext* context) +void register_tasks(Legion::Runtime* runtime, const detail::Library* core_library) { - auto init_nccl_id_task_id = context->get_task_id(LEGATE_CORE_INIT_NCCL_ID_TASK_ID); + auto init_nccl_id_task_id = core_library->get_task_id(LEGATE_CORE_INIT_NCCL_ID_TASK_ID); const char* init_nccl_id_task_name = "core::comm::nccl::init_id"; runtime->attach_name( init_nccl_id_task_id, init_nccl_id_task_name, false /*mutable*/, true /*local only*/); - auto init_nccl_task_id = context->get_task_id(LEGATE_CORE_INIT_NCCL_TASK_ID); + auto init_nccl_task_id = core_library->get_task_id(LEGATE_CORE_INIT_NCCL_TASK_ID); const char* init_nccl_task_name = "core::comm::nccl::init"; runtime->attach_name( init_nccl_task_id, init_nccl_task_name, false /*mutable*/, true /*local only*/); - auto finalize_nccl_task_id = context->get_task_id(LEGATE_CORE_FINALIZE_NCCL_TASK_ID); + auto finalize_nccl_task_id = core_library->get_task_id(LEGATE_CORE_FINALIZE_NCCL_TASK_ID); const char* finalize_nccl_task_name = "core::comm::nccl::finalize"; runtime->attach_name( finalize_nccl_task_id, finalize_nccl_task_name, false /*mutable*/, true /*local only*/); @@ -233,10 +233,10 @@ bool needs_barrier() return true; } -void register_factory(const LibraryContext* context) +void register_factory(const detail::Library* core_library) { auto* comm_mgr = detail::Runtime::get_runtime()->communicator_manager(); - comm_mgr->register_factory("nccl", std::make_unique(context)); + comm_mgr->register_factory("nccl", std::make_unique(core_library)); } } // namespace legate::comm::nccl diff --git a/src/core/comm/comm_nccl.h b/src/core/comm/comm_nccl.h index a879e21b84..dca3e0cef4 100644 --- a/src/core/comm/comm_nccl.h +++ b/src/core/comm/comm_nccl.h @@ -18,18 +18,16 @@ #include "legion.h" -namespace legate { -class LibraryContext; -} // namespace legate +namespace legate::detail { +class Library; +} // namespace legate::detail namespace legate::comm::nccl { -void register_tasks(Legion::Machine machine, - Legion::Runtime* runtime, - const LibraryContext* context); +void register_tasks(Legion::Runtime* runtime, const detail::Library* core_library); bool needs_barrier(); -void register_factory(const LibraryContext* context); +void register_factory(const detail::Library* core_library); } // namespace legate::comm::nccl diff --git a/src/core/comm/local_comm.cc b/src/core/comm/local_comm.cc index 3d301dc357..e38d9e6eb4 100644 --- a/src/core/comm/local_comm.cc +++ b/src/core/comm/local_comm.cc @@ -20,8 +20,7 @@ #include #include "coll.h" -#include "legate.h" -#include "legion.h" +#include "core/utilities/typedefs.h" namespace legate::comm::coll { diff --git a/src/core/comm/mpi_comm.cc b/src/core/comm/mpi_comm.cc index c9356e6dff..e24ace6565 100644 --- a/src/core/comm/mpi_comm.cc +++ b/src/core/comm/mpi_comm.cc @@ -20,7 +20,7 @@ #include #include "coll.h" -#include "legate.h" +#include "legate_defines.h" #include "legion.h" namespace legate::comm::coll { diff --git a/src/core/data/allocator.cc b/src/core/data/allocator.cc index 7f45120641..ee0a269556 100644 --- a/src/core/data/allocator.cc +++ b/src/core/data/allocator.cc @@ -1,4 +1,4 @@ -/* Copyright 2022 NVIDIA Corporation +/* Copyright 2023 NVIDIA Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,16 +15,36 @@ */ #include "core/data/allocator.h" -#include "legate.h" +#include "core/data/buffer.h" +#include "core/utilities/typedefs.h" namespace legate { -ScopedAllocator::ScopedAllocator(Memory::Kind kind, bool scoped, size_t alignment) +class ScopedAllocator::Impl { + public: + using ByteBuffer = Buffer; + + public: + Impl(Memory::Kind kind, bool scoped, size_t alignment); + ~Impl(); + + public: + void* allocate(size_t bytes); + void deallocate(void* ptr); + + private: + Memory::Kind target_kind_{Memory::Kind::SYSTEM_MEM}; + bool scoped_; + size_t alignment_; + std::unordered_map buffers_{}; +}; + +ScopedAllocator::Impl::Impl(Memory::Kind kind, bool scoped, size_t alignment) : target_kind_(kind), scoped_(scoped), alignment_(alignment) { } -ScopedAllocator::~ScopedAllocator() +ScopedAllocator::Impl::~Impl() { if (scoped_) { for (auto& pair : buffers_) { pair.second.destroy(); } @@ -32,7 +52,7 @@ ScopedAllocator::~ScopedAllocator() } } -void* ScopedAllocator::allocate(size_t bytes) +void* ScopedAllocator::Impl::allocate(size_t bytes) { if (bytes == 0) return nullptr; @@ -44,19 +64,38 @@ void* ScopedAllocator::allocate(size_t bytes) return ptr; } -void ScopedAllocator::deallocate(void* ptr) +void ScopedAllocator::Impl::deallocate(void* ptr) { ByteBuffer buffer; auto finder = buffers_.find(ptr); - assert(finder != buffers_.end()); - if (finder == buffers_.end()) { - legate::log_legate.error("Invalid address during deallocation"); - LEGATE_ABORT; - } + if (finder == buffers_.end()) { throw std::runtime_error("Invalid address for deallocation"); } buffer = finder->second; buffers_.erase(finder); buffer.destroy(); } +ScopedAllocator::ScopedAllocator(Memory::Kind kind, bool scoped, size_t alignment) + : impl_(new Impl(kind, scoped, alignment)) +{ +} + +ScopedAllocator::~ScopedAllocator() { delete impl_; } + +void* ScopedAllocator::allocate(size_t bytes) { return impl_->allocate(bytes); } + +void ScopedAllocator::deallocate(void* ptr) { impl_->deallocate(ptr); } + +ScopedAllocator::ScopedAllocator(ScopedAllocator&& other) : impl_(other.impl_) +{ + other.impl_ = nullptr; +} + +ScopedAllocator& ScopedAllocator::operator=(ScopedAllocator&& other) +{ + impl_ = other.impl_; + other.impl_ = nullptr; + return *this; +} + } // namespace legate diff --git a/src/core/data/allocator.h b/src/core/data/allocator.h index f9c80a64fc..ec9143e599 100644 --- a/src/core/data/allocator.h +++ b/src/core/data/allocator.h @@ -1,4 +1,4 @@ -/* Copyright 2022 NVIDIA Corporation +/* Copyright 2023 NVIDIA Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,9 +16,7 @@ #pragma once -#include "core/data/buffer.h" - -#include +#include "core/utilities/typedefs.h" /** * @file @@ -38,11 +36,6 @@ namespace legate { */ class ScopedAllocator { public: - using ByteBuffer = Buffer; - - public: - ScopedAllocator() = default; - // Iff 'scoped', all allocations will be released upon destruction. // Otherwise this is up to the runtime after the task has finished. /** @@ -77,11 +70,17 @@ class ScopedAllocator { */ void deallocate(void* ptr); + public: + ScopedAllocator(ScopedAllocator&&); + ScopedAllocator& operator=(ScopedAllocator&&); + + private: + ScopedAllocator(const ScopedAllocator&) = delete; + ScopedAllocator& operator=(const ScopedAllocator&) = delete; + private: - Memory::Kind target_kind_{Memory::Kind::SYSTEM_MEM}; - bool scoped_; - size_t alignment_; - std::unordered_map buffers_{}; + class Impl; + Impl* impl_; }; } // namespace legate diff --git a/src/core/data/detail/logical_store.cc b/src/core/data/detail/logical_store.cc index 341a89897c..d063dd03cd 100644 --- a/src/core/data/detail/logical_store.cc +++ b/src/core/data/detail/logical_store.cc @@ -16,12 +16,13 @@ #include "core/data/detail/logical_store.h" -#include "core/mapping/machine.h" +#include "core/data/detail/store.h" +#include "core/data/detail/transform.h" #include "core/operation/detail/projection.h" #include "core/runtime/detail/partition_manager.h" #include "core/runtime/detail/runtime.h" +#include "core/type/detail/type_info.h" #include "core/type/type_traits.h" -#include "core/utilities/buffer_builder.h" #include "core/utilities/dispatch.h" #include "legate_defines.h" @@ -35,7 +36,7 @@ namespace legate::detail { // legate::detail::Storage //////////////////////////////////////////////////// -Storage::Storage(int32_t dim, std::unique_ptr type) +Storage::Storage(int32_t dim, std::shared_ptr type) : storage_id_(Runtime::get_runtime()->get_unique_storage_id()), unbound_(true), dim_(dim), @@ -47,7 +48,7 @@ Storage::Storage(int32_t dim, std::unique_ptr type) #endif } -Storage::Storage(const Shape& extents, std::unique_ptr type, bool optimize_scalar) +Storage::Storage(const Shape& extents, std::shared_ptr type, bool optimize_scalar) : storage_id_(Runtime::get_runtime()->get_unique_storage_id()), dim_(extents.size()), extents_(extents), @@ -61,7 +62,7 @@ Storage::Storage(const Shape& extents, std::unique_ptr type, bool optimize #endif } -Storage::Storage(const Shape& extents, std::unique_ptr type, const Legion::Future& future) +Storage::Storage(const Shape& extents, std::shared_ptr type, const Legion::Future& future) : storage_id_(Runtime::get_runtime()->get_unique_storage_id()), dim_(extents.size()), extents_(extents), @@ -77,7 +78,7 @@ Storage::Storage(const Shape& extents, std::unique_ptr type, const Legion: } Storage::Storage(Shape&& extents, - std::unique_ptr type, + std::shared_ptr type, std::shared_ptr parent, Shape&& color, Shape&& offsets) @@ -207,7 +208,7 @@ Restrictions Storage::compute_restrictions() const return Restrictions(dim_, Restriction::ALLOW); } -Partition* Storage::find_key_partition(const mapping::MachineDesc& machine, +Partition* Storage::find_key_partition(const mapping::detail::Machine& machine, const Restrictions& restrictions) const { uint32_t new_num_pieces = machine.count(); @@ -220,7 +221,7 @@ Partition* Storage::find_key_partition(const mapping::MachineDesc& machine, return nullptr; } -void Storage::set_key_partition(const mapping::MachineDesc& machine, +void Storage::set_key_partition(const mapping::detail::Machine& machine, std::unique_ptr&& key_partition) { num_pieces_ = machine.count(); @@ -278,7 +279,7 @@ std::shared_ptr StoragePartition::get_child_storage(const Shape& color) auto child_extents = tiling->get_child_extents(parent_->extents(), color); auto child_offsets = tiling->get_child_offsets(color); return std::make_shared(std::move(child_extents), - parent_->type().clone(), + parent_->type(), shared_from_this(), Shape(color), std::move(child_offsets)); @@ -292,7 +293,7 @@ std::shared_ptr StoragePartition::get_child_data(const Shape return parent_->get_region_field()->get_child(tiling, color, complete_); } -Partition* StoragePartition::find_key_partition(const mapping::MachineDesc& machine, +Partition* StoragePartition::find_key_partition(const mapping::detail::Machine& machine, const Restrictions& restrictions) const { return parent_->find_key_partition(machine, restrictions); @@ -358,7 +359,7 @@ const Shape& LogicalStore::extents() const size_t LogicalStore::volume() const { return extents().volume(); } -size_t LogicalStore::storage_size() const { return storage_->volume() * type().size(); } +size_t LogicalStore::storage_size() const { return storage_->volume() * type()->size(); } int32_t LogicalStore::dim() const { @@ -367,7 +368,7 @@ int32_t LogicalStore::dim() const bool LogicalStore::has_scalar_storage() const { return storage_->kind() == Storage::Kind::FUTURE; } -const Type& LogicalStore::type() const { return storage_->type(); } +std::shared_ptr LogicalStore::type() const { return storage_->type(); } bool LogicalStore::transformed() const { return !transform_->identity(); } @@ -532,17 +533,17 @@ std::shared_ptr LogicalStore::get_physical_store() if (storage_->kind() == Storage::Kind::FUTURE) { // TODO: future wrappers from inline mappings are read-only for now auto domain = to_domain(storage_->extents()); - FutureWrapper future(true, type().size(), domain, storage_->get_future()); + FutureWrapper future(true, type()->size(), domain, storage_->get_future()); // Physical stores for future-backed stores shouldn't be cached, as they are not automatically // remapped to reflect changes by the runtime. - return std::make_shared(dim(), type().clone(), -1, future, transform_); + return std::make_shared(dim(), type(), -1, future, transform_); } #ifdef DEBUG_LEGATE assert(storage_->kind() == Storage::Kind::REGION_FIELD); #endif auto region_field = storage_->map(); - mapped_ = std::make_shared(dim(), type().clone(), -1, std::move(region_field), transform_); + mapped_ = std::make_shared(dim(), type(), -1, std::move(region_field), transform_); return mapped_; } @@ -570,7 +571,7 @@ Legion::ProjectionID LogicalStore::compute_projection(int32_t launch_ndim) const } std::shared_ptr LogicalStore::find_or_create_key_partition( - const mapping::MachineDesc& machine, const Restrictions& restrictions) + const mapping::detail::Machine& machine, const Restrictions& restrictions) { uint32_t new_num_pieces = machine.count(); if (num_pieces_ == new_num_pieces && key_partition_ != nullptr && @@ -607,7 +608,7 @@ std::shared_ptr LogicalStore::find_or_create_key_partition( return key_partition_; } -bool LogicalStore::has_key_partition(const mapping::MachineDesc& machine, +bool LogicalStore::has_key_partition(const mapping::detail::Machine& machine, const Restrictions& restrictions) const { uint32_t new_num_pieces = machine.count(); @@ -619,7 +620,7 @@ bool LogicalStore::has_key_partition(const mapping::MachineDesc& machine, storage_->find_key_partition(machine, transform_->invert(restrictions)) != nullptr; } -void LogicalStore::set_key_partition(const mapping::MachineDesc& machine, +void LogicalStore::set_key_partition(const mapping::detail::Machine& machine, const Partition* partition) { num_pieces_ = machine.count(); @@ -650,7 +651,7 @@ void LogicalStore::pack(BufferBuilder& buffer) const buffer.pack(has_scalar_storage()); buffer.pack(unbound()); buffer.pack(dim()); - type().pack(buffer); + type()->pack(buffer); transform_->pack(buffer); } diff --git a/src/core/data/detail/logical_store.h b/src/core/data/detail/logical_store.h index 7a738d658f..097b8fde95 100644 --- a/src/core/data/detail/logical_store.h +++ b/src/core/data/detail/logical_store.h @@ -19,19 +19,19 @@ #include #include "core/data/detail/logical_region_field.h" +#include "core/data/detail/store.h" #include "core/data/slice.h" #include "core/data/store.h" +#include "core/mapping/detail/machine.h" #include "core/partitioning/partition.h" #include "core/partitioning/restriction.h" - -namespace legate::mapping { -class MachineDesc; -} // namespace legate::mapping +#include "core/utilities/detail/buffer_builder.h" namespace legate::detail { class ProjectionInfo; class StoragePartition; +class Store; class LogicalStorePartition; class Storage : public std::enable_shared_from_this { @@ -43,14 +43,14 @@ class Storage : public std::enable_shared_from_this { public: // Create a RegionField-backed storage whose size is unbound. Initialized lazily. - Storage(int32_t dim, std::unique_ptr type); + Storage(int32_t dim, std::shared_ptr type); // Create a RegionField-backed or a Future-backed storage. Initialized lazily. - Storage(const Shape& extents, std::unique_ptr type, bool optimize_scalar); + Storage(const Shape& extents, std::shared_ptr type, bool optimize_scalar); // Create a Future-backed storage. Initialized eagerly. - Storage(const Shape& extents, std::unique_ptr type, const Legion::Future& future); + Storage(const Shape& extents, std::shared_ptr type, const Legion::Future& future); // Create a RegionField-bakced sub-storage. Initialized lazily. Storage(Shape&& extents, - std::unique_ptr type, + std::shared_ptr type, std::shared_ptr parent, Shape&& color, Shape&& offsets); @@ -61,7 +61,7 @@ class Storage : public std::enable_shared_from_this { const Shape& offsets() const; size_t volume() const; int32_t dim() { return dim_; } - const Type& type() const { return *type_; } + std::shared_ptr type() const { return type_; } Kind kind() const { return kind_; } int32_t level() const { return level_; } @@ -81,9 +81,9 @@ class Storage : public std::enable_shared_from_this { public: Restrictions compute_restrictions() const; - Partition* find_key_partition(const mapping::MachineDesc& machine, + Partition* find_key_partition(const mapping::detail::Machine& machine, const Restrictions& restrictions) const; - void set_key_partition(const mapping::MachineDesc& machine, + void set_key_partition(const mapping::detail::Machine& machine, std::unique_ptr&& key_partition); void reset_key_partition(); @@ -100,7 +100,7 @@ class Storage : public std::enable_shared_from_this { int32_t dim_{-1}; Shape extents_; size_t volume_; - std::unique_ptr type_{nullptr}; + std::shared_ptr type_{nullptr}; Kind kind_{Kind::REGION_FIELD}; private: @@ -134,7 +134,7 @@ class StoragePartition : public std::enable_shared_from_this { std::shared_ptr get_child_data(const Shape& color); public: - Partition* find_key_partition(const mapping::MachineDesc& machine, + Partition* find_key_partition(const mapping::detail::Machine& machine, const Restrictions& restrictions) const; Legion::LogicalPartition get_legion_partition(); @@ -180,7 +180,7 @@ class LogicalStore : public std::enable_shared_from_this { size_t storage_size() const; int32_t dim() const; bool has_scalar_storage() const; - const Type& type() const; + std::shared_ptr type() const; bool transformed() const; uint64_t id() const; @@ -206,11 +206,11 @@ class LogicalStore : public std::enable_shared_from_this { public: Restrictions compute_restrictions() const; - std::shared_ptr find_or_create_key_partition(const mapping::MachineDesc& machine, + std::shared_ptr find_or_create_key_partition(const mapping::detail::Machine& machine, const Restrictions& restrictions); - bool has_key_partition(const mapping::MachineDesc& machine, + bool has_key_partition(const mapping::detail::Machine& machine, const Restrictions& restrictions) const; - void set_key_partition(const mapping::MachineDesc& machine, const Partition* partition); + void set_key_partition(const mapping::detail::Machine& machine, const Partition* partition); void reset_key_partition(); public: diff --git a/src/core/data/detail/scalar.cc b/src/core/data/detail/scalar.cc new file mode 100644 index 0000000000..01fd4b5738 --- /dev/null +++ b/src/core/data/detail/scalar.cc @@ -0,0 +1,103 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/data/detail/scalar.h" +#include "core/type/detail/type_info.h" + +namespace legate::detail { + +Scalar::Scalar(std::shared_ptr type, const void* data, bool copy) + : own_(copy), type_(std::move(type)), data_(data) +{ + if (copy) { data_ = copy_data(data, size()); } +} + +Scalar::Scalar(const std::string& value) : own_(true), type_(string_type()) +{ + auto data_size = sizeof(char) * value.size(); + auto buffer = malloc(sizeof(uint32_t) + data_size); + *static_cast(buffer) = value.size(); + memcpy(static_cast(buffer) + sizeof(uint32_t), value.data(), data_size); + data_ = buffer; +} + +Scalar::~Scalar() +{ + if (own_) { + // We know we own this buffer + free(const_cast(data_)); + } +} + +Scalar::Scalar(const Scalar& other) : own_(other.own_), type_(other.type_) +{ + if (other.own_) { + data_ = copy_data(other.data_, other.size()); + } else { + data_ = other.data_; + } +} + +Scalar::Scalar(Scalar&& other) : own_(other.own_), type_(std::move(other.type_)), data_(other.data_) +{ + other.own_ = false; + other.data_ = nullptr; +} + +Scalar& Scalar::operator=(const Scalar& other) +{ + own_ = other.own_; + type_ = other.type_; + if (other.own_) { + data_ = copy_data(other.data_, other.size()); + } else { + data_ = other.data_; + } + return *this; +} + +Scalar& Scalar::operator=(Scalar&& other) +{ + own_ = other.own_; + type_ = std::move(other.type_); + data_ = other.data_; + other.own_ = false; + other.data_ = nullptr; + return *this; +} + +const void* Scalar::copy_data(const void* data, size_t size) +{ + auto buffer = malloc(size); + memcpy(buffer, data, size); + return buffer; +} + +size_t Scalar::size() const +{ + if (type_->code == Type::Code::STRING) + return *static_cast(data_) + sizeof(uint32_t); + else + return type_->size(); +} + +void Scalar::pack(BufferBuilder& buffer) const +{ + type_->pack(buffer); + buffer.pack_buffer(data_, size()); +} + +} // namespace legate::detail diff --git a/src/core/data/detail/scalar.h b/src/core/data/detail/scalar.h new file mode 100644 index 0000000000..904c45d2c3 --- /dev/null +++ b/src/core/data/detail/scalar.h @@ -0,0 +1,71 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include + +#include "core/type/detail/type_info.h" +#include "core/type/type_traits.h" +#include "core/utilities/detail/buffer_builder.h" + +namespace legate::detail { + +class Type; + +class Scalar { + public: + Scalar(std::shared_ptr type, const void* data, bool copy); + Scalar(const std::string& value); + ~Scalar(); + + public: + template + Scalar(T value) : own_(true), type_(detail::primitive_type(legate_type_code_of)) + { + static_assert(legate_type_code_of != Type::Code::FIXED_ARRAY); + static_assert(legate_type_code_of != Type::Code::STRUCT); + static_assert(legate_type_code_of != Type::Code::STRING); + static_assert(legate_type_code_of != Type::Code::INVALID); + data_ = copy_data(&value, sizeof(T)); + } + + public: + Scalar(const Scalar& other); + Scalar(Scalar&& other); + + public: + Scalar& operator=(const Scalar& other); + Scalar& operator=(Scalar&& other); + + private: + const void* copy_data(const void* data, size_t size); + + public: + std::shared_ptr type() const { return type_; } + const void* data() const { return data_; } + size_t size() const; + + public: + void pack(BufferBuilder& buffer) const; + + private: + bool own_; + std::shared_ptr type_; + const void* data_; +}; + +} // namespace legate::detail diff --git a/src/core/data/detail/store.cc b/src/core/data/detail/store.cc new file mode 100644 index 0000000000..7599d1892c --- /dev/null +++ b/src/core/data/detail/store.cc @@ -0,0 +1,453 @@ +/* Copyright 2021-2022 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/data/detail/store.h" + +#include "core/data/detail/transform.h" +#include "core/runtime/detail/runtime.h" +#include "core/utilities/dispatch.h" +#include "core/utilities/machine.h" + +#ifdef LEGATE_USE_CUDA +#include "core/cuda/cuda_help.h" +#include "core/cuda/stream_pool.h" +#endif + +namespace legate::detail { + +RegionField::RegionField(int32_t dim, const Legion::PhysicalRegion& pr, Legion::FieldID fid) + : dim_(dim), pr_(pr), fid_(fid) +{ + auto priv = pr.get_privilege(); + readable_ = static_cast(priv & LEGION_READ_PRIV); + writable_ = static_cast(priv & LEGION_WRITE_PRIV); + reducible_ = static_cast(priv & LEGION_REDUCE) || (readable_ && writable_); +} + +RegionField::RegionField(RegionField&& other) noexcept + : dim_(other.dim_), + pr_(other.pr_), + fid_(other.fid_), + readable_(other.readable_), + writable_(other.writable_), + reducible_(other.reducible_) +{ +} + +RegionField& RegionField::operator=(RegionField&& other) noexcept +{ + dim_ = other.dim_; + pr_ = other.pr_; + fid_ = other.fid_; + readable_ = other.readable_; + writable_ = other.writable_; + reducible_ = other.reducible_; + return *this; +} + +bool RegionField::valid() const +{ + return pr_.get_logical_region() != Legion::LogicalRegion::NO_REGION; +} + +namespace { + +struct get_domain_fn { + template + Domain operator()(const Legion::PhysicalRegion& pr) + { + return Domain(pr.get_bounds()); + } +}; + +} // namespace + +Domain RegionField::domain() const { return dim_dispatch(dim_, get_domain_fn{}, pr_); } + +void RegionField::unmap() { detail::Runtime::get_runtime()->unmap_physical_region(pr_); } + +UnboundRegionField::UnboundRegionField(const Legion::OutputRegion& out, Legion::FieldID fid) + : out_(out), + fid_(fid), + num_elements_( + Legion::UntypedDeferredValue(sizeof(size_t), find_memory_kind_for_executing_processor())) +{ +} + +UnboundRegionField::UnboundRegionField(UnboundRegionField&& other) noexcept + : bound_(other.bound_), out_(other.out_), fid_(other.fid_), num_elements_(other.num_elements_) +{ + other.bound_ = false; + other.out_ = Legion::OutputRegion(); + other.fid_ = -1; + other.num_elements_ = Legion::UntypedDeferredValue(); +} + +UnboundRegionField& UnboundRegionField::operator=(UnboundRegionField&& other) noexcept +{ + bound_ = other.bound_; + out_ = other.out_; + fid_ = other.fid_; + num_elements_ = other.num_elements_; + + other.bound_ = false; + other.out_ = Legion::OutputRegion(); + other.fid_ = -1; + other.num_elements_ = Legion::UntypedDeferredValue(); + + return *this; +} + +void UnboundRegionField::bind_empty_data(int32_t ndim) +{ + update_num_elements(0); + DomainPoint extents; + extents.dim = ndim; + for (int32_t dim = 0; dim < ndim; ++dim) extents[dim] = 0; + auto empty_buffer = create_buffer(0); + out_.return_data(extents, fid_, empty_buffer.get_instance(), false); + bound_ = true; +} + +ReturnValue UnboundRegionField::pack_weight() const +{ +#ifdef DEBUG_LEGATE + if (!bound_) { + legate::log_legate.error( + "Found an uninitialized unbound store. Please make sure you return buffers to all unbound " + "stores in the task"); + LEGATE_ABORT; + } +#endif + return ReturnValue(num_elements_, sizeof(size_t)); +} + +void UnboundRegionField::update_num_elements(size_t num_elements) +{ + AccessorWO acc(num_elements_, sizeof(size_t), false); + acc[0] = num_elements; +} + +FutureWrapper::FutureWrapper(bool read_only, + uint32_t field_size, + Domain domain, + Legion::Future future, + bool initialize /*= false*/) + : read_only_(read_only), field_size_(field_size), domain_(domain), future_(future) +{ + if (!read_only) { +#ifdef DEBUG_LEGATE + assert(!initialize || future_.get_untyped_size() == field_size); +#endif + auto mem_kind = find_memory_kind_for_executing_processor( +#ifdef LEGATE_NO_FUTURES_ON_FB + true +#else + false +#endif + ); + if (initialize) { + auto p_init_value = future_.get_buffer(mem_kind); +#ifdef LEGATE_USE_CUDA + if (mem_kind == Memory::Kind::GPU_FB_MEM) { + // TODO: This should be done by Legion + buffer_ = Legion::UntypedDeferredValue(field_size, mem_kind); + AccessorWO acc(buffer_, field_size, false); + auto stream = cuda::StreamPool::get_stream_pool().get_stream(); + CHECK_CUDA( + cudaMemcpyAsync(acc.ptr(0), p_init_value, field_size, cudaMemcpyDeviceToDevice, stream)); + } else +#endif + buffer_ = Legion::UntypedDeferredValue(field_size, mem_kind, p_init_value); + } else + buffer_ = Legion::UntypedDeferredValue(field_size, mem_kind); + } +} + +FutureWrapper::FutureWrapper(const FutureWrapper& other) noexcept + : read_only_(other.read_only_), + field_size_(other.field_size_), + domain_(other.domain_), + future_(other.future_), + buffer_(other.buffer_) +{ +} + +FutureWrapper& FutureWrapper::operator=(const FutureWrapper& other) noexcept +{ + read_only_ = other.read_only_; + field_size_ = other.field_size_; + domain_ = other.domain_; + future_ = other.future_; + buffer_ = other.buffer_; + return *this; +} + +Domain FutureWrapper::domain() const { return domain_; } + +void FutureWrapper::initialize_with_identity(int32_t redop_id) +{ + auto untyped_acc = AccessorWO(buffer_, field_size_); + auto ptr = untyped_acc.ptr(0); + + auto redop = Legion::Runtime::get_reduction_op(redop_id); +#ifdef DEBUG_LEGATE + assert(redop->sizeof_lhs == field_size_); +#endif + auto identity = redop->identity; +#ifdef LEGATE_USE_CUDA + if (buffer_.get_instance().get_location().kind() == Memory::Kind::GPU_FB_MEM) { + auto stream = cuda::StreamPool::get_stream_pool().get_stream(); + CHECK_CUDA(cudaMemcpyAsync(ptr, identity, field_size_, cudaMemcpyHostToDevice, stream)); + } else +#endif + memcpy(ptr, identity, field_size_); +} + +ReturnValue FutureWrapper::pack() const { return ReturnValue(buffer_, field_size_); } + +Store::Store(int32_t dim, + std::shared_ptr type, + int32_t redop_id, + FutureWrapper future, + std::shared_ptr&& transform) + : is_future_(true), + is_unbound_store_(false), + dim_(dim), + type_(std::move(type)), + redop_id_(redop_id), + future_(future), + transform_(std::move(transform)), + readable_(true) +{ +} + +Store::Store(int32_t dim, + std::shared_ptr type, + int32_t redop_id, + RegionField&& region_field, + std::shared_ptr&& transform) + : is_future_(false), + is_unbound_store_(false), + dim_(dim), + type_(std::move(type)), + redop_id_(redop_id), + region_field_(std::move(region_field)), + transform_(std::move(transform)) +{ + readable_ = region_field_.is_readable(); + writable_ = region_field_.is_writable(); + reducible_ = region_field_.is_reducible(); +} + +Store::Store(int32_t dim, + std::shared_ptr type, + UnboundRegionField&& unbound_field, + std::shared_ptr&& transform) + : is_future_(false), + is_unbound_store_(true), + dim_(dim), + type_(std::move(type)), + redop_id_(-1), + unbound_field_(std::move(unbound_field)), + transform_(std::move(transform)) +{ +} + +Store::Store(int32_t dim, + std::shared_ptr type, + int32_t redop_id, + FutureWrapper future, + const std::shared_ptr& transform) + : is_future_(true), + is_unbound_store_(false), + dim_(dim), + type_(std::move(type)), + redop_id_(redop_id), + future_(future), + transform_(transform), + readable_(true) +{ +} + +Store::Store(int32_t dim, + std::shared_ptr type, + int32_t redop_id, + RegionField&& region_field, + const std::shared_ptr& transform) + : is_future_(false), + is_unbound_store_(false), + dim_(dim), + type_(std::move(type)), + redop_id_(redop_id), + region_field_(std::move(region_field)), + transform_(transform) +{ + readable_ = region_field_.is_readable(); + writable_ = region_field_.is_writable(); + reducible_ = region_field_.is_reducible(); +} + +Store::Store(Store&& other) noexcept + : is_future_(other.is_future_), + is_unbound_store_(other.is_unbound_store_), + dim_(other.dim_), + type_(std::move(other.type_)), + redop_id_(other.redop_id_), + future_(other.future_), + region_field_(std::move(other.region_field_)), + unbound_field_(std::move(other.unbound_field_)), + transform_(std::move(other.transform_)), + readable_(other.readable_), + writable_(other.writable_), + reducible_(other.reducible_) +{ +} + +Store& Store::operator=(Store&& other) noexcept +{ + is_future_ = other.is_future_; + is_unbound_store_ = other.is_unbound_store_; + dim_ = other.dim_; + type_ = std::move(other.type_); + redop_id_ = other.redop_id_; + if (is_future_) + future_ = other.future_; + else if (is_unbound_store_) + unbound_field_ = std::move(other.unbound_field_); + else + region_field_ = std::move(other.region_field_); + transform_ = std::move(other.transform_); + readable_ = other.readable_; + writable_ = other.writable_; + reducible_ = other.reducible_; + return *this; +} + +bool Store::valid() const { return is_future_ || is_unbound_store_ || region_field_.valid(); } + +bool Store::transformed() const { return !transform_->identity(); } + +Domain Store::domain() const +{ + if (is_unbound_store_) + throw std::invalid_argument("Invalid to retrieve the domain of an unbound store"); + auto result = is_future_ ? future_.domain() : region_field_.domain(); + if (!transform_->identity()) result = transform_->transform(result); +#ifdef DEBUG_LEGATE + assert(result.dim == dim_ || dim_ == 0); +#endif + return result; +} + +void Store::unmap() +{ + if (is_future_ || is_unbound_store_) return; + region_field_.unmap(); +} + +void Store::bind_empty_data() +{ + check_valid_binding(true); + unbound_field_.bind_empty_data(dim_); +} + +bool Store::is_future() const { return is_future_; } + +bool Store::is_unbound_store() const { return is_unbound_store_; } + +void Store::check_accessor_dimension(const int32_t dim) const +{ + if (!(dim == dim_ || (dim_ == 0 && dim == 1))) { + throw std::invalid_argument("Dimension mismatch: invalid to create a " + std::to_string(dim) + + "-D accessor to a " + std::to_string(dim_) + "-D store"); + } +} + +void Store::check_buffer_dimension(const int32_t dim) const +{ + if (dim != dim_) { + throw std::invalid_argument("Dimension mismatch: invalid to bind a " + std::to_string(dim) + + "-D buffer to a " + std::to_string(dim_) + "-D store"); + } +} + +void Store::check_shape_dimension(const int32_t dim) const +{ + if (!(dim == dim_ || (dim_ == 0 && dim == 1))) { + throw std::invalid_argument("Dimension mismatch: invalid to retrieve a " + std::to_string(dim) + + "-D rect from a " + std::to_string(dim_) + "-D store"); + } +} + +void Store::check_valid_binding(bool bind_buffer) const +{ + if (!is_unbound_store_) { + throw std::invalid_argument("Buffer can be bound only to an unbound store"); + } + if (bind_buffer && unbound_field_.bound()) { + throw std::invalid_argument("A buffer has already been bound to the store"); + } +} + +Legion::DomainAffineTransform Store::get_inverse_transform() const +{ + return transform_->inverse_transform(dim_); +} + +bool Store::is_read_only_future() const { return future_.is_read_only(); } + +void Store::get_region_field(Legion::PhysicalRegion& pr, Legion::FieldID& fid) const +{ +#ifdef DEBUG_LEGATE + assert(!(is_future() || is_unbound_store())); +#endif + pr = region_field_.get_physical_region(); + fid = region_field_.get_field_id(); +} + +Legion::Future Store::get_future() const +{ +#ifdef DEBUG_LEGATE + assert(is_future()); +#endif + return future_.get_future(); +} + +Legion::UntypedDeferredValue Store::get_buffer() const +{ +#ifdef DEBUG_LEGATE + assert(is_future()); +#endif + return future_.get_buffer(); +} + +void Store::get_output_field(Legion::OutputRegion& out, Legion::FieldID& fid) +{ +#ifdef DEBUG_LEGATE + assert(is_unbound_store()); +#endif + out = unbound_field_.get_output_region(); + fid = unbound_field_.get_field_id(); +} + +void Store::update_num_elements(size_t num_elements) +{ + unbound_field_.update_num_elements(num_elements); + unbound_field_.set_bound(true); +} + +} // namespace legate::detail diff --git a/src/core/data/detail/store.h b/src/core/data/detail/store.h new file mode 100644 index 0000000000..0afad392e7 --- /dev/null +++ b/src/core/data/detail/store.h @@ -0,0 +1,256 @@ +/* Cop)yright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include "core/data/buffer.h" +#include "core/task/detail/return.h" +#include "core/type/detail/type_info.h" + +namespace legate { +class Store; +} // namespace legate +namespace legate::detail { + +class TransformStack; + +class RegionField { + public: + RegionField() {} + RegionField(int32_t dim, const Legion::PhysicalRegion& pr, Legion::FieldID fid); + + public: + RegionField(RegionField&& other) noexcept; + RegionField& operator=(RegionField&& other) noexcept; + + private: + RegionField(const RegionField& other) = delete; + RegionField& operator=(const RegionField& other) = delete; + + public: + bool valid() const; + + public: + int32_t dim() const { return dim_; } + + public: + Domain domain() const; + + public: + void unmap(); + + public: + bool is_readable() const { return readable_; } + bool is_writable() const { return writable_; } + bool is_reducible() const { return reducible_; } + + public: + Legion::PhysicalRegion get_physical_region() const { return pr_; } + Legion::FieldID get_field_id() const { return fid_; } + + private: + int32_t dim_{-1}; + Legion::PhysicalRegion pr_{}; + Legion::FieldID fid_{-1U}; + + private: + bool readable_{false}; + bool writable_{false}; + bool reducible_{false}; +}; + +class UnboundRegionField { + public: + UnboundRegionField() {} + UnboundRegionField(const Legion::OutputRegion& out, Legion::FieldID fid); + + public: + UnboundRegionField(UnboundRegionField&& other) noexcept; + UnboundRegionField& operator=(UnboundRegionField&& other) noexcept; + + private: + UnboundRegionField(const UnboundRegionField& other) = delete; + UnboundRegionField& operator=(const UnboundRegionField& other) = delete; + + public: + bool bound() const { return bound_; } + + public: + void bind_empty_data(int32_t dim); + + public: + ReturnValue pack_weight() const; + + public: + void set_bound(bool bound) { bound_ = bound; } + void update_num_elements(size_t num_elements); + + public: + Legion::OutputRegion get_output_region() const { return out_; } + Legion::FieldID get_field_id() const { return fid_; } + + private: + bool bound_{false}; + Legion::UntypedDeferredValue num_elements_; + Legion::OutputRegion out_{}; + Legion::FieldID fid_{-1U}; +}; + +class FutureWrapper { + public: + FutureWrapper() {} + FutureWrapper(bool read_only, + uint32_t field_size, + Domain domain, + Legion::Future future, + bool initialize = false); + + public: + FutureWrapper(const FutureWrapper& other) noexcept; + FutureWrapper& operator=(const FutureWrapper& other) noexcept; + + public: + int32_t dim() const { return domain_.dim; } + + public: + Domain domain() const; + + public: + void initialize_with_identity(int32_t redop_id); + + public: + ReturnValue pack() const; + + public: + bool is_read_only() const { return read_only_; } + Legion::Future get_future() const { return future_; } + Legion::UntypedDeferredValue get_buffer() const { return buffer_; } + + private: + bool read_only_{true}; + uint32_t field_size_{0}; + Domain domain_{}; + Legion::Future future_{}; + Legion::UntypedDeferredValue buffer_{}; +}; + +class Store { + public: + Store() {} + Store(int32_t dim, + std::shared_ptr type, + int32_t redop_id, + FutureWrapper future, + std::shared_ptr&& transform = nullptr); + Store(int32_t dim, + std::shared_ptr type, + int32_t redop_id, + RegionField&& region_field, + std::shared_ptr&& transform = nullptr); + Store(int32_t dim, + std::shared_ptr type, + UnboundRegionField&& unbound_field, + std::shared_ptr&& transform = nullptr); + Store(int32_t dim, + std::shared_ptr type, + int32_t redop_id, + FutureWrapper future, + const std::shared_ptr& transform); + Store(int32_t dim, + std::shared_ptr type, + int32_t redop_id, + RegionField&& region_field, + const std::shared_ptr& transform); + + public: + Store(Store&& other) noexcept; + Store& operator=(Store&& other) noexcept; + + private: + Store(const Store& other) = delete; + Store& operator=(const Store& other) = delete; + + public: + bool valid() const; + bool transformed() const; + + public: + int32_t dim() const { return dim_; } + std::shared_ptr type() const { return type_; } + + public: + Domain domain() const; + + public: + void unmap(); + + public: + bool is_readable() const { return readable_; } + bool is_writable() const { return writable_; } + bool is_reducible() const { return reducible_; } + + public: + void bind_empty_data(); + + public: + bool is_future() const; + bool is_unbound_store() const; + ReturnValue pack() const { return future_.pack(); } + ReturnValue pack_weight() const { return unbound_field_.pack_weight(); } + + private: + friend class legate::Store; + void check_accessor_dimension(const int32_t dim) const; + void check_buffer_dimension(const int32_t dim) const; + void check_shape_dimension(const int32_t dim) const; + void check_valid_binding(bool bind_buffer) const; + Legion::DomainAffineTransform get_inverse_transform() const; + + private: + void get_region_field(Legion::PhysicalRegion& pr, Legion::FieldID& fid) const; + int32_t get_redop_id() const { return redop_id_; } + + private: + bool is_read_only_future() const; + Legion::Future get_future() const; + Legion::UntypedDeferredValue get_buffer() const; + + private: + void get_output_field(Legion::OutputRegion& out, Legion::FieldID& fid); + void update_num_elements(size_t num_elements); + + private: + bool is_future_{false}; + bool is_unbound_store_{false}; + int32_t dim_{-1}; + std::shared_ptr type_{}; + int32_t redop_id_{-1}; + + private: + FutureWrapper future_; + RegionField region_field_; + UnboundRegionField unbound_field_; + + private: + std::shared_ptr transform_{nullptr}; + + private: + bool readable_{false}; + bool writable_{false}; + bool reducible_{false}; +}; + +} // namespace legate::detail diff --git a/src/core/data/transform.cc b/src/core/data/detail/transform.cc similarity index 99% rename from src/core/data/transform.cc rename to src/core/data/detail/transform.cc index 030fe8c95a..1ce0fa9134 100644 --- a/src/core/data/transform.cc +++ b/src/core/data/detail/transform.cc @@ -14,12 +14,11 @@ * */ -#include "core/data/transform.h" -#include "core/legate_c.h" +#include "core/data/detail/transform.h" #include "core/partitioning/partition.h" -#include "core/utilities/buffer_builder.h" +#include "core/utilities/detail/buffer_builder.h" -namespace legate { +namespace legate::detail { Legion::DomainAffineTransform combine(const Legion::DomainAffineTransform& lhs, const Legion::DomainAffineTransform& rhs) @@ -905,4 +904,4 @@ std::ostream& operator<<(std::ostream& out, const Transform& transform) return out; } -} // namespace legate +} // namespace legate::detail diff --git a/src/core/data/transform.h b/src/core/data/detail/transform.h similarity index 98% rename from src/core/data/transform.h rename to src/core/data/detail/transform.h index 417d21f324..44b3e69b53 100644 --- a/src/core/data/transform.h +++ b/src/core/data/detail/transform.h @@ -20,13 +20,15 @@ #include "core/data/shape.h" #include "core/partitioning/restriction.h" -#include "core/runtime/projection.h" +#include "core/runtime/detail/projection.h" +#include "core/utilities/detail/buffer_builder.h" #include "core/utilities/typedefs.h" namespace legate { - -class BufferBuilder; class Partition; +} // namespace legate + +namespace legate::detail { class NonInvertibleTransformation : public std::exception { public: @@ -248,4 +250,4 @@ class Delinearize : public StoreTransform { std::ostream& operator<<(std::ostream& out, const Transform& transform); -} // namespace legate +} // namespace legate::detail diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index d72a273d42..ec498b9f21 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -20,10 +20,8 @@ #include "core/data/logical_store.h" #include "core/data/store.h" #include "core/partitioning/partition.h" +#include "core/type/detail/type_info.h" #include "core/type/type_traits.h" -#include "core/utilities/buffer_builder.h" -#include "core/utilities/dispatch.h" -#include "legate_defines.h" namespace legate { @@ -33,7 +31,7 @@ LogicalStore::LogicalStore(std::shared_ptr&& impl) : impl_ int32_t LogicalStore::dim() const { return impl_->dim(); } -const Type& LogicalStore::type() const { return impl_->type(); } +Type LogicalStore::type() const { return Type(impl_->type()); } const Shape& LogicalStore::extents() const { return impl_->extents(); } @@ -73,13 +71,7 @@ LogicalStore LogicalStore::delinearize(int32_t dim, std::vector&& sizes return LogicalStore(impl_->delinearize(dim, std::move(sizes))); } -std::shared_ptr LogicalStore::get_physical_store() { return impl_->get_physical_store(); } - -void LogicalStore::set_key_partition(const mapping::MachineDesc& machine, - const Partition* partition) -{ - impl_->set_key_partition(machine, partition); -} +Store LogicalStore::get_physical_store() { return Store(impl_->get_physical_store()); } LogicalStorePartition::LogicalStorePartition(std::shared_ptr&& impl) : impl_(std::move(impl)) @@ -88,9 +80,9 @@ LogicalStorePartition::LogicalStorePartition(std::shared_ptrstore()); } -std::shared_ptr LogicalStorePartition::partition() const +const Shape& LogicalStorePartition::color_shape() const { - return impl_->storage_partition()->partition(); + return impl_->partition()->color_shape(); } } // namespace legate diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index 850ed9243b..18539872ee 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -19,11 +19,9 @@ #include #include -#include "legion.h" - #include "core/data/shape.h" #include "core/data/slice.h" -#include "core/data/transform.h" +#include "core/data/store.h" #include "core/type/type_info.h" #include "core/utilities/typedefs.h" @@ -33,10 +31,6 @@ * legate::LogicalStorePartition */ -namespace legate::mapping { -class MachineDesc; -} // namespace legate::mapping - namespace legate::detail { class LogicalStore; class LogicalStorePartition; @@ -44,11 +38,8 @@ class LogicalStorePartition; namespace legate { -class LibraryContext; class LogicalStorePartition; -class Partition; class Runtime; -class Store; /** * @ingroup data @@ -108,7 +99,7 @@ class LogicalStore { * * @return Type of elements in the store */ - const Type& type() const; + Type type() const; /** * @brief Returns the shape of the store. * @@ -330,10 +321,7 @@ class LogicalStore { * * @return A physical store of the logical store */ - std::shared_ptr get_physical_store(); - - public: - void set_key_partition(const mapping::MachineDesc& machine, const Partition* partition); + Store get_physical_store(); public: std::shared_ptr impl() const { return impl_; } @@ -349,7 +337,7 @@ class LogicalStorePartition { public: LogicalStore store() const; - std::shared_ptr partition() const; + const Shape& color_shape() const; public: std::shared_ptr impl() const { return impl_; } diff --git a/src/core/data/scalar.cc b/src/core/data/scalar.cc index 9a5df97a17..7e6dc5221f 100644 --- a/src/core/data/scalar.cc +++ b/src/core/data/scalar.cc @@ -15,77 +15,37 @@ */ #include "core/data/scalar.h" -#include "core/utilities/buffer_builder.h" +#include "core/data/detail/scalar.h" namespace legate { -Scalar::Scalar(const Scalar& other) : own_(other.own_), type_(other.type_->clone()) { copy(other); } +Scalar::Scalar(std::unique_ptr impl) : impl_(impl.release()) {} -Scalar::Scalar(Scalar&& other) : own_(other.own_), type_(std::move(other.type_)), data_(other.data_) -{ - other.own_ = false; - other.type_ = nullptr; - other.data_ = nullptr; -} +Scalar::Scalar(const Scalar& other) : impl_(new detail::Scalar(*other.impl_)) {} -Scalar::Scalar(std::unique_ptr type, const void* data, bool copy) - : type_(std::move(type)), own_(copy) -{ - if (copy) { - auto buffer = malloc(type_->size()); - memcpy(buffer, data, type_->size()); - data_ = buffer; - } else - data_ = data; -} +Scalar::Scalar(Scalar&& other) : impl_(other.impl_) { other.impl_ = nullptr; } -Scalar::Scalar(const std::string& string) : own_(true), type_(string_type()) -{ - auto data_size = sizeof(char) * string.size(); - auto buffer = malloc(sizeof(uint32_t) + data_size); - *static_cast(buffer) = string.size(); - memcpy(static_cast(buffer) + sizeof(uint32_t), string.data(), data_size); - data_ = buffer; -} +Scalar::~Scalar() { delete impl_; } -Scalar::~Scalar() -{ - if (own_) - // We know we own this buffer - free(const_cast(data_)); -} +Scalar::Scalar(Type type, const void* data, bool copy) : impl_(create_impl(type, data, copy)) {} + +Scalar::Scalar(const std::string& string) : impl_(new detail::Scalar(string)) {} Scalar& Scalar::operator=(const Scalar& other) { - own_ = other.own_; - type_ = other.type_->clone(); - copy(other); + *impl_ = *other.impl_; return *this; } -void Scalar::copy(const Scalar& other) -{ - if (other.own_) { - auto size = other.size(); - auto buffer = malloc(size); - memcpy(buffer, other.data_, size); - data_ = buffer; - } else - data_ = other.data_; -} +Type Scalar::type() const { return Type(impl_->type()); } -size_t Scalar::size() const -{ - if (type_->code == Type::Code::STRING) - return *static_cast(data_) + sizeof(uint32_t); - else - return type_->size(); -} +size_t Scalar::size() const { return impl_->size(); } + +const void* Scalar::ptr() const { return impl_->data(); } -void Scalar::pack(BufferBuilder& buffer) const +/*static*/ detail::Scalar* Scalar::create_impl(Type type, const void* data, bool copy) { - type_->pack(buffer); - buffer.pack_buffer(data_, size()); + return new detail::Scalar(std::move(type.impl()), data, copy); } } // namespace legate diff --git a/src/core/data/scalar.h b/src/core/data/scalar.h index 4d3b0c9a31..542c5f3179 100644 --- a/src/core/data/scalar.h +++ b/src/core/data/scalar.h @@ -25,9 +25,15 @@ * @brief Class definition for legate::Scalar */ +namespace legate::detail { +class Scalar; +} // namespace legate::detail + namespace legate { -class BufferBuilder; +class AutoTask; +class ManualTask; +class Runtime; /** * @ingroup data @@ -40,9 +46,12 @@ class BufferBuilder; */ class Scalar { public: + Scalar(std::unique_ptr impl); Scalar(const Scalar& other); Scalar(Scalar&& other); + ~Scalar(); + public: /** * @brief Creates a shared `Scalar` with an existing allocation. The caller is responsible * for passing in a sufficiently big allocation. When `copy` is true, the scalar copies the @@ -51,10 +60,7 @@ class Scalar { * @param type Type of the scalar(s) * @param data Allocation containing the data. */ - Scalar(std::unique_ptr type, const void* data, bool copy = false); - ~Scalar(); - - public: + Scalar(Type type, const void* data, bool copy = false); /** * @brief Creates an owned scalar from a scalar value * @@ -73,7 +79,7 @@ class Scalar { * @param value A scalar value to create a `Scalar` with */ template - Scalar(T value, std::unique_ptr type); + Scalar(T value, Type type); /** * @brief Creates an owned scalar from a string. The value from the * original string will be copied. @@ -110,16 +116,13 @@ class Scalar { public: Scalar& operator=(const Scalar& other); - private: - void copy(const Scalar& other); - public: /** * @brief Returns the data type of the scalar * * @return Data type */ - const Type& type() const { return *type_; } + Type type() const; /** * @brief Returns the size of allocation for the `Scalar`. * @@ -163,20 +166,19 @@ class Scalar { * * @return A raw pointer to the `Scalar`'s data */ - const void* ptr() const { return data_; } + const void* ptr() const; public: - /** - * @brief Serializes the scalar into a buffer - * - * @param buffer A BufferBuilder into which the scalar needs to be serialized - */ - void pack(BufferBuilder& buffer) const; + detail::Scalar* impl() const { return impl_; } + + private: + static detail::Scalar* create_impl(Type type, const void* data, bool copy); private: - bool own_{false}; - std::unique_ptr type_{nullptr}; - const void* data_; + friend class AutoTask; + friend class ManualTask; + friend class Runtime; + detail::Scalar* impl_{nullptr}; }; } // namespace legate diff --git a/src/core/data/scalar.inl b/src/core/data/scalar.inl index eff6987820..a492eb2ad1 100644 --- a/src/core/data/scalar.inl +++ b/src/core/data/scalar.inl @@ -24,73 +24,60 @@ namespace legate { template -Scalar::Scalar(T value) : own_(true), type_(primitive_type(legate_type_code_of)) +Scalar::Scalar(T value) : impl_(create_impl(primitive_type(legate_type_code_of), &value, true)) { static_assert(legate_type_code_of != Type::Code::FIXED_ARRAY); static_assert(legate_type_code_of != Type::Code::STRUCT); static_assert(legate_type_code_of != Type::Code::STRING); static_assert(legate_type_code_of != Type::Code::INVALID); - auto buffer = malloc(sizeof(T)); - memcpy(buffer, &value, sizeof(T)); - data_ = buffer; } template -Scalar::Scalar(T value, std::unique_ptr type) : own_(true), type_(std::move(type)) +Scalar::Scalar(T value, Type type) : impl_(create_impl(type, &value, true)) { - if (type_->code == Type::Code::INVALID) + if (type.code() == Type::Code::INVALID) throw std::invalid_argument("Invalid type cannot be used"); - if (type_->size() != sizeof(T)) + if (type.size() != sizeof(T)) throw std::invalid_argument("Size of the value doesn't match with the type"); - auto buffer = malloc(sizeof(T)); - memcpy(buffer, &value, sizeof(T)); - data_ = buffer; } template Scalar::Scalar(const std::vector& values) - : own_(true), type_(fixed_array_type(primitive_type(legate_type_code_of), values.size())) + : impl_(create_impl( + fixed_array_type(primitive_type(legate_type_code_of), values.size()), values.data(), true)) { - auto size = type_->size(); - auto buffer = malloc(size); - memcpy(buffer, values.data(), size); - data_ = buffer; } template -Scalar::Scalar(const Point& point) : own_(true), type_(point_type(DIM)) +Scalar::Scalar(const Point& point) : impl_(create_impl(point_type(DIM), &point, true)) { - auto buffer = malloc(sizeof(Point)); - memcpy(buffer, &point, sizeof(Point)); - data_ = buffer; } template -Scalar::Scalar(const Rect& rect) : own_(true), type_(rect_type(DIM)) +Scalar::Scalar(const Rect& rect) : impl_(create_impl(rect_type(DIM), &rect, true)) { - auto buffer = malloc(sizeof(Rect)); - memcpy(buffer, &rect, sizeof(Rect)); - data_ = buffer; } template VAL Scalar::value() const { - if (type_->code == Type::Code::STRING) + auto ty = type(); + if (ty.code() == Type::Code::STRING) throw std::invalid_argument("String cannot be casted to other types"); - if (sizeof(VAL) != type_->size()) - throw std::invalid_argument("Size of the scalar is " + std::to_string(type_->size()) + + if (sizeof(VAL) != ty.size()) + throw std::invalid_argument("Size of the scalar is " + std::to_string(ty.size()) + ", but the requested type has size " + std::to_string(sizeof(VAL))); - return *static_cast(data_); + return *static_cast(ptr()); } template <> inline std::string Scalar::value() const { - if (type_->code != Type::Code::STRING) + if (type().code() != Type::Code::STRING) throw std::invalid_argument("Type of the scalar is not string"); - auto len = *static_cast(data_); - const auto* begin = static_cast(data_) + sizeof(uint32_t); + const void* data = ptr(); + auto len = *static_cast(data); + const auto* begin = static_cast(data) + sizeof(uint32_t); const auto* end = begin + len; return std::string(begin, end); } @@ -98,38 +85,41 @@ inline std::string Scalar::value() const template <> inline std::string_view Scalar::value() const { - if (type_->code != Type::Code::STRING) + if (type().code() != Type::Code::STRING) throw std::invalid_argument("Type of the scalar is not string"); - auto len = *static_cast(data_); - const auto* begin = static_cast(data_) + sizeof(uint32_t); + const void* data = ptr(); + auto len = *static_cast(data); + const auto* begin = static_cast(data) + sizeof(uint32_t); return std::string_view(begin, len); } template Span Scalar::values() const { - if (type_->code == Type::Code::FIXED_ARRAY) { - auto arr_type = static_cast(type_.get()); - const auto& elem_type = arr_type->element_type(); + auto ty = type(); + if (ty.code() == Type::Code::FIXED_ARRAY) { + auto arr_type = ty.as_fixed_array_type(); + auto elem_type = arr_type.element_type(); if (sizeof(VAL) != elem_type.size()) throw std::invalid_argument( "The scalar's element type has size " + std::to_string(elem_type.size()) + ", but the requested element type has size " + std::to_string(sizeof(VAL))); - auto size = arr_type->num_elements(); - return Span(reinterpret_cast(data_), size); - } else if (type_->code == Type::Code::STRING) { + auto size = arr_type.num_elements(); + return Span(reinterpret_cast(ptr()), size); + } else if (ty.code() == Type::Code::STRING) { if (sizeof(VAL) != 1) throw std::invalid_argument( "String scalar can only be converted into a span of a type whose size is 1 byte"); - auto len = *static_cast(data_); - const auto* begin = static_cast(data_) + sizeof(uint32_t); + auto data = ptr(); + auto len = *static_cast(data); + const auto* begin = static_cast(data) + sizeof(uint32_t); return Span(reinterpret_cast(begin), len); } else { - if (sizeof(VAL) != type_->size()) - throw std::invalid_argument("Size of the scalar is " + std::to_string(type_->size()) + + if (sizeof(VAL) != ty.size()) + throw std::invalid_argument("Size of the scalar is " + std::to_string(ty.size()) + ", but the requested element type has size " + std::to_string(sizeof(VAL))); - return Span(static_cast(data_), 1); + return Span(static_cast(ptr()), 1); } } diff --git a/src/core/data/store.cc b/src/core/data/store.cc index 2791c4cfb7..8b89cd7cf3 100644 --- a/src/core/data/store.cc +++ b/src/core/data/store.cc @@ -1,4 +1,4 @@ -/* Copyright 2021-2022 NVIDIA Corporation +/* Copyright 2023 NVIDIA Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,383 +15,90 @@ */ #include "core/data/store.h" - -#include "core/data/buffer.h" -#include "core/runtime/detail/runtime.h" -#include "core/utilities/dispatch.h" -#include "core/utilities/machine.h" -#include "legate_defines.h" - -#ifdef LEGATE_USE_CUDA -#include "core/cuda/cuda_help.h" -#include "core/cuda/stream_pool.h" -#endif +#include "core/data/detail/store.h" namespace legate { -RegionField::RegionField(int32_t dim, const Legion::PhysicalRegion& pr, Legion::FieldID fid) - : dim_(dim), pr_(pr), fid_(fid) -{ - auto priv = pr.get_privilege(); - readable_ = static_cast(priv & LEGION_READ_PRIV); - writable_ = static_cast(priv & LEGION_WRITE_PRIV); - reducible_ = static_cast(priv & LEGION_REDUCE) || (readable_ && writable_); -} +void Store::bind_empty_data() { impl_->bind_empty_data(); } -RegionField::RegionField(RegionField&& other) noexcept - : dim_(other.dim_), - pr_(other.pr_), - fid_(other.fid_), - readable_(other.readable_), - writable_(other.writable_), - reducible_(other.reducible_) -{ -} +int32_t Store::dim() const { return impl_->dim(); } -RegionField& RegionField::operator=(RegionField&& other) noexcept -{ - dim_ = other.dim_; - pr_ = other.pr_; - fid_ = other.fid_; - readable_ = other.readable_; - writable_ = other.writable_; - reducible_ = other.reducible_; - return *this; -} - -bool RegionField::valid() const -{ - return pr_.get_logical_region() != Legion::LogicalRegion::NO_REGION; -} +Type Store::type() const { return Type(impl_->type()); } -Domain RegionField::domain() const { return dim_dispatch(dim_, get_domain_fn{}, pr_); } - -void RegionField::unmap() { detail::Runtime::get_runtime()->unmap_physical_region(pr_); } - -UnboundRegionField::UnboundRegionField(const Legion::OutputRegion& out, Legion::FieldID fid) - : out_(out), - fid_(fid), - num_elements_( - Legion::UntypedDeferredValue(sizeof(size_t), find_memory_kind_for_executing_processor())) -{ -} +Domain Store::domain() const { return impl_->domain(); } -UnboundRegionField::UnboundRegionField(UnboundRegionField&& other) noexcept - : bound_(other.bound_), out_(other.out_), fid_(other.fid_), num_elements_(other.num_elements_) -{ - other.bound_ = false; - other.out_ = Legion::OutputRegion(); - other.fid_ = -1; - other.num_elements_ = Legion::UntypedDeferredValue(); -} +bool Store::is_readable() const { return impl_->is_readable(); } -UnboundRegionField& UnboundRegionField::operator=(UnboundRegionField&& other) noexcept -{ - bound_ = other.bound_; - out_ = other.out_; - fid_ = other.fid_; - num_elements_ = other.num_elements_; +bool Store::is_writable() const { return impl_->is_writable(); } - other.bound_ = false; - other.out_ = Legion::OutputRegion(); - other.fid_ = -1; - other.num_elements_ = Legion::UntypedDeferredValue(); +bool Store::is_reducible() const { return impl_->is_reducible(); } - return *this; -} +bool Store::valid() const { return impl_ != nullptr && impl_->valid(); } -void UnboundRegionField::bind_empty_data(int32_t ndim) -{ - update_num_elements(0); - DomainPoint extents; - extents.dim = ndim; - for (int32_t dim = 0; dim < ndim; ++dim) extents[dim] = 0; - auto empty_buffer = create_buffer(0); - out_.return_data(extents, fid_, empty_buffer.get_instance(), false); - bound_ = true; -} +bool Store::transformed() const { return impl_->transformed(); } -ReturnValue UnboundRegionField::pack_weight() const -{ -#ifdef DEBUG_LEGATE - if (!bound_) { - legate::log_legate.error( - "Found an uninitialized unbound store. Please make sure you return buffers to all unbound " - "stores in the task"); - LEGATE_ABORT; - } -#endif - return ReturnValue(num_elements_, sizeof(size_t)); -} +bool Store::is_future() const { return impl_->is_future(); } -void UnboundRegionField::update_num_elements(size_t num_elements) -{ - AccessorWO acc(num_elements_, sizeof(size_t), false); - acc[0] = num_elements; -} +bool Store::is_unbound_store() const { return impl_->is_unbound_store(); } -FutureWrapper::FutureWrapper(bool read_only, - uint32_t field_size, - Domain domain, - Legion::Future future, - bool initialize /*= false*/) - : read_only_(read_only), field_size_(field_size), domain_(domain), future_(future) -{ - if (!read_only) { -#ifdef DEBUG_LEGATE - assert(!initialize || future_.get_untyped_size() == field_size); -#endif - auto mem_kind = find_memory_kind_for_executing_processor( -#ifdef LEGATE_NO_FUTURES_ON_FB - true -#else - false -#endif - ); - if (initialize) { - auto p_init_value = future_.get_buffer(mem_kind); -#ifdef LEGATE_USE_CUDA - if (mem_kind == Memory::Kind::GPU_FB_MEM) { - // TODO: This should be done by Legion - buffer_ = Legion::UntypedDeferredValue(field_size, mem_kind); - AccessorWO acc(buffer_, field_size, false); - auto stream = cuda::StreamPool::get_stream_pool().get_stream(); - CHECK_CUDA( - cudaMemcpyAsync(acc.ptr(0), p_init_value, field_size, cudaMemcpyDeviceToDevice, stream)); - } else -#endif - buffer_ = Legion::UntypedDeferredValue(field_size, mem_kind, p_init_value); - } else - buffer_ = Legion::UntypedDeferredValue(field_size, mem_kind); - } -} +void Store::unmap() { impl_->unmap(); } -FutureWrapper::FutureWrapper(const FutureWrapper& other) noexcept - : read_only_(other.read_only_), - field_size_(other.field_size_), - domain_(other.domain_), - future_(other.future_), - buffer_(other.buffer_) +void Store::check_accessor_dimension(const int32_t dim) const { + impl_->check_accessor_dimension(dim); } -FutureWrapper& FutureWrapper::operator=(const FutureWrapper& other) noexcept -{ - read_only_ = other.read_only_; - field_size_ = other.field_size_; - domain_ = other.domain_; - future_ = other.future_; - buffer_ = other.buffer_; - return *this; -} +void Store::check_buffer_dimension(const int32_t dim) const { impl_->check_buffer_dimension(dim); } -Domain FutureWrapper::domain() const { return domain_; } +void Store::check_shape_dimension(const int32_t dim) const { impl_->check_shape_dimension(dim); } -void FutureWrapper::initialize_with_identity(int32_t redop_id) -{ - auto untyped_acc = AccessorWO(buffer_, field_size_); - auto ptr = untyped_acc.ptr(0); - - auto redop = Legion::Runtime::get_reduction_op(redop_id); -#ifdef DEBUG_LEGATE - assert(redop->sizeof_lhs == field_size_); -#endif - auto identity = redop->identity; -#ifdef LEGATE_USE_CUDA - if (buffer_.get_instance().get_location().kind() == Memory::Kind::GPU_FB_MEM) { - auto stream = cuda::StreamPool::get_stream_pool().get_stream(); - CHECK_CUDA(cudaMemcpyAsync(ptr, identity, field_size_, cudaMemcpyHostToDevice, stream)); - } else -#endif - memcpy(ptr, identity, field_size_); -} +void Store::check_valid_binding(bool bind_buffer) const { impl_->check_valid_binding(bind_buffer); } -ReturnValue FutureWrapper::pack() const { return ReturnValue(buffer_, field_size_); } - -Store::Store(int32_t dim, - std::unique_ptr type, - int32_t redop_id, - FutureWrapper future, - std::shared_ptr&& transform) - : is_future_(true), - is_unbound_store_(false), - dim_(dim), - type_(std::move(type)), - redop_id_(redop_id), - future_(future), - transform_(std::move(transform)), - readable_(true) +Legion::DomainAffineTransform Store::get_inverse_transform() const { + return impl_->get_inverse_transform(); } -Store::Store(int32_t dim, - std::unique_ptr type, - int32_t redop_id, - RegionField&& region_field, - std::shared_ptr&& transform) - : is_future_(false), - is_unbound_store_(false), - dim_(dim), - type_(std::move(type)), - redop_id_(redop_id), - region_field_(std::move(region_field)), - transform_(std::move(transform)) -{ - readable_ = region_field_.is_readable(); - writable_ = region_field_.is_writable(); - reducible_ = region_field_.is_reducible(); -} +bool Store::is_read_only_future() const { return impl_->is_read_only_future(); } -Store::Store(int32_t dim, - std::unique_ptr type, - UnboundRegionField&& unbound_field, - std::shared_ptr&& transform) - : is_future_(false), - is_unbound_store_(true), - dim_(dim), - type_(std::move(type)), - redop_id_(-1), - unbound_field_(std::move(unbound_field)), - transform_(std::move(transform)) +void Store::get_region_field(Legion::PhysicalRegion& pr, Legion::FieldID& fid) const { + impl_->get_region_field(pr, fid); } -Store::Store(int32_t dim, - std::unique_ptr type, - int32_t redop_id, - FutureWrapper future, - const std::shared_ptr& transform) - : is_future_(true), - is_unbound_store_(false), - dim_(dim), - type_(std::move(type)), - redop_id_(redop_id), - future_(future), - transform_(transform), - readable_(true) -{ -} +int32_t Store::get_redop_id() const { return impl_->get_redop_id(); } -Store::Store(int32_t dim, - std::unique_ptr type, - int32_t redop_id, - RegionField&& region_field, - const std::shared_ptr& transform) - : is_future_(false), - is_unbound_store_(false), - dim_(dim), - type_(std::move(type)), - redop_id_(redop_id), - region_field_(std::move(region_field)), - transform_(transform) -{ - readable_ = region_field_.is_readable(); - writable_ = region_field_.is_writable(); - reducible_ = region_field_.is_reducible(); -} +Legion::Future Store::get_future() const { return impl_->get_future(); } -Store::Store(Store&& other) noexcept - : is_future_(other.is_future_), - is_unbound_store_(other.is_unbound_store_), - dim_(other.dim_), - type_(std::move(other.type_)), - redop_id_(other.redop_id_), - future_(other.future_), - region_field_(std::move(other.region_field_)), - unbound_field_(std::move(other.unbound_field_)), - transform_(std::move(other.transform_)), - readable_(other.readable_), - writable_(other.writable_), - reducible_(other.reducible_) -{ -} +Legion::UntypedDeferredValue Store::get_buffer() const { return impl_->get_buffer(); } -Store& Store::operator=(Store&& other) noexcept +void Store::get_output_field(Legion::OutputRegion& out, Legion::FieldID& fid) { - is_future_ = other.is_future_; - is_unbound_store_ = other.is_unbound_store_; - dim_ = other.dim_; - type_ = std::move(other.type_); - redop_id_ = other.redop_id_; - if (is_future_) - future_ = other.future_; - else if (is_unbound_store_) - unbound_field_ = std::move(other.unbound_field_); - else - region_field_ = std::move(other.region_field_); - transform_ = std::move(other.transform_); - readable_ = other.readable_; - writable_ = other.writable_; - reducible_ = other.reducible_; - return *this; + impl_->get_output_field(out, fid); } -bool Store::valid() const { return is_future_ || is_unbound_store_ || region_field_.valid(); } - -Domain Store::domain() const -{ - if (is_unbound_store_) - throw std::invalid_argument("Invalid to retrieve the domain of an unbound store"); - auto result = is_future_ ? future_.domain() : region_field_.domain(); - if (!transform_->identity()) result = transform_->transform(result); -#ifdef DEBUG_LEGATE - assert(result.dim == dim_ || dim_ == 0); -#endif - return result; -} +void Store::update_num_elements(size_t num_elements) { impl_->update_num_elements(num_elements); } -void Store::unmap() -{ - if (is_future_ || is_unbound_store_) return; - region_field_.unmap(); -} +Store::Store() {} -void Store::bind_empty_data() -{ - check_valid_binding(); - unbound_field_.bind_empty_data(dim_); -} +Store::Store(std::shared_ptr impl) : impl_(std::move(impl)) {} -void Store::remove_transform() -{ -#ifdef DEBUG_LEGATE - assert(transformed()); -#endif - dim_ = transform_->pop()->target_ndim(dim_); -} +Store::Store(const Store& other) : impl_(other.impl_) {} -void Store::check_accessor_dimension(const int32_t dim) const +Store& Store::operator=(const Store& other) { - if (!(dim == dim_ || (dim_ == 0 && dim == 1))) { - throw std::invalid_argument("Dimension mismatch: invalid to create a " + std::to_string(dim) + - "-D accessor to a " + std::to_string(dim_) + "-D store"); - } + impl_ = other.impl_; + return *this; } -void Store::check_buffer_dimension(const int32_t dim) const -{ - if (dim != dim_) { - throw std::invalid_argument("Dimension mismatch: invalid to bind a " + std::to_string(dim) + - "-D buffer to a " + std::to_string(dim_) + "-D store"); - } -} +Store::Store(Store&& other) : impl_(std::move(other.impl_)) {} -void Store::check_shape_dimension(const int32_t dim) const +Store& Store::operator=(Store&& other) { - if (!(dim == dim_ || (dim_ == 0 && dim == 1))) { - throw std::invalid_argument("Dimension mismatch: invalid to retrieve a " + std::to_string(dim) + - "-D rect from a " + std::to_string(dim_) + "-D store"); - } + impl_ = std::move(other.impl_); + return *this; } -void Store::check_valid_binding() const -{ - if (!is_unbound_store_) { - throw std::invalid_argument("Buffer can be bound only to an unbound store"); - } - if (unbound_field_.bound()) { - throw std::invalid_argument("A buffer has already been bound to the store"); - } -} +Store::~Store() {} } // namespace legate diff --git a/src/core/data/store.h b/src/core/data/store.h index 9e03562d0f..790f16e73b 100644 --- a/src/core/data/store.h +++ b/src/core/data/store.h @@ -1,4 +1,4 @@ -/* Copyright 2021-2022 NVIDIA Corporation +/* Copyright 2023 NVIDIA Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,14 +17,8 @@ #pragma once #include "core/data/buffer.h" -#include "core/data/transform.h" -#include "core/task/return.h" -#include "core/type/type_info.h" #include "core/type/type_traits.h" -#include "core/utilities/machine.h" -#include "core/utilities/typedefs.h" -#include "legate_defines.h" -#include "legion.h" +#include "core/utilities/dispatch.h" /** @defgroup data Data abstractions and allocators */ @@ -34,239 +28,11 @@ * @brief Class definition for legate::Store */ -namespace legate { - -class RegionField { - public: - RegionField() {} - RegionField(int32_t dim, const Legion::PhysicalRegion& pr, Legion::FieldID fid); - - public: - RegionField(RegionField&& other) noexcept; - RegionField& operator=(RegionField&& other) noexcept; - - private: - RegionField(const RegionField& other) = delete; - RegionField& operator=(const RegionField& other) = delete; - - public: - bool valid() const; +namespace legate::detail { +class Store; +} // namespace legate::detail - public: - int32_t dim() const { return dim_; } - - private: - template - struct trans_accessor_fn { - template - ACC operator()(const Legion::PhysicalRegion& pr, - Legion::FieldID fid, - const Legion::AffineTransform& transform) - { - return ACC(pr, fid, transform); - } - template - ACC operator()(const Legion::PhysicalRegion& pr, - Legion::FieldID fid, - const Legion::AffineTransform& transform, - const Rect& bounds) - { - return ACC(pr, fid, transform, bounds); - } - template - ACC operator()(const Legion::PhysicalRegion& pr, - Legion::FieldID fid, - int32_t redop_id, - const Legion::AffineTransform& transform) - { - return ACC(pr, fid, redop_id, transform); - } - template - ACC operator()(const Legion::PhysicalRegion& pr, - Legion::FieldID fid, - int32_t redop_id, - const Legion::AffineTransform& transform, - const Rect& bounds) - { - return ACC(pr, fid, redop_id, transform, bounds); - } - }; - - struct get_domain_fn { - template - Domain operator()(const Legion::PhysicalRegion& pr) - { - return Domain(pr.get_bounds()); - } - }; - - public: - template - AccessorRO read_accessor() const; - template - AccessorWO write_accessor() const; - template - AccessorRW read_write_accessor() const; - template - AccessorRD reduce_accessor(int32_t redop_id) const; - - public: - template - AccessorRO read_accessor(const Legion::DomainAffineTransform& transform) const; - template - AccessorWO write_accessor(const Legion::DomainAffineTransform& transform) const; - template - AccessorRW read_write_accessor(const Legion::DomainAffineTransform& transform) const; - template - AccessorRD reduce_accessor( - int32_t redop_id, const Legion::DomainAffineTransform& transform) const; - - public: - template - AccessorRO read_accessor(const Rect& bounds) const; - template - AccessorWO write_accessor(const Rect& bounds) const; - template - AccessorRW read_write_accessor(const Rect& bounds) const; - template - AccessorRD reduce_accessor(int32_t redop_id, const Rect& bounds) const; - - public: - template - AccessorRO read_accessor(const Rect& bounds, - const Legion::DomainAffineTransform& transform) const; - template - AccessorWO write_accessor(const Rect& bounds, - const Legion::DomainAffineTransform& transform) const; - template - AccessorRW read_write_accessor(const Rect& bounds, - const Legion::DomainAffineTransform& transform) const; - template - AccessorRD reduce_accessor( - int32_t redop_id, - const Rect& bounds, - const Legion::DomainAffineTransform& transform) const; - - public: - template - Rect shape() const; - Domain domain() const; - - public: - void unmap(); - - public: - bool is_readable() const { return readable_; } - bool is_writable() const { return writable_; } - bool is_reducible() const { return reducible_; } - - private: - int32_t dim_{-1}; - Legion::PhysicalRegion pr_{}; - Legion::FieldID fid_{-1U}; - - private: - bool readable_{false}; - bool writable_{false}; - bool reducible_{false}; -}; - -class UnboundRegionField { - public: - UnboundRegionField() {} - UnboundRegionField(const Legion::OutputRegion& out, Legion::FieldID fid); - - public: - UnboundRegionField(UnboundRegionField&& other) noexcept; - UnboundRegionField& operator=(UnboundRegionField&& other) noexcept; - - private: - UnboundRegionField(const UnboundRegionField& other) = delete; - UnboundRegionField& operator=(const UnboundRegionField& other) = delete; - - public: - bool bound() const { return bound_; } - - public: - template - Buffer create_output_buffer(const Point& extents, bool bind_buffer); - - public: - template - void bind_data(Buffer& buffer, const Point& extents); - void bind_empty_data(int32_t dim); - - public: - ReturnValue pack_weight() const; - - private: - void update_num_elements(size_t num_elements); - - private: - bool bound_{false}; - Legion::UntypedDeferredValue num_elements_; - Legion::OutputRegion out_{}; - Legion::FieldID fid_{-1U}; -}; - -class FutureWrapper { - public: - FutureWrapper() {} - FutureWrapper(bool read_only, - uint32_t field_size, - Domain domain, - Legion::Future future, - bool initialize = false); - - public: - FutureWrapper(const FutureWrapper& other) noexcept; - FutureWrapper& operator=(const FutureWrapper& other) noexcept; - - public: - int32_t dim() const { return domain_.dim; } - - public: - template - AccessorRO read_accessor() const; - template - AccessorWO write_accessor() const; - template - AccessorRW read_write_accessor() const; - template - AccessorRD reduce_accessor(int32_t redop_id) const; - - public: - template - AccessorRO read_accessor(const Rect& bounds) const; - template - AccessorWO write_accessor(const Rect& bounds) const; - template - AccessorRW read_write_accessor(const Rect& bounds) const; - template - AccessorRD reduce_accessor(int32_t redop_id, const Rect& bounds) const; - - public: - template - VAL scalar() const; - - public: - template - Rect shape() const; - Domain domain() const; - - public: - void initialize_with_identity(int32_t redop_id); - - public: - ReturnValue pack() const; - - private: - bool read_only_{true}; - uint32_t field_size_{0}; - Domain domain_{}; - Legion::Future future_{}; - Legion::UntypedDeferredValue buffer_{}; -}; +namespace legate { /** * @ingroup data @@ -274,82 +40,6 @@ class FutureWrapper { * @brief A multi-dimensional data container storing task data */ class Store { - public: - Store() {} - Store(int32_t dim, - std::unique_ptr type, - int32_t redop_id, - FutureWrapper future, - std::shared_ptr&& transform = nullptr); - Store(int32_t dim, - std::unique_ptr type, - int32_t redop_id, - RegionField&& region_field, - std::shared_ptr&& transform = nullptr); - Store(int32_t dim, - std::unique_ptr type, - UnboundRegionField&& unbound_field, - std::shared_ptr&& transform = nullptr); - Store(int32_t dim, - std::unique_ptr type, - int32_t redop_id, - FutureWrapper future, - const std::shared_ptr& transform); - Store(int32_t dim, - std::unique_ptr type, - int32_t redop_id, - RegionField&& region_field, - const std::shared_ptr& transform); - - public: - Store(Store&& other) noexcept; - Store& operator=(Store&& other) noexcept; - - private: - Store(const Store& other) = delete; - Store& operator=(const Store& other) = delete; - - public: - /** - * @brief Indicates whether the store is valid. A store passed to a task can be invalid - * only for reducer tasks for tree reduction. - * - * @return true The store is valid - * @return false The store is invalid and cannot be used in any data access - */ - bool valid() const; - /** - * @brief Indicates whether the store is transformed in any way. - * - * @return true The store is transformed - * @return false The store is not transformed - */ - bool transformed() const { return !transform_->identity(); } - - public: - /** - * @brief Returns the dimension of the store - * - * @return The store's dimension - */ - int32_t dim() const { return dim_; } - /** - * @brief Returns the type metadata of the store - * - * @return The store's type metadata - */ - const Type& type() const { return *type_; } - /** - * @brief Returns the type code of the store - * - * @return The store's type code - */ - template - TYPE_CODE code() const - { - return static_cast(type_->code); - } - public: /** * @brief Returns a read-only accessor to the store for the entire domain. @@ -373,12 +63,6 @@ class Store { * * @tparam VALIDATE_TYPE If `true` (default), validates type and number of dimensions * - * @tparam T Element type - * - * @tparam DIM Number of dimensions - * - * @tparam VALIDATE_TYPE If `true` (default), validates type and number of dimensions - * * @return A write-only accessor to the store */ template @@ -392,12 +76,6 @@ class Store { * * @tparam VALIDATE_TYPE If `true` (default), validates type and number of dimensions * - * @tparam T Element type - * - * @tparam DIM Number of dimensions - * - * @tparam VALIDATE_TYPE If `true` (default), validates type and number of dimensions - * * @return A read-write accessor to the store */ template @@ -406,7 +84,7 @@ class Store { * @brief Returns a reduction accessor to the store for the entire domain. * * @tparam OP Reduction operator class. For details about reduction operators, See - * LibraryContext::register_reduction_operator. + * Library::register_reduction_operator. * * @tparam EXCLUSIVE Indicates whether reductions can be performed in exclusive mode. If * `EXCLUSIVE` is `false`, every reduction via the accessor is performed atomically. @@ -415,10 +93,6 @@ class Store { * * @tparam VALIDATE_TYPE If `true` (default), validates type and number of dimensions * - * @tparam DIM Number of dimensions - * - * @tparam VALIDATE_TYPE If `true` (default), validates type and number of dimensions - * * @return A reduction accessor to the store */ template @@ -434,12 +108,6 @@ class Store { * * @tparam VALIDATE_TYPE If `true` (default), validates type and number of dimensions * - * @tparam T Element type - * - * @tparam DIM Number of dimensions - * - * @tparam VALIDATE_TYPE If `true` (default), validates type and number of dimensions - * * @param bounds Domain within which accesses should be allowed. * The actual bounds for valid access are determined by an intersection between * the store's domain and the bounds. @@ -457,12 +125,6 @@ class Store { * * @tparam VALIDATE_TYPE If `true` (default), validates type and number of dimensions * - * @tparam T Element type - * - * @tparam DIM Number of dimensions - * - * @tparam VALIDATE_TYPE If `true` (default), validates type and number of dimensions - * * @param bounds Domain within which accesses should be allowed. * The actual bounds for valid access are determined by an intersection between * the store's domain and the bounds. @@ -480,12 +142,6 @@ class Store { * * @tparam VALIDATE_TYPE If `true` (default), validates type and number of dimensions * - * @tparam T Element type - * - * @tparam DIM Number of dimensions - * - * @tparam VALIDATE_TYPE If `true` (default), validates type and number of dimensions - * * @param bounds Domain within which accesses should be allowed. * The actual bounds for valid access are determined by an intersection between * the store's domain and the bounds. @@ -502,7 +158,7 @@ class Store { * the store's domain and the bounds. * * @tparam OP Reduction operator class. For details about reduction operators, See - * LibraryContext::register_reduction_operator. + * Library::register_reduction_operator. * * @tparam EXCLUSIVE Indicates whether reductions can be performed in exclusive mode. If * `EXCLUSIVE` is `false`, every reduction via the accessor is performed atomically. @@ -511,15 +167,25 @@ class Store { * * @tparam VALIDATE_TYPE If `true` (default), validates type and number of dimensions * - * @tparam DIM Number of dimensions - * - * @tparam VALIDATE_TYPE If `true` (default), validates type and number of dimensions - * * @return A reduction accessor to the store */ template AccessorRD reduce_accessor(const Rect& bounds) const; + public: + /** + * @brief Returns the scalar value stored in the store. + * + * The requested type must match with the store's data type. If the store is not + * backed by the future, the runtime will fail with an error message. + * + * @tparam VAL Type of the scalar value + * + * @return The scalar value stored in the store + */ + template + VAL scalar() const; + public: /** * @brief Creates a buffer of specified extents for the unbound store. The returned @@ -535,6 +201,50 @@ class Store { */ template Buffer create_output_buffer(const Point& extents, bool bind_buffer = false); + /** + * @brief Binds a buffer to the store. Valid only when the store is unbound and + * has not yet been bound to another buffer. The buffer must be consistent with + * the mapping policy for the store. Recommend that the buffer be created by + * a `create_output_buffer` call. + * + * @param buffer Buffer to bind to the store + * + * @param extents Extents of the buffer. Passing extents smaller than the actual + * extents of the buffer is legal; the runtime uses the passed extents as the + * extents of this store. + * + */ + template + void bind_data(Buffer& buffer, const Point& extents); + /** + * @brief Makes the unbound store empty. Valid only when the store is unbound and + * has not yet been bound to another buffer. + */ + void bind_empty_data(); + + public: + /** + * @brief Returns the dimension of the store + * + * @return The store's dimension + */ + int32_t dim() const; + /** + * @brief Returns the type metadata of the store + * + * @return The store's type metadata + */ + Type type() const; + /** + * @brief Returns the type code of the store + * + * @return The store's type code + */ + template + TYPE_CODE code() const + { + return static_cast(type().code()); + } public: /** @@ -551,12 +261,6 @@ class Store { */ Domain domain() const; - public: - /** - * @brief Releases all inline allocations of the store - */ - void unmap(); - public: /** * @brief Indicates whether the store can have a read accessor @@ -564,57 +268,38 @@ class Store { * @return true The store can have a read accessor * @return false The store cannot have a read accesor */ - bool is_readable() const { return readable_; } + bool is_readable() const; /** * @brief Indicates whether the store can have a write accessor * * @return true The store can have a write accessor * @return false The store cannot have a write accesor */ - bool is_writable() const { return writable_; } + bool is_writable() const; /** * @brief Indicates whether the store can have a reduction accessor * * @return true The store can have a reduction accessor * @return false The store cannot have a reduction accesor */ - bool is_reducible() const { return reducible_; } + bool is_reducible() const; public: /** - * @brief Returns the scalar value stored in the store. - * - * The requested type must match with the store's data type. If the store is not - * backed by the future, the runtime will fail with an error message. - * - * @tparam VAL Type of the scalar value + * @brief Indicates whether the store is valid. A store passed to a task can be invalid + * only for reducer tasks for tree reduction. * - * @return The scalar value stored in the store + * @return true The store is valid + * @return false The store is invalid and cannot be used in any data access */ - template - VAL scalar() const; - - public: + bool valid() const; /** - * @brief Binds a buffer to the store. Valid only when the store is unbound and - * has not yet been bound to another buffer. The buffer must be consistent with - * the mapping policy for the store. Recommend that the buffer be created by - * a `create_output_buffer` call. - * - * @param buffer Buffer to bind to the store - * - * @param extents Extents of the buffer. Passing extents smaller than the actual - * extents of the buffer is legal; the runtime uses the passed extents as the - * extents of this store. + * @brief Indicates whether the store is transformed in any way. * + * @return true The store is transformed + * @return false The store is not transformed */ - template - void bind_data(Buffer& buffer, const Point& extents); - /** - * @brief Makes the unbound store empty. Valid only when the store is unbound and - * has not yet been bound to another buffer. - */ - void bind_empty_data(); + bool transformed() const; public: /** @@ -624,7 +309,7 @@ class Store { * @return true The store is backed by a future * @return false The store is backed by a region field */ - bool is_future() const { return is_future_; } + bool is_future() const; /** * @brief Indicates whether the store is an unbound store. The value DOES NOT indicate * that the store has already assigned to a buffer; i.e., the store may have been assigned @@ -633,43 +318,56 @@ class Store { * @return true The store is an unbound store * @return false The store is a normal store */ - bool is_unbound_store() const { return is_unbound_store_; } - ReturnValue pack() const { return future_.pack(); } - ReturnValue pack_weight() const { return unbound_field_.pack_weight(); } + bool is_unbound_store() const; public: - // TODO: It'd be btter to return a parent store from this method than permanently - // losing the transform. This requires the backing storages to be referenced by multiple - // stores, which isn't possible as they use move-only types. - void remove_transform(); + /** + * @brief Releases all inline allocations of the store + */ + void unmap(); private: void check_accessor_dimension(const int32_t dim) const; void check_buffer_dimension(const int32_t dim) const; void check_shape_dimension(const int32_t dim) const; - void check_valid_binding() const; + void check_valid_binding(bool bind_buffer) const; template void check_accessor_type() const; + Legion::DomainAffineTransform get_inverse_transform() const; private: - bool is_future_{false}; - bool is_unbound_store_{false}; - int32_t dim_{-1}; - std::unique_ptr type_{nullptr}; - int32_t redop_id_{-1}; + void get_region_field(Legion::PhysicalRegion& pr, Legion::FieldID& fid) const; + int32_t get_redop_id() const; + template + ACC create_field_accessor(const Rect& bounds) const; + template + ACC create_reduction_accessor(const Rect& bounds) const; private: - FutureWrapper future_; - RegionField region_field_; - UnboundRegionField unbound_field_; + bool is_read_only_future() const; + Legion::Future get_future() const; + Legion::UntypedDeferredValue get_buffer() const; private: - std::shared_ptr transform_{nullptr}; + void get_output_field(Legion::OutputRegion& out, Legion::FieldID& fid); + void update_num_elements(size_t num_elements); + + public: + Store(); + Store(std::shared_ptr impl); + std::shared_ptr impl() const { return impl_; } + + public: + Store(const Store&); + Store& operator=(const Store&); + Store(Store&&); + Store& operator=(Store&&); + + public: + ~Store(); private: - bool readable_{false}; - bool writable_{false}; - bool reducible_{false}; + std::shared_ptr impl_{nullptr}; }; } // namespace legate diff --git a/src/core/data/store.inl b/src/core/data/store.inl index 50bc3b2753..c294349057 100644 --- a/src/core/data/store.inl +++ b/src/core/data/store.inl @@ -19,281 +19,34 @@ // Useful for IDEs #include "core/data/store.h" -namespace legate { - -template -AccessorRO RegionField::read_accessor() const -{ - return AccessorRO(pr_, fid_); -} - -template -AccessorWO RegionField::write_accessor() const -{ - return AccessorWO(pr_, fid_); -} - -template -AccessorRW RegionField::read_write_accessor() const -{ - return AccessorRW(pr_, fid_); -} - -template -AccessorRD RegionField::reduce_accessor(int32_t redop_id) const -{ - return AccessorRD(pr_, fid_, redop_id); -} - -template -AccessorRO RegionField::read_accessor(const Legion::DomainAffineTransform& transform) const -{ - using ACC = AccessorRO; - return dim_dispatch(transform.transform.m, trans_accessor_fn{}, pr_, fid_, transform); -} - -template -AccessorWO RegionField::write_accessor(const Legion::DomainAffineTransform& transform) const -{ - using ACC = AccessorWO; - return dim_dispatch(transform.transform.m, trans_accessor_fn{}, pr_, fid_, transform); -} - -template -AccessorRW RegionField::read_write_accessor( - const Legion::DomainAffineTransform& transform) const -{ - using ACC = AccessorRW; - return dim_dispatch(transform.transform.m, trans_accessor_fn{}, pr_, fid_, transform); -} - -template -AccessorRD RegionField::reduce_accessor( - int32_t redop_id, const Legion::DomainAffineTransform& transform) const -{ - using ACC = AccessorRD; - return dim_dispatch( - transform.transform.m, trans_accessor_fn{}, pr_, fid_, redop_id, transform); -} - -template -AccessorRO RegionField::read_accessor(const Rect& bounds) const -{ - return AccessorRO(pr_, fid_, bounds); -} - -template -AccessorWO RegionField::write_accessor(const Rect& bounds) const -{ - return AccessorWO(pr_, fid_, bounds); -} - -template -AccessorRW RegionField::read_write_accessor(const Rect& bounds) const -{ - return AccessorRW(pr_, fid_, bounds); -} - -template -AccessorRD RegionField::reduce_accessor(int32_t redop_id, - const Rect& bounds) const -{ - return AccessorRD(pr_, fid_, redop_id, bounds); -} - -template -AccessorRO RegionField::read_accessor(const Rect& bounds, - const Legion::DomainAffineTransform& transform) const -{ - using ACC = AccessorRO; - return dim_dispatch( - transform.transform.m, trans_accessor_fn{}, pr_, fid_, transform, bounds); -} - -template -AccessorWO RegionField::write_accessor(const Rect& bounds, - const Legion::DomainAffineTransform& transform) const -{ - using ACC = AccessorWO; - return dim_dispatch( - transform.transform.m, trans_accessor_fn{}, pr_, fid_, transform, bounds); -} - -template -AccessorRW RegionField::read_write_accessor( - const Rect& bounds, const Legion::DomainAffineTransform& transform) const -{ - using ACC = AccessorRW; - return dim_dispatch( - transform.transform.m, trans_accessor_fn{}, pr_, fid_, transform, bounds); -} - -template -AccessorRD RegionField::reduce_accessor( - int32_t redop_id, const Rect& bounds, const Legion::DomainAffineTransform& transform) const -{ - using ACC = AccessorRD; - return dim_dispatch( - transform.transform.m, trans_accessor_fn{}, pr_, fid_, redop_id, transform, bounds); -} - -template -Rect RegionField::shape() const -{ - return Rect(pr_); -} - -template -AccessorRO FutureWrapper::read_accessor() const -{ -#ifdef DEBUG_LEGATE - assert(sizeof(T) == field_size_); -#endif - if (read_only_) { - auto memkind = Memory::Kind::NO_MEMKIND; - return AccessorRO(future_, memkind); - } else - return AccessorRO(buffer_); -} - -template -AccessorWO FutureWrapper::write_accessor() const -{ -#ifdef DEBUG_LEGATE - assert(sizeof(T) == field_size_); - assert(!read_only_); -#endif - return AccessorWO(buffer_); -} - -template -AccessorRW FutureWrapper::read_write_accessor() const -{ -#ifdef DEBUG_LEGATE - assert(sizeof(T) == field_size_); - assert(!read_only_); -#endif - return AccessorRW(buffer_); -} - -template -AccessorRD FutureWrapper::reduce_accessor(int32_t redop_id) const -{ -#ifdef DEBUG_LEGATE - assert(sizeof(typename OP::LHS) == field_size_); - assert(!read_only_); -#endif - return AccessorRD(buffer_); -} - -template -AccessorRO FutureWrapper::read_accessor(const Rect& bounds) const -{ -#ifdef DEBUG_LEGATE - assert(sizeof(T) == field_size_); -#endif - if (read_only_) { - auto memkind = Memory::Kind::NO_MEMKIND; - return AccessorRO(future_, bounds, memkind); - } else - return AccessorRO(buffer_, bounds); -} - -template -AccessorWO FutureWrapper::write_accessor(const Rect& bounds) const -{ -#ifdef DEBUG_LEGATE - assert(sizeof(T) == field_size_); - assert(!read_only_); -#endif - return AccessorWO(buffer_, bounds); -} - -template -AccessorRW FutureWrapper::read_write_accessor(const Rect& bounds) const -{ -#ifdef DEBUG_LEGATE - assert(sizeof(T) == field_size_); - assert(!read_only_); -#endif - return AccessorRW(buffer_, bounds); -} - -template -AccessorRD FutureWrapper::reduce_accessor(int32_t redop_id, - const Rect& bounds) const -{ -#ifdef DEBUG_LEGATE - assert(sizeof(typename OP::LHS) == field_size_); - assert(!read_only_); -#endif - return AccessorRD(buffer_, bounds); -} - -template -Rect FutureWrapper::shape() const -{ - return Rect(domain()); -} - -template -VAL FutureWrapper::scalar() const -{ -#ifdef DEBUG_LEGATE - assert(sizeof(VAL) == field_size_); -#endif - if (!read_only_) - return buffer_.operator Legion::DeferredValue().read(); - else - return future_.get_result(); -} - -template -Buffer UnboundRegionField::create_output_buffer(const Point& extents, bool bind_buffer) -{ - if (bind_buffer) { -#ifdef DEBUG_LEGATE - assert(!bound_); -#endif - // We will use this value only when the unbound store is 1D - update_num_elements(extents[0]); - bound_ = true; +namespace legate::detail { +namespace { + +template +struct trans_accessor_fn { + template + ACC operator()(const Legion::PhysicalRegion& pr, + Legion::FieldID fid, + const Legion::AffineTransform& transform, + const Rect& bounds) + { + return ACC(pr, fid, transform, bounds); } - return out_.create_buffer(extents, fid_, nullptr, bind_buffer); -} + template + ACC operator()(const Legion::PhysicalRegion& pr, + Legion::FieldID fid, + int32_t redop_id, + const Legion::AffineTransform& transform, + const Rect& bounds) + { + return ACC(pr, fid, redop_id, transform, bounds); + } +}; -template -void UnboundRegionField::bind_data(Buffer& buffer, const Point& extents) -{ -#ifdef DEBUG_LEGATE - assert(!bound_); -#endif - out_.return_data(extents, fid_, buffer); - // We will use this value only when the unbound store is 1D - update_num_elements(extents[0]); - bound_ = true; -} +} // namespace +} // namespace legate::detail -template -void Store::check_accessor_type() const -{ - auto in_type = legate_type_code_of; - if (in_type == this->code()) return; - // Test exact match for primitive types - if (in_type != Type::Code::INVALID) { - throw std::invalid_argument( - "Type mismatch: " + PrimitiveType(in_type).to_string() + " accessor to a " + - type().to_string() + - " store. Disable type checking via accessor template parameter if this is intended."); - } - // Test size matches for other types - if (sizeof(T) != type().size()) { - throw std::invalid_argument( - "Type size mismatch: store type " + type().to_string() + " has size " + - std::to_string(type().size()) + ", requested type has size " + std::to_string(sizeof(T)) + - ". Disable type checking via accessor template parameter if this is intended."); - } -} +namespace legate { template AccessorRO Store::read_accessor() const @@ -303,13 +56,15 @@ AccessorRO Store::read_accessor() const check_accessor_type(); } - if (is_future_) return future_.read_accessor(shape()); - - if (!transform_->identity()) { - auto transform = transform_->inverse_transform(dim_); - return region_field_.read_accessor(shape(), transform); + if (is_future()) { + if (is_read_only_future()) { + return AccessorRO(get_future(), shape(), Memory::Kind::NO_MEMKIND); + } else { + return AccessorRO(get_buffer(), shape()); + } } - return region_field_.read_accessor(shape()); + + return create_field_accessor, DIM>(shape()); } template @@ -320,13 +75,9 @@ AccessorWO Store::write_accessor() const check_accessor_type(); } - if (is_future_) return future_.write_accessor(shape()); + if (is_future()) { return AccessorWO(get_buffer(), shape()); } - if (!transform_->identity()) { - auto transform = transform_->inverse_transform(dim_); - return region_field_.write_accessor(shape(), transform); - } - return region_field_.write_accessor(shape()); + return create_field_accessor, DIM>(shape()); } template @@ -337,13 +88,9 @@ AccessorRW Store::read_write_accessor() const check_accessor_type(); } - if (is_future_) return future_.read_write_accessor(shape()); + if (is_future()) { return AccessorRW(get_buffer(), shape()); } - if (!transform_->identity()) { - auto transform = transform_->inverse_transform(dim_); - return region_field_.read_write_accessor(shape(), transform); - } - return region_field_.read_write_accessor(shape()); + return create_field_accessor, DIM>(shape()); } template @@ -355,13 +102,9 @@ AccessorRD Store::reduce_accessor() const check_accessor_type(); } - if (is_future_) return future_.reduce_accessor(redop_id_, shape()); + if (is_future()) { return AccessorRD(get_buffer(), shape()); } - if (!transform_->identity()) { - auto transform = transform_->inverse_transform(dim_); - return region_field_.reduce_accessor(redop_id_, shape(), transform); - } - return region_field_.reduce_accessor(redop_id_, shape()); + return create_reduction_accessor, DIM>(shape()); } template @@ -372,13 +115,15 @@ AccessorRO Store::read_accessor(const Rect& bounds) const check_accessor_type(); } - if (is_future_) return future_.read_accessor(bounds); - - if (!transform_->identity()) { - auto transform = transform_->inverse_transform(dim_); - return region_field_.read_accessor(bounds, transform); + if (is_future()) { + if (is_read_only_future()) { + return AccessorRO(get_future(), bounds, Memory::Kind::NO_MEMKIND); + } else { + return AccessorRO(get_buffer(), bounds); + } } - return region_field_.read_accessor(bounds); + + return create_field_accessor, DIM>(bounds); } template @@ -389,13 +134,9 @@ AccessorWO Store::write_accessor(const Rect& bounds) const check_accessor_type(); } - if (is_future_) return future_.write_accessor(bounds); + if (is_future()) { return AccessorWO(get_buffer(), bounds); } - if (!transform_->identity()) { - auto transform = transform_->inverse_transform(dim_); - return region_field_.write_accessor(bounds, transform); - } - return region_field_.write_accessor(bounds); + return create_field_accessor, DIM>(bounds); } template @@ -406,13 +147,9 @@ AccessorRW Store::read_write_accessor(const Rect& bounds) const check_accessor_type(); } - if (is_future_) return future_.read_write_accessor(bounds); + if (is_future()) { return AccessorRW(get_buffer(), bounds); } - if (!transform_->identity()) { - auto transform = transform_->inverse_transform(dim_); - return region_field_.read_write_accessor(bounds, transform); - } - return region_field_.read_write_accessor(bounds); + return create_field_accessor, DIM>(bounds); } template @@ -424,28 +161,32 @@ AccessorRD Store::reduce_accessor(const Rect& bounds) c check_accessor_type(); } - if (is_future_) return future_.reduce_accessor(redop_id_, bounds); + if (is_future()) { return AccessorRD(get_buffer(), bounds); } - if (!transform_->identity()) { - auto transform = transform_->inverse_transform(dim_); - return region_field_.reduce_accessor(redop_id_, bounds, transform); - } - return region_field_.reduce_accessor(redop_id_, bounds); + return create_reduction_accessor, DIM>(bounds); } template Buffer Store::create_output_buffer(const Point& extents, bool bind_buffer /*= false*/) { - check_valid_binding(); + check_valid_binding(bind_buffer); check_buffer_dimension(DIM); - return unbound_field_.create_output_buffer(extents, bind_buffer); + + Legion::OutputRegion out; + Legion::FieldID fid; + get_output_field(out, fid); + + auto result = out.create_buffer(extents, fid, nullptr, bind_buffer); + // We will use this value only when the unbound store is 1D + if (bind_buffer) update_num_elements(extents[0]); + return result; } template Rect Store::shape() const { check_shape_dimension(DIM); - if (dim_ > 0) { + if (dim() > 0) { return domain().bounds(); } else { auto p = Point::ZEROES(); @@ -456,16 +197,85 @@ Rect Store::shape() const template VAL Store::scalar() const { - if (!is_future_) throw std::invalid_argument("Scalars can be retrieved only from scalar stores"); - return future_.scalar(); + if (!is_future()) { + throw std::invalid_argument("Scalars can be retrieved only from scalar stores"); + } + if (is_read_only_future()) { + return get_future().get_result(); + } else { + return get_buffer().operator Legion::DeferredValue().read(); + } } template void Store::bind_data(Buffer& buffer, const Point& extents) { - check_valid_binding(); + check_valid_binding(true); check_buffer_dimension(DIM); - unbound_field_.bind_data(buffer, extents); + + Legion::OutputRegion out; + Legion::FieldID fid; + get_output_field(out, fid); + + out.return_data(extents, fid, buffer); + // We will use this value only when the unbound store is 1D + update_num_elements(extents[0]); +} + +template +void Store::check_accessor_type() const +{ + auto in_type = legate_type_code_of; + if (in_type == this->code()) return; + // Test exact match for primitive types + if (in_type != Type::Code::INVALID) { + throw std::invalid_argument( + "Type mismatch: " + primitive_type(in_type).to_string() + " accessor to a " + + type().to_string() + + " store. Disable type checking via accessor template parameter if this is intended."); + } + // Test size matches for other types + if (sizeof(T) != type().size()) { + throw std::invalid_argument( + "Type size mismatch: store type " + type().to_string() + " has size " + + std::to_string(type().size()) + ", requested type has size " + std::to_string(sizeof(T)) + + ". Disable type checking via accessor template parameter if this is intended."); + } +} + +template +ACC Store::create_field_accessor(const Rect& bounds) const +{ + Legion::PhysicalRegion pr; + Legion::FieldID fid; + get_region_field(pr, fid); + + if (transformed()) { + auto transform = get_inverse_transform(); + return dim_dispatch( + transform.transform.m, detail::trans_accessor_fn{}, pr, fid, transform, bounds); + } + return ACC(pr, fid, bounds); +} + +template +ACC Store::create_reduction_accessor(const Rect& bounds) const +{ + Legion::PhysicalRegion pr; + Legion::FieldID fid; + get_region_field(pr, fid); + + if (transformed()) { + auto transform = get_inverse_transform(); + return dim_dispatch(transform.transform.m, + detail::trans_accessor_fn{}, + pr, + fid, + get_redop_id(), + transform, + bounds); + } + return ACC(pr, fid, get_redop_id(), bounds); } } // namespace legate diff --git a/src/core/mapping/base_mapper.cc b/src/core/mapping/detail/base_mapper.cc similarity index 89% rename from src/core/mapping/base_mapper.cc rename to src/core/mapping/detail/base_mapper.cc index aea7b52330..ebcc09fe34 100644 --- a/src/core/mapping/base_mapper.cc +++ b/src/core/mapping/detail/base_mapper.cc @@ -14,23 +14,19 @@ * */ -#include -#include -#include +#include "core/mapping/detail/base_mapper.h" -#include "legion/legion_mapping.h" -#include "mappers/mapping_utilities.h" - -#include "core/data/store.h" -#include "core/mapping/base_mapper.h" -#include "core/mapping/instance_manager.h" +#include "core/mapping/detail/instance_manager.h" +#include "core/mapping/detail/mapping.h" +#include "core/mapping/detail/operation.h" +#include "core/mapping/detail/store.h" #include "core/mapping/operation.h" -#include "core/runtime/projection.h" -#include "core/runtime/shard.h" +#include "core/runtime/detail/projection.h" +#include "core/runtime/detail/shard.h" #include "core/utilities/linearize.h" -#include "legate_defines.h" +#include "mappers/mapping_utilities.h" -namespace legate::mapping { +namespace legate::mapping::detail { namespace { @@ -70,19 +66,19 @@ std::string log_mappable(const Legion::Mappable& mappable, bool prefix_only = fa BaseMapper::BaseMapper(mapping::Mapper* legate_mapper, Legion::Mapping::MapperRuntime* _mapper_runtime, - const LibraryContext* _context) + const legate::detail::Library* _library) : Mapper(_mapper_runtime), legate_mapper_(legate_mapper), mapper_runtime(_mapper_runtime), legion_machine(Legion::Machine::get_machine()), - context(_context), + library(_library), logger(create_logger_name().c_str()), local_instances(InstanceManager::get_instance_manager()), reduction_instances(ReductionInstanceManager::get_instance_manager()), - machine() + local_machine() { std::stringstream ss; - ss << context->get_library_name() << " on Node " << machine.local_node; + ss << library->get_library_name() << " on Node " << local_machine.node_id; mapper_name = ss.str(); legate_mapper_->set_machine(this); @@ -105,7 +101,7 @@ BaseMapper::~BaseMapper() logger.print( "%s used %ld bytes of %s memory %llx with " "%ld total bytes (%.2g%%)", - context->get_library_name().c_str(), + library->get_library_name().c_str(), pair.second, memory_kinds[mem.kind()], mem.id, @@ -118,7 +114,7 @@ BaseMapper::~BaseMapper() std::string BaseMapper::create_logger_name() const { std::stringstream ss; - ss << context->get_library_name() << ".mapper"; + ss << library->get_library_name() << ".mapper"; return ss.str(); } @@ -133,7 +129,7 @@ void BaseMapper::select_task_options(const Legion::Mapping::MapperContext ctx, const Legion::Task& task, TaskOptions& output) { - Task legate_task(&task, context, runtime, ctx); + Task legate_task(&task, library, runtime, ctx); #ifdef LEGATE_USE_COLLECTIVE auto hi = task.index_domain.hi(); auto lo = task.index_domain.lo(); @@ -159,7 +155,7 @@ void BaseMapper::select_task_options(const Legion::Mapping::MapperContext ctx, #endif - auto& machine_desc = legate_task.machine_desc(); + auto& machine_desc = legate_task.machine(); auto all_targets = machine_desc.valid_targets(); std::vector options; @@ -172,9 +168,9 @@ void BaseMapper::select_task_options(const Legion::Mapping::MapperContext ctx, LEGATE_ABORT; } - auto target = legate_mapper_->task_target(legate_task, options); + auto target = legate_mapper_->task_target(mapping::Task(&legate_task), options); // The initial processor just needs to have the same kind as the eventual target of this task - output.initial_proc = machine.procs(target).front(); + output.initial_proc = local_machine.procs(target).front(); // We never want valid instances output.valid_instances = false; @@ -193,10 +189,10 @@ void BaseMapper::slice_task(const Legion::Mapping::MapperContext ctx, const SliceTaskInput& input, SliceTaskOutput& output) { - Task legate_task(&task, context, runtime, ctx); + Task legate_task(&task, library, runtime, ctx); - auto& machine_desc = legate_task.machine_desc(); - auto local_range = machine.slice(legate_task.target(), machine_desc); + auto& machine_desc = legate_task.machine(); + auto local_range = local_machine.slice(legate_task.target(), machine_desc); Legion::ProjectionID projection = 0; for (auto& req : task.regions) @@ -204,7 +200,7 @@ void BaseMapper::slice_task(const Legion::Mapping::MapperContext ctx, projection = req.projection; break; } - auto key_functor = find_legate_projection_functor(projection); + auto key_functor = legate::detail::find_legate_projection_functor(projection); // For multi-node cases we should already have been sharded so we // should just have one or a few points here on this node, so iterate @@ -283,7 +279,7 @@ void BaseMapper::map_task(const Legion::Mapping::MapperContext ctx, #endif output.chosen_variant = *variant; - Task legate_task(&task, context, runtime, ctx); + Task legate_task(&task, library, runtime, ctx); if (task.is_index_space) // If this is an index task, point tasks already have the right targets, so we just need to @@ -292,7 +288,7 @@ void BaseMapper::map_task(const Legion::Mapping::MapperContext ctx, else { // If this is a single task, here is the right place to compute the final target processor auto local_range = - machine.slice(legate_task.target(), legate_task.machine_desc(), task.local_function); + local_machine.slice(legate_task.target(), legate_task.machine(), task.local_function); #ifdef DEBUG_LEGATE assert(!local_range.empty()); #endif @@ -301,38 +297,33 @@ void BaseMapper::map_task(const Legion::Mapping::MapperContext ctx, const auto& options = default_store_targets(task.target_proc.kind()); - auto mappings = legate_mapper_->store_mappings(legate_task, options); + auto client_mappings = legate_mapper_->store_mappings(mapping::Task(&legate_task), options); - auto validate_colocation = [this](const auto& mapping) { - if (mapping.stores.empty()) { - logger.error("Store mapping must contain at least one store"); - LEGATE_ABORT; - } - if (mapping.stores.size() > 1 && mapping.policy.ordering.relative) { - logger.error("Colocation with relative dimension ordering is illegal"); - LEGATE_ABORT; - } - auto& first_store = mapping.stores.front(); - for (auto it = mapping.stores.begin() + 1; it != mapping.stores.end(); ++it) { - if (!it->get().can_colocate_with(first_store)) { + auto validate_colocation = [this](const auto* mapping) { + auto* first_store = mapping->stores.front(); + for (auto it = mapping->stores.begin() + 1; it != mapping->stores.end(); ++it) { + if (!(*it)->can_colocate_with(*first_store)) { logger.error("Mapper %s tried to colocate stores that cannot colocate", get_mapper_name()); LEGATE_ABORT; } } - assert(!(mapping.for_future() || mapping.for_unbound_store()) || mapping.stores.size() == 1); + assert(!(mapping->for_future() || mapping->for_unbound_store()) || mapping->stores.size() == 1); }; #ifdef DEBUG_LEGATE - for (auto& mapping : mappings) validate_colocation(mapping); + for (auto& client_mapping : client_mappings) validate_colocation(client_mapping.impl()); #endif - std::vector for_futures, for_unbound_stores, for_stores; + std::vector> for_futures; + std::vector> for_unbound_stores; + std::vector> for_stores; std::set mapped_futures; std::set mapped_regions; - for (auto& mapping : mappings) { - if (mapping.for_future()) { - auto fut_idx = mapping.store().future_index(); + for (auto& client_mapping : client_mappings) { + auto* mapping = client_mapping.impl(); + if (mapping->for_future()) { + auto fut_idx = mapping->store()->future_index(); // Only need to map Future-backed Stores corresponding to inputs (i.e. one of task.futures) if (fut_idx >= task.futures.size()) continue; if (mapped_futures.count(fut_idx) > 0) { @@ -340,26 +331,27 @@ void BaseMapper::map_task(const Legion::Mapping::MapperContext ctx, LEGATE_ABORT; } mapped_futures.insert(fut_idx); - for_futures.push_back(std::move(mapping)); - } else if (mapping.for_unbound_store()) { - mapped_regions.insert(mapping.store().unique_region_field_id()); - for_unbound_stores.push_back(std::move(mapping)); + for_futures.emplace_back(client_mapping.release()); + } else if (mapping->for_unbound_store()) { + mapped_regions.insert(mapping->store()->unique_region_field_id()); + for_unbound_stores.emplace_back(client_mapping.release()); } else { - for (auto& store : mapping.stores) - mapped_regions.insert(store.get().unique_region_field_id()); - for_stores.push_back(std::move(mapping)); + for (const auto* store : mapping->stores) + mapped_regions.insert(store->unique_region_field_id()); + for_stores.emplace_back(client_mapping.release()); } } + client_mappings.clear(); auto check_consistency = [this](const auto& mappings) { std::map policies; - for (const auto& mapping : mappings) - for (auto& store : mapping.stores) { - auto key = store.get().unique_region_field_id(); + for (auto& mapping : mappings) + for (auto& store : mapping->stores) { + auto key = store->unique_region_field_id(); auto finder = policies.find(key); if (policies.end() == finder) - policies[key] = mapping.policy; - else if (mapping.policy != finder->second) { + policies[key] = mapping->policy; + else if (mapping->policy != finder->second) { logger.error("Mapper %s returned inconsistent store mappings", get_mapper_name()); LEGATE_ABORT; } @@ -373,7 +365,7 @@ void BaseMapper::map_task(const Legion::Mapping::MapperContext ctx, auto default_option = options.front(); auto generate_default_mappings = [&](auto& stores, bool exact) { for (auto& store : stores) { - auto mapping = StoreMapping::default_mapping(store, default_option, exact); + auto mapping = StoreMapping::default_mapping(&store, default_option, exact); if (store.is_future()) { auto fut_idx = store.future_index(); // Only need to map Future-backed Stores corresponding to inputs (i.e. one of task.futures) @@ -407,20 +399,21 @@ void BaseMapper::map_task(const Legion::Mapping::MapperContext ctx, // Map future-backed stores output.future_locations.resize(mapped_futures.size()); for (auto& mapping : for_futures) { - auto fut_idx = mapping.store().future_index(); - StoreTarget target = mapping.policy.target; + auto fut_idx = mapping->store()->future_index(); + StoreTarget target = mapping->policy.target; #ifdef LEGATE_NO_FUTURES_ON_FB if (target == StoreTarget::FBMEM) target = StoreTarget::ZCMEM; #endif - output.future_locations[fut_idx] = machine.get_memory(task.target_proc, target); + output.future_locations[fut_idx] = local_machine.get_memory(task.target_proc, target); } // Map unbound stores auto map_unbound_stores = [&](auto& mappings) { for (auto& mapping : mappings) { - auto req_idx = mapping.requirement_index(); - output.output_targets[req_idx] = machine.get_memory(task.target_proc, mapping.policy.target); - auto ndim = mapping.store().dim(); + auto req_idx = mapping->requirement_index(); + output.output_targets[req_idx] = + local_machine.get_memory(task.target_proc, mapping->policy.target); + auto ndim = mapping->store()->dim(); // FIXME: Unbound stores can have more than one dimension later std::vector dimension_ordering; for (int32_t dim = ndim - 1; dim >= 0; --dim) @@ -453,7 +446,7 @@ void BaseMapper::map_replicate_task(const Legion::Mapping::MapperContext ctx, void BaseMapper::map_legate_stores(const Legion::Mapping::MapperContext ctx, const Legion::Mappable& mappable, - std::vector& mappings, + std::vector>& mappings, Processor target_proc, OutputMap& output_map) { @@ -462,8 +455,8 @@ void BaseMapper::map_legate_stores(const Legion::Mapping::MapperContext ctx, std::vector instances; for (auto& mapping : mappings) { Legion::Mapping::PhysicalInstance result = NO_INST; - auto reqs = mapping.requirements(); - while (map_legate_store(ctx, mappable, mapping, reqs, target_proc, result, can_fail)) { + auto reqs = mapping->requirements(); + while (map_legate_store(ctx, mappable, *mapping, reqs, target_proc, result, can_fail)) { if (NO_INST == result) { #ifdef DEBUG_LEGATE assert(can_fail); @@ -473,7 +466,7 @@ void BaseMapper::map_legate_stores(const Legion::Mapping::MapperContext ctx, } #ifdef DEBUG_LEGATE std::stringstream reqs_ss; - for (auto req_idx : mapping.requirement_indices()) reqs_ss << " " << req_idx; + for (auto req_idx : mapping->requirement_indices()) reqs_ss << " " << req_idx; #endif if (runtime->acquire_instance(ctx, result)) { #ifdef DEBUG_LEGATE @@ -502,7 +495,7 @@ void BaseMapper::map_legate_stores(const Legion::Mapping::MapperContext ctx, for (uint32_t idx = 0; idx < mappings.size(); ++idx) { auto& mapping = mappings[idx]; auto& instance = instances[idx]; - for (auto& req : mapping.requirements()) output_map[req]->push_back(instance); + for (auto& req : mapping->requirements()) output_map[req]->push_back(instance); } return true; }; @@ -510,7 +503,7 @@ void BaseMapper::map_legate_stores(const Legion::Mapping::MapperContext ctx, // We can retry the mapping with tightened policies only if at least one of the policies // is lenient bool can_fail = false; - for (auto& mapping : mappings) can_fail = can_fail || !mapping.policy.exact; + for (auto& mapping : mappings) can_fail = can_fail || !mapping->policy.exact; if (!try_mapping(can_fail)) { #ifdef DEBUG_LEGATE @@ -527,24 +520,24 @@ void BaseMapper::map_legate_stores(const Legion::Mapping::MapperContext ctx, } void BaseMapper::tighten_write_policies(const Legion::Mappable& mappable, - std::vector& mappings) + std::vector>& mappings) { for (auto& mapping : mappings) { // If the policy is exact, there's nothing we can tighten - if (mapping.policy.exact) continue; + if (mapping->policy.exact) continue; int32_t priv = LEGION_NO_ACCESS; - for (auto* req : mapping.requirements()) priv |= req->privilege; + for (auto* req : mapping->requirements()) priv |= req->privilege; // We tighten only write requirements if (!(priv & LEGION_WRITE_PRIV)) continue; #ifdef DEBUG_LEGATE std::stringstream reqs_ss; - for (auto req_idx : mapping.requirement_indices()) reqs_ss << " " << req_idx; + for (auto req_idx : mapping->requirement_indices()) reqs_ss << " " << req_idx; logger.debug() << log_mappable(mappable) << ": tightened mapping policy for reqs:" << reqs_ss.str(); #endif - mapping.policy.exact = true; + mapping->policy.exact = true; } } @@ -561,7 +554,7 @@ bool BaseMapper::map_legate_store(const Legion::Mapping::MapperContext ctx, const auto& policy = mapping.policy; std::vector regions; for (auto* req : reqs) regions.push_back(req->region); - auto target_memory = machine.get_memory(target_proc, policy.target); + auto target_memory = local_machine.get_memory(target_proc, policy.target); auto redop = (*reqs.begin())->redop; #ifdef DEBUG_LEGATE @@ -848,7 +841,7 @@ void find_source_instance_bandwidth( std::map& source_memory_bandwidths, /* inout */ const Legion::Mapping::PhysicalInstance& source_instance, const Memory& target_memory, - const Legion::Machine& machine) + const Legion::Machine& legion_machine) { Memory source_memory = source_instance.get_location(); auto finder = source_memory_bandwidths.find(source_memory); @@ -856,7 +849,7 @@ void find_source_instance_bandwidth( uint32_t bandwidth{0}; if (source_memory_bandwidths.end() == finder) { std::vector affinities; - machine.get_mem_mem_affinity( + legion_machine.get_mem_mem_affinity( affinities, source_memory, target_memory, false /*not just local affinities*/); // affinities being empty means that there's no direct channel between the source // and target memories, in which case we assign the smallest bandwidth @@ -949,10 +942,10 @@ void BaseMapper::map_inline(const Legion::Mapping::MapperContext ctx, MapInlineOutput& output) { Processor target_proc{Processor::NO_PROC}; - if (machine.has_omps()) - target_proc = machine.omps().front(); + if (local_machine.has_omps()) + target_proc = local_machine.omps().front(); else - target_proc = machine.cpus().front(); + target_proc = local_machine.cpus().front(); auto store_target = default_store_targets(target_proc.kind()).front(); @@ -961,12 +954,12 @@ void BaseMapper::map_inline(const Legion::Mapping::MapperContext ctx, #endif Store store(mapper_runtime, ctx, &inline_op.requirement); - std::vector mappings; - mappings.push_back(StoreMapping::default_mapping(store, store_target, false)); + std::vector> mappings; + mappings.push_back(StoreMapping::default_mapping(&store, store_target, false)); std::map*> output_map; - for (auto* req : mappings.front().requirements()) output_map[req] = &output.chosen_instances; + for (auto* req : mappings.front()->requirements()) output_map[req] = &output.chosen_instances; map_legate_stores(ctx, inline_op, mappings, target_proc, output_map); } @@ -994,7 +987,7 @@ void BaseMapper::map_copy(const Legion::Mapping::MapperContext ctx, MapCopyOutput& output) { Copy legate_copy(©, runtime, ctx); - auto& machine_desc = legate_copy.machine_desc(); + auto& machine_desc = legate_copy.machine(); auto copy_target = [&]() { // If we're mapping an indirect copy and have data resident in GPU memory, // map everything to CPU memory, as indirect copies on GPUs are currently @@ -1014,7 +1007,7 @@ void BaseMapper::map_copy(const Legion::Mapping::MapperContext ctx, return valid_targets.front(); }(); - auto local_range = machine.slice(copy_target, machine_desc, true); + auto local_range = local_machine.slice(copy_target, machine_desc, true); Processor target_proc; if (copy.is_index_space) { Domain sharding_domain = copy.index_domain; @@ -1024,7 +1017,7 @@ void BaseMapper::map_copy(const Legion::Mapping::MapperContext ctx, // FIXME: We might later have non-identity projections for copy requirements, // in which case we should find the key store and use its projection functor // for the linearization - auto* key_functor = find_legate_projection_functor(0); + auto* key_functor = legate::detail::find_legate_projection_functor(0); auto lo = key_functor->project_point(sharding_domain.lo(), sharding_domain); auto hi = key_functor->project_point(sharding_domain.hi(), sharding_domain); auto p = key_functor->project_point(copy.index_point, sharding_domain); @@ -1059,16 +1052,16 @@ void BaseMapper::map_copy(const Legion::Mapping::MapperContext ctx, output_map[©.dst_indirect_requirements.front()] = &output.dst_indirect_instances; } - std::vector mappings; + std::vector> mappings; for (auto& store : legate_copy.inputs()) - mappings.push_back(StoreMapping::default_mapping(store, store_target, false)); + mappings.push_back(StoreMapping::default_mapping(&store, store_target, false)); for (auto& store : legate_copy.outputs()) - mappings.push_back(StoreMapping::default_mapping(store, store_target, false)); + mappings.push_back(StoreMapping::default_mapping(&store, store_target, false)); for (auto& store : legate_copy.input_indirections()) - mappings.push_back(StoreMapping::default_mapping(store, store_target, false)); + mappings.push_back(StoreMapping::default_mapping(&store, store_target, false)); for (auto& store : legate_copy.output_indirections()) - mappings.push_back(StoreMapping::default_mapping(store, store_target, false)); + mappings.push_back(StoreMapping::default_mapping(&store, store_target, false)); map_legate_stores(ctx, copy, mappings, target_proc, output_map); } @@ -1199,10 +1192,10 @@ void BaseMapper::map_partition(const Legion::Mapping::MapperContext ctx, MapPartitionOutput& output) { Processor target_proc{Processor::NO_PROC}; - if (machine.has_omps()) - target_proc = machine.omps().front(); + if (local_machine.has_omps()) + target_proc = local_machine.omps().front(); else - target_proc = machine.cpus().front(); + target_proc = local_machine.cpus().front(); auto store_target = default_store_targets(target_proc.kind()).front(); @@ -1211,12 +1204,12 @@ void BaseMapper::map_partition(const Legion::Mapping::MapperContext ctx, #endif Store store(mapper_runtime, ctx, &partition.requirement); - std::vector mappings; - mappings.push_back(StoreMapping::default_mapping(store, store_target, false)); + std::vector> mappings; + mappings.push_back(StoreMapping::default_mapping(&store, store_target, false)); std::map*> output_map; - for (auto* req : mappings.front().requirements()) output_map[req] = &output.chosen_instances; + for (auto* req : mappings.front()->requirements()) output_map[req] = &output.chosen_instances; map_legate_stores(ctx, partition, mappings, target_proc, output_map); } @@ -1267,7 +1260,7 @@ void BaseMapper::map_future_map_reduction(const Legion::Mapping::MapperContext c { output.serdez_upper_bound = LEGATE_MAX_SIZE_SCALAR_RETURN; - if (machine.has_gpus()) { + if (local_machine.has_gpus()) { // TODO: It's been reported that blindly mapping target instances of future map reductions // to framebuffers hurts performance. Until we find a better mapping policy, we guard // the current policy with a macro. @@ -1276,14 +1269,16 @@ void BaseMapper::map_future_map_reduction(const Legion::Mapping::MapperContext c // If this was joining exceptions, we should put instances on a host-visible memory // because they need serdez if (input.tag == LEGATE_CORE_JOIN_EXCEPTION_TAG) - output.destination_memories.push_back(machine.zerocopy_memory()); + output.destination_memories.push_back(local_machine.zerocopy_memory()); else - for (auto& pair : machine.frame_buffers()) output.destination_memories.push_back(pair.second); + for (auto& pair : local_machine.frame_buffers()) + output.destination_memories.push_back(pair.second); #else - output.destination_memories.push_back(machine.zerocopy_memory()); + output.destination_memories.push_back(local_machine.zerocopy_memory()); #endif - } else if (machine.has_socket_memory()) - for (auto& pair : machine.socket_memories()) output.destination_memories.push_back(pair.second); + } else if (local_machine.has_socket_memory()) + for (auto& pair : local_machine.socket_memories()) + output.destination_memories.push_back(pair.second); } void BaseMapper::select_tunable_value(const Legion::Mapping::MapperContext ctx, @@ -1367,4 +1362,4 @@ void BaseMapper::handle_task_result(const Legion::Mapping::MapperContext ctx, LEGATE_ABORT; } -} // namespace legate::mapping +} // namespace legate::mapping::detail diff --git a/src/core/mapping/base_mapper.h b/src/core/mapping/detail/base_mapper.h similarity index 96% rename from src/core/mapping/base_mapper.h rename to src/core/mapping/detail/base_mapper.h index 0e08244316..369275ce0d 100644 --- a/src/core/mapping/base_mapper.h +++ b/src/core/mapping/detail/base_mapper.h @@ -23,21 +23,21 @@ #include "legion.h" #include "core/data/scalar.h" -#include "core/mapping/mapping.h" -#include "core/runtime/context.h" +#include "core/mapping/detail/machine.h" +#include "core/mapping/detail/mapping.h" +#include "core/runtime/detail/library.h" #include "core/utilities/typedefs.h" -namespace legate::mapping { +namespace legate::mapping::detail { class InstanceManager; -class Machine; class ReductionInstanceManager; class BaseMapper : public Legion::Mapping::Mapper, public MachineQueryInterface { public: BaseMapper(mapping::Mapper* legate_mapper, Legion::Mapping::MapperRuntime* mapper_runtime, - const LibraryContext* context); + const legate::detail::Library* library); virtual ~BaseMapper(); private: @@ -51,10 +51,10 @@ class BaseMapper : public Legion::Mapping::Mapper, public MachineQueryInterface public: // MachineQueryInterface - virtual const std::vector& cpus() const override { return machine.cpus(); } - virtual const std::vector& gpus() const override { return machine.gpus(); } - virtual const std::vector& omps() const override { return machine.omps(); } - virtual uint32_t total_nodes() const override { return machine.total_nodes; } + virtual const std::vector& cpus() const override { return local_machine.cpus(); } + virtual const std::vector& gpus() const override { return local_machine.gpus(); } + virtual const std::vector& omps() const override { return local_machine.omps(); } + virtual uint32_t total_nodes() const override { return local_machine.total_nodes; } public: virtual const char* get_mapper_name() const override; @@ -254,11 +254,11 @@ class BaseMapper : public Legion::Mapping::Mapper, public MachineQueryInterface std::map*>; void map_legate_stores(const Legion::Mapping::MapperContext ctx, const Legion::Mappable& mappable, - std::vector& mappings, + std::vector>& mappings, Processor target_proc, OutputMap& output_map); void tighten_write_policies(const Legion::Mappable& mappable, - std::vector& mappings); + std::vector>& mappings); bool map_legate_store(const Legion::Mapping::MapperContext ctx, const Legion::Mappable& mappable, const StoreMapping& mapping, @@ -296,7 +296,7 @@ class BaseMapper : public Legion::Mapping::Mapper, public MachineQueryInterface public: Legion::Mapping::MapperRuntime* const mapper_runtime; const Legion::Machine legion_machine; - const LibraryContext* context; + const legate::detail::Library* library; Legion::Logger logger; private: @@ -309,7 +309,7 @@ class BaseMapper : public Legion::Mapping::Mapper, public MachineQueryInterface protected: InstanceManager* local_instances; ReductionInstanceManager* reduction_instances; - Machine machine; + LocalMachine local_machine; }; -} // namespace legate::mapping +} // namespace legate::mapping::detail diff --git a/src/core/mapping/core_mapper.cc b/src/core/mapping/detail/core_mapper.cc similarity index 92% rename from src/core/mapping/core_mapper.cc rename to src/core/mapping/detail/core_mapper.cc index 46f527d62e..ebb2a176a6 100644 --- a/src/core/mapping/core_mapper.cc +++ b/src/core/mapping/detail/core_mapper.cc @@ -15,10 +15,10 @@ */ #include "env_defaults.h" -#include "legate.h" -#include "core/mapping/core_mapper.h" -#include "core/mapping/machine.h" +#include "core/mapping/detail/core_mapper.h" + +#include "core/mapping/detail/machine.h" #ifdef LEGATE_USE_CUDA #include "core/comm/comm_nccl.h" #endif @@ -40,7 +40,7 @@ uint32_t extract_env(const char* env_name, const uint32_t default_value, const u } // namespace legate -namespace legate::mapping { +namespace legate::mapping::detail { // This is a custom mapper implementation that only has to map // start-up tasks associated with the Legate core, no one else @@ -60,7 +60,7 @@ class CoreMapper : public Mapper { legate::Scalar tunable_value(legate::TunableID tunable_id) override; private: - const Machine machine; + const LocalMachine machine; const int64_t min_gpu_chunk; const int64_t min_cpu_chunk; const int64_t min_omp_chunk; @@ -95,13 +95,14 @@ CoreMapper::CoreMapper() void CoreMapper::set_machine(const legate::mapping::MachineQueryInterface* m) {} -TaskTarget CoreMapper::task_target(const Task& task, const std::vector& options) +TaskTarget CoreMapper::task_target(const legate::mapping::Task& task, + const std::vector& options) { return options.front(); } -std::vector CoreMapper::store_mappings(const Task& task, - const std::vector& options) +std::vector CoreMapper::store_mappings( + const legate::mapping::Task& task, const std::vector& options) { return {}; } @@ -174,4 +175,4 @@ Scalar CoreMapper::tunable_value(TunableID tunable_id) std::unique_ptr create_core_mapper() { return std::make_unique(); } -} // namespace legate::mapping +} // namespace legate::mapping::detail diff --git a/src/core/mapping/core_mapper.h b/src/core/mapping/detail/core_mapper.h similarity index 90% rename from src/core/mapping/core_mapper.h rename to src/core/mapping/detail/core_mapper.h index e94ce8245f..a7268c0697 100644 --- a/src/core/mapping/core_mapper.h +++ b/src/core/mapping/detail/core_mapper.h @@ -18,8 +18,8 @@ #include "core/mapping/mapping.h" -namespace legate::mapping { +namespace legate::mapping::detail { std::unique_ptr create_core_mapper(); -} // namespace legate::mapping +} // namespace legate::mapping::detail diff --git a/src/core/mapping/default_mapper.cc b/src/core/mapping/detail/default_mapper.cc similarity index 68% rename from src/core/mapping/default_mapper.cc rename to src/core/mapping/detail/default_mapper.cc index fac10b8051..b29d1ded6e 100644 --- a/src/core/mapping/default_mapper.cc +++ b/src/core/mapping/detail/default_mapper.cc @@ -14,20 +14,21 @@ * */ -#include "core/mapping/default_mapper.h" +#include "core/mapping/detail/default_mapper.h" -namespace legate::mapping { +namespace legate::mapping::detail { // Default mapper doesn't use the machine query interface void DefaultMapper::set_machine(const MachineQueryInterface* machine) {} -TaskTarget DefaultMapper::task_target(const Task& task, const std::vector& options) +TaskTarget DefaultMapper::task_target(const mapping::Task& task, + const std::vector& options) { return options.front(); } -std::vector DefaultMapper::store_mappings(const Task& task, - const std::vector& options) +std::vector DefaultMapper::store_mappings( + const mapping::Task& task, const std::vector& options) { return {}; } @@ -38,4 +39,4 @@ Scalar DefaultMapper::tunable_value(TunableID tunable_id) return Scalar(0); } -} // namespace legate::mapping +} // namespace legate::mapping::detail diff --git a/src/core/mapping/default_mapper.h b/src/core/mapping/detail/default_mapper.h similarity index 71% rename from src/core/mapping/default_mapper.h rename to src/core/mapping/detail/default_mapper.h index 3fca215c22..80d0f5d45b 100644 --- a/src/core/mapping/default_mapper.h +++ b/src/core/mapping/detail/default_mapper.h @@ -18,7 +18,7 @@ #pragma once -namespace legate::mapping { +namespace legate::mapping::detail { class DefaultMapper : public Mapper { public: @@ -26,10 +26,11 @@ class DefaultMapper : public Mapper { public: void set_machine(const MachineQueryInterface* machine) override; - TaskTarget task_target(const Task& task, const std::vector& options) override; - std::vector store_mappings(const Task& task, - const std::vector& options) override; + TaskTarget task_target(const mapping::Task& task, + const std::vector& options) override; + std::vector store_mappings( + const mapping::Task& task, const std::vector& options) override; Scalar tunable_value(TunableID tunable_id) override; }; -} // namespace legate::mapping +} // namespace legate::mapping::detail diff --git a/src/core/mapping/instance_manager.cc b/src/core/mapping/detail/instance_manager.cc similarity index 99% rename from src/core/mapping/instance_manager.cc rename to src/core/mapping/detail/instance_manager.cc index 5c7d9f232c..5269183722 100644 --- a/src/core/mapping/instance_manager.cc +++ b/src/core/mapping/detail/instance_manager.cc @@ -14,10 +14,10 @@ * */ -#include "core/mapping/instance_manager.h" +#include "core/mapping/detail/instance_manager.h" #include "core/utilities/dispatch.h" -namespace legate::mapping { +namespace legate::mapping::detail { using RegionGroupP = std::shared_ptr; @@ -479,4 +479,4 @@ void ReductionInstanceManager::erase(Instance inst) return manager; } -} // namespace legate::mapping +} // namespace legate::mapping::detail diff --git a/src/core/mapping/instance_manager.h b/src/core/mapping/detail/instance_manager.h similarity index 98% rename from src/core/mapping/instance_manager.h rename to src/core/mapping/detail/instance_manager.h index eb7ac6877b..f1755594ac 100644 --- a/src/core/mapping/instance_manager.h +++ b/src/core/mapping/detail/instance_manager.h @@ -23,7 +23,7 @@ #include "core/mapping/mapping.h" -namespace legate::mapping { +namespace legate::mapping::detail { // This class represents a set of regions that colocate in an instance struct RegionGroup { @@ -234,4 +234,4 @@ class ReductionInstanceManager : public BaseInstanceManager { std::map instance_sets_{}; }; -} // namespace legate::mapping +} // namespace legate::mapping::detail diff --git a/src/core/mapping/detail/machine.cc b/src/core/mapping/detail/machine.cc new file mode 100644 index 0000000000..db186f38ba --- /dev/null +++ b/src/core/mapping/detail/machine.cc @@ -0,0 +1,346 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/mapping/detail/machine.h" +#include "core/utilities/detail/buffer_builder.h" +#include "realm/network.h" + +namespace legate::mapping::detail { + +/////////////////////////////////////////// +// legate::mapping::detail::Machine +////////////////////////////////////////// + +Machine::Machine(const std::map& ranges) : processor_ranges(ranges) +{ + for (auto& [target, processor_range] : processor_ranges) + if (!processor_range.empty()) { + preferred_target = target; + return; + } +} + +Machine::Machine(std::map&& ranges) + : processor_ranges(std::move(ranges)) +{ + for (auto& [target, processor_range] : processor_ranges) + if (!processor_range.empty()) { + preferred_target = target; + return; + } +} + +const ProcessorRange& Machine::processor_range() const { return processor_range(preferred_target); } +static const ProcessorRange EMPTY_RANGE{}; + +const ProcessorRange& Machine::processor_range(TaskTarget target) const +{ + auto finder = processor_ranges.find(target); + if (finder == processor_ranges.end()) { return EMPTY_RANGE; } + return finder->second; +} + +std::vector Machine::valid_targets() const +{ + std::vector result; + for (auto& [target, _] : processor_ranges) result.push_back(target); + return result; +} + +std::vector Machine::valid_targets_except(const std::set& to_exclude) const +{ + std::vector result; + for (auto& [target, _] : processor_ranges) + if (to_exclude.find(target) == to_exclude.end()) result.push_back(target); + return result; +} + +uint32_t Machine::count() const { return count(preferred_target); } + +uint32_t Machine::count(TaskTarget target) const { return processor_range(target).count(); } + +std::string Machine::to_string() const +{ + std::stringstream ss; + ss << "Machine(preferred_target: " << preferred_target; + for (auto& [kind, range] : processor_ranges) ss << ", " << kind << ": " << range.to_string(); + ss << ")"; + return ss.str(); +} + +void Machine::pack(legate::detail::BufferBuilder& buffer) const +{ + buffer.pack(static_cast(preferred_target)); + buffer.pack(processor_ranges.size()); + for (auto& [target, processor_range] : processor_ranges) { + buffer.pack(static_cast(target)); + buffer.pack(processor_range.low); + buffer.pack(processor_range.high); + buffer.pack(processor_range.per_node_count); + } +} + +Machine Machine::only(TaskTarget target) const { return only(std::vector({target})); } + +Machine Machine::only(const std::vector& targets) const +{ + std::map new_processor_ranges; + for (auto t : targets) new_processor_ranges.insert({t, processor_range(t)}); + + return Machine(new_processor_ranges); +} + +Machine Machine::slice(uint32_t from, uint32_t to, TaskTarget target, bool keep_others) const +{ + if (keep_others) { + std::map new_ranges(processor_ranges); + new_ranges[target] = processor_range(target).slice(from, to); + return Machine(std::move(new_ranges)); + } else + return Machine({{target, processor_range(target).slice(from, to)}}); +} + +Machine Machine::slice(uint32_t from, uint32_t to, bool keep_others) const +{ + return slice(from, to, preferred_target, keep_others); +} + +Machine Machine::operator[](TaskTarget target) const { return only(target); } + +Machine Machine::operator[](const std::vector& targets) const { return only(targets); } + +bool Machine::operator==(const Machine& other) const +{ + if (preferred_target != other.preferred_target) return false; + if (processor_ranges.size() != other.processor_ranges.size()) return false; + for (auto const& r : processor_ranges) { + auto finder = other.processor_ranges.find(r.first); + if (finder == other.processor_ranges.end() || r.second != finder->second) return false; + } + return true; +} + +bool Machine::operator!=(const Machine& other) const { return !(*this == other); } + +Machine Machine::operator&(const Machine& other) const +{ + std::map new_processor_ranges; + for (const auto& [target, range] : processor_ranges) { + auto finder = other.processor_ranges.find(target); + if (finder != other.processor_ranges.end()) { + new_processor_ranges[target] = finder->second & range; + } + } + return Machine(new_processor_ranges); +} + +bool Machine::empty() const +{ + for (const auto& r : processor_ranges) + if (!r.second.empty()) return false; + return true; +} + +std::ostream& operator<<(std::ostream& stream, const Machine& machine) +{ + stream << machine.to_string(); + return stream; +} + +/////////////////////////////////////////// +// legate::mapping::LocalProcessorRange +/////////////////////////////////////////// + +LocalProcessorRange::LocalProcessorRange() : offset_(0), total_proc_count_(0), procs_() {} + +LocalProcessorRange::LocalProcessorRange(const std::vector& procs) + : offset_(0), total_proc_count_(procs.size()), procs_(procs.data(), procs.size()) +{ +} + +LocalProcessorRange::LocalProcessorRange(uint32_t offset, + uint32_t total_proc_count, + const Processor* local_procs, + size_t num_local_procs) + : offset_(offset), total_proc_count_(total_proc_count), procs_(local_procs, num_local_procs) +{ +} + +const Processor& LocalProcessorRange::operator[](uint32_t idx) const +{ + auto local_idx = (idx % total_proc_count_) - offset_; +#ifdef DEBUG_LEGATE + assert(local_idx < procs_.size()); +#endif + return procs_[local_idx]; +} + +std::string LocalProcessorRange::to_string() const +{ + std::stringstream ss; + ss << "{offset: " << offset_ << ", total processor count: " << total_proc_count_ + << ", processors: "; + for (uint32_t idx = 0; idx < procs_.size(); ++idx) ss << procs_[idx] << ","; + ss << "}"; + return std::move(ss).str(); +} + +std::ostream& operator<<(std::ostream& stream, const LocalProcessorRange& range) +{ + stream << range.to_string(); + return stream; +} + +/////////////////////////////////////////// +// legate::mapping::LocalMachine +/////////////////////////////////////////// +LocalMachine::LocalMachine() + : node_id(Realm::Network::my_node_id), + total_nodes(Legion::Machine::get_machine().get_address_space_count()) +{ + auto legion_machine = Legion::Machine::get_machine(); + Legion::Machine::ProcessorQuery procs(legion_machine); + // Query to find all our local processors + procs.local_address_space(); + for (auto proc : procs) { + switch (proc.kind()) { + case Processor::LOC_PROC: { + cpus_.push_back(proc); + continue; + } + case Processor::TOC_PROC: { + gpus_.push_back(proc); + continue; + } + case Processor::OMP_PROC: { + omps_.push_back(proc); + continue; + } + default: { + continue; + } + } + } + + // Now do queries to find all our local memories + Legion::Machine::MemoryQuery sysmem(legion_machine); + sysmem.local_address_space().only_kind(Legion::Memory::SYSTEM_MEM); + assert(sysmem.count() > 0); + system_memory_ = sysmem.first(); + + if (!gpus_.empty()) { + Legion::Machine::MemoryQuery zcmem(legion_machine); + zcmem.local_address_space().only_kind(Legion::Memory::Z_COPY_MEM); + assert(zcmem.count() > 0); + zerocopy_memory_ = zcmem.first(); + } + for (auto& gpu : gpus_) { + Legion::Machine::MemoryQuery framebuffer(legion_machine); + framebuffer.local_address_space().only_kind(Legion::Memory::GPU_FB_MEM).best_affinity_to(gpu); + assert(framebuffer.count() > 0); + frame_buffers_[gpu] = framebuffer.first(); + } + for (auto& omp : omps_) { + Legion::Machine::MemoryQuery sockmem(legion_machine); + sockmem.local_address_space().only_kind(Legion::Memory::SOCKET_MEM).best_affinity_to(omp); + // If we have socket memories then use them + if (sockmem.count() > 0) socket_memories_[omp] = sockmem.first(); + // Otherwise we just use the local system memory + else + socket_memories_[omp] = system_memory_; + } +} + +const std::vector& LocalMachine::procs(TaskTarget target) const +{ + switch (target) { + case TaskTarget::GPU: return gpus_; + case TaskTarget::OMP: return omps_; + case TaskTarget::CPU: return cpus_; + default: LEGATE_ABORT; + } + assert(false); + return cpus_; +} + +size_t LocalMachine::total_frame_buffer_size() const +{ + // We assume that all memories of the same kind are symmetric in size + size_t per_node_size = frame_buffers_.size() * frame_buffers_.begin()->second.capacity(); + return per_node_size * total_nodes; +} + +size_t LocalMachine::total_socket_memory_size() const +{ + // We assume that all memories of the same kind are symmetric in size + size_t per_node_size = socket_memories_.size() * socket_memories_.begin()->second.capacity(); + return per_node_size * total_nodes; +} + +bool LocalMachine::has_socket_memory() const +{ + return !socket_memories_.empty() && + socket_memories_.begin()->second.kind() == Legion::Memory::SOCKET_MEM; +} + +LocalProcessorRange LocalMachine::slice(TaskTarget target, + const Machine& machine, + bool fallback_to_global /*=false*/) const +{ + const auto& local_procs = procs(target); + + auto finder = machine.processor_ranges.find(target); + if (machine.processor_ranges.end() == finder) { + if (fallback_to_global) + return LocalProcessorRange(local_procs); + else + return LocalProcessorRange(); + } + + auto& global_range = finder->second; + + uint32_t num_local_procs = local_procs.size(); + uint32_t my_low = num_local_procs * node_id; + ProcessorRange my_range(my_low, my_low + num_local_procs, global_range.per_node_count); + + auto slice = global_range & my_range; + if (slice.empty()) { + if (fallback_to_global) + return LocalProcessorRange(local_procs); + else + return LocalProcessorRange(); + } + + return LocalProcessorRange(slice.low - global_range.low, + global_range.count(), + local_procs.data() + (slice.low - my_low), + slice.count()); +} + +Legion::Memory LocalMachine::get_memory(Processor proc, StoreTarget target) const +{ + switch (target) { + case StoreTarget::SYSMEM: return system_memory_; + case StoreTarget::FBMEM: return frame_buffers_.at(proc); + case StoreTarget::ZCMEM: return zerocopy_memory_; + case StoreTarget::SOCKETMEM: return socket_memories_.at(proc); + default: LEGATE_ABORT; + } + assert(false); + return Legion::Memory::NO_MEMORY; +} + +} // namespace legate::mapping::detail diff --git a/src/core/mapping/detail/machine.h b/src/core/mapping/detail/machine.h new file mode 100644 index 0000000000..1665cdd0a5 --- /dev/null +++ b/src/core/mapping/detail/machine.h @@ -0,0 +1,152 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include "core/mapping/detail/mapping.h" +#include "core/mapping/machine.h" +#include "core/mapping/mapping.h" +#include "core/utilities/detail/buffer_builder.h" +#include "core/utilities/typedefs.h" + +namespace legate::mapping::detail { + +struct Machine { + Machine() {} + Machine(const std::map& processor_ranges); + Machine(std::map&& processor_ranges); + + Machine(const Machine&) = default; + Machine& operator=(const Machine&) = default; + + Machine(Machine&&) = default; + Machine& operator=(Machine&&) = default; + + const ProcessorRange& processor_range() const; + const ProcessorRange& processor_range(TaskTarget target) const; + + std::vector valid_targets() const; + std::vector valid_targets_except(const std::set& to_exclude) const; + + uint32_t count() const; + uint32_t count(TaskTarget target) const; + + std::string to_string() const; + + void pack(legate::detail::BufferBuilder& buffer) const; + + Machine only(TaskTarget target) const; + Machine only(const std::vector& targets) const; + Machine slice(uint32_t from, uint32_t to, TaskTarget target, bool keep_others = false) const; + Machine slice(uint32_t from, uint32_t to, bool keep_others = false) const; + + Machine operator[](TaskTarget target) const; + Machine operator[](const std::vector& targets) const; + bool operator==(const Machine& other) const; + bool operator!=(const Machine& other) const; + Machine operator&(const Machine& other) const; + + bool empty() const; + + TaskTarget preferred_target{TaskTarget::CPU}; + std::map processor_ranges{}; +}; + +std::ostream& operator<<(std::ostream& stream, const Machine& machine); + +class LocalProcessorRange { + public: + LocalProcessorRange(); + LocalProcessorRange(const std::vector& procs); + LocalProcessorRange(uint32_t offset, + uint32_t total_proc_count, + const Processor* local_procs, + size_t num_local_procs); + + public: + const Processor& first() const { return *procs_.begin(); } + const Processor& operator[](uint32_t idx) const; + + public: + bool empty() const { return procs_.size() == 0; } + + public: + std::string to_string() const; + + private: + uint32_t offset_; + uint32_t total_proc_count_; + Span procs_; +}; + +std::ostream& operator<<(std::ostream& stream, const LocalProcessorRange& info); + +// A machine object holding handles to local processors and memories +class LocalMachine { + public: + LocalMachine(); + + public: + const std::vector& cpus() const { return cpus_; } + const std::vector& gpus() const { return gpus_; } + const std::vector& omps() const { return omps_; } + const std::vector& procs(TaskTarget target) const; + + public: + size_t total_cpu_count() const { return total_nodes * cpus_.size(); } + size_t total_gpu_count() const { return total_nodes * gpus_.size(); } + size_t total_omp_count() const { return total_nodes * omps_.size(); } + + public: + size_t total_frame_buffer_size() const; + size_t total_socket_memory_size() const; + + public: + bool has_cpus() const { return !cpus_.empty(); } + bool has_gpus() const { return !gpus_.empty(); } + bool has_omps() const { return !omps_.empty(); } + + public: + bool has_socket_memory() const; + + public: + Memory get_memory(Processor proc, StoreTarget target) const; + Memory system_memory() const { return system_memory_; } + Memory zerocopy_memory() const { return zerocopy_memory_; } + const std::map& frame_buffers() const { return frame_buffers_; } + const std::map& socket_memories() const { return socket_memories_; } + + public: + LocalProcessorRange slice(TaskTarget target, + const Machine& machine, + bool fallback_to_global = false) const; + + public: + const uint32_t node_id; + const uint32_t total_nodes; + + private: + std::vector cpus_; + std::vector gpus_; + std::vector omps_; + + private: + Memory system_memory_, zerocopy_memory_; + std::map frame_buffers_; + std::map socket_memories_; +}; + +} // namespace legate::mapping::detail diff --git a/src/core/mapping/detail/mapping.cc b/src/core/mapping/detail/mapping.cc new file mode 100644 index 0000000000..7942f2c080 --- /dev/null +++ b/src/core/mapping/detail/mapping.cc @@ -0,0 +1,194 @@ +/* Copyright 2021-2022 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/mapping/detail/mapping.h" + +namespace legate::mapping::detail { + +TaskTarget to_target(Processor::Kind kind) +{ + switch (kind) { + case Processor::Kind::TOC_PROC: return TaskTarget::GPU; + case Processor::Kind::OMP_PROC: return TaskTarget::OMP; + case Processor::Kind::LOC_PROC: return TaskTarget::CPU; + default: LEGATE_ABORT; + } + assert(false); + return TaskTarget::CPU; +} + +Processor::Kind to_kind(TaskTarget target) +{ + switch (target) { + case TaskTarget::GPU: return Processor::Kind::TOC_PROC; + case TaskTarget::OMP: return Processor::Kind::OMP_PROC; + case TaskTarget::CPU: return Processor::Kind::LOC_PROC; + default: LEGATE_ABORT; + } + assert(false); + return Processor::Kind::LOC_PROC; +} + +Memory::Kind to_kind(StoreTarget target) +{ + switch (target) { + case StoreTarget::SYSMEM: return Memory::Kind::SYSTEM_MEM; + case StoreTarget::FBMEM: return Memory::Kind::GPU_FB_MEM; + case StoreTarget::ZCMEM: return Memory::Kind::Z_COPY_MEM; + case StoreTarget::SOCKETMEM: return Memory::Kind::SOCKET_MEM; + default: LEGATE_ABORT; + } + assert(false); + return Memory::Kind::SYSTEM_MEM; +} + +LegateVariantCode to_variant_code(TaskTarget target) +{ + switch (target) { + case TaskTarget::GPU: return LEGATE_GPU_VARIANT; + case TaskTarget::OMP: return LEGATE_OMP_VARIANT; + case TaskTarget::CPU: return LEGATE_CPU_VARIANT; + default: LEGATE_ABORT; + } + assert(false); + return LEGATE_CPU_VARIANT; +} + +void DimOrdering::populate_dimension_ordering(const Store* store, + std::vector& ordering) const +{ + // TODO: We need to implement the relative dimension ordering + switch (kind) { + case Kind::C: { + auto dim = store->region_field().dim(); + for (int32_t idx = dim - 1; idx >= 0; --idx) + ordering.push_back(static_cast(DIM_X + idx)); + break; + } + case Kind::FORTRAN: { + auto dim = store->region_field().dim(); + for (int32_t idx = 0; idx < dim; ++idx) + ordering.push_back(static_cast(DIM_X + idx)); + break; + } + case Kind::CUSTOM: { + for (auto idx : dims) ordering.push_back(static_cast(DIM_X + idx)); + break; + } + } +} + +bool StoreMapping::for_future() const +{ + for (auto& store : stores) return store->is_future(); + return false; +} + +bool StoreMapping::for_unbound_store() const +{ + for (auto& store : stores) return store->unbound(); + return false; +} + +const Store* StoreMapping::store() const { return stores.front(); } + +uint32_t StoreMapping::requirement_index() const +{ +#ifdef DEBUG_LEGATE + assert(stores.size() > 0); + uint32_t result = -1U; + for (auto& store : stores) { + auto idx = store->requirement_index(); + assert(result == -1U || result == idx); + result = idx; + } + return result; +#else + static constexpr uint32_t invalid = -1U; + if (stores.empty()) return invalid; + return stores.front()->requirement_index(); +#endif +} + +std::set StoreMapping::requirement_indices() const +{ + std::set indices; + for (auto& store : stores) { + if (store->is_future()) continue; + indices.insert(store->region_field().index()); + } + return indices; +} + +std::set StoreMapping::requirements() const +{ + std::set reqs; + for (auto& store : stores) { + if (store->is_future()) continue; + auto* req = store->region_field().get_requirement(); + if (!req->region.exists()) continue; + reqs.insert(req); + } + return reqs; +} + +void StoreMapping::populate_layout_constraints( + Legion::LayoutConstraintSet& layout_constraints) const +{ + std::vector dimension_ordering{}; + + if (policy.layout == InstLayout::AOS) dimension_ordering.push_back(DIM_F); + policy.ordering.impl()->populate_dimension_ordering(stores.front(), dimension_ordering); + if (policy.layout == InstLayout::SOA) dimension_ordering.push_back(DIM_F); + + layout_constraints.add_constraint( + Legion::OrderingConstraint(dimension_ordering, false /*contiguous*/)); + + layout_constraints.add_constraint(Legion::MemoryConstraint(to_kind(policy.target))); + + std::vector fields{}; + if (stores.size() > 1) { + std::set field_set{}; + for (auto& store : stores) { + auto field_id = store->region_field().field_id(); + if (field_set.find(field_id) == field_set.end()) { + fields.push_back(field_id); + field_set.insert(field_id); + } + } + } else + fields.push_back(stores.front()->region_field().field_id()); + layout_constraints.add_constraint( + Legion::FieldConstraint(fields, false /*contiguous*/, false /*inorder*/)); +} + +/*static*/ std::unique_ptr StoreMapping::default_mapping(const Store* store, + StoreTarget target, + bool exact) +{ + return create(store, InstanceMappingPolicy{}.with_target(target).with_exact(exact)); +} + +/*static*/ std::unique_ptr StoreMapping::create(const Store* store, + InstanceMappingPolicy&& policy) +{ + auto mapping = std::make_unique(); + mapping->policy = std::move(policy); + mapping->stores.push_back(store); + return mapping; +} + +} // namespace legate::mapping::detail diff --git a/src/core/mapping/detail/mapping.h b/src/core/mapping/detail/mapping.h new file mode 100644 index 0000000000..e8f19fd221 --- /dev/null +++ b/src/core/mapping/detail/mapping.h @@ -0,0 +1,96 @@ +/* Copyright 2021-2022 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include + +#include "core/mapping/detail/store.h" +#include "core/mapping/mapping.h" + +namespace legate::mapping::detail { + +TaskTarget to_target(Processor::Kind kind); + +Processor::Kind to_kind(TaskTarget target); + +Memory::Kind to_kind(StoreTarget target); + +LegateVariantCode to_variant_code(TaskTarget target); + +struct DimOrdering { + public: + using Kind = mapping::DimOrdering::Kind; + + public: + DimOrdering(Kind _kind) : kind(_kind) {} + DimOrdering(const std::vector& _dims) : kind(Kind::CUSTOM), dims(_dims) {} + + public: + DimOrdering(const DimOrdering&) = default; + + public: + bool operator==(const DimOrdering& other) const + { + return kind == other.kind && dims == other.dims; + } + + public: + void populate_dimension_ordering(const Store* store, + std::vector& ordering) const; + + public: + Kind kind; + std::vector dims{}; +}; + +struct StoreMapping { + public: + std::vector stores{}; + InstanceMappingPolicy policy; + + public: + StoreMapping() {} + + public: + StoreMapping(const StoreMapping&) = default; + StoreMapping& operator=(const StoreMapping&) = default; + + public: + StoreMapping(StoreMapping&&) = default; + StoreMapping& operator=(StoreMapping&&) = default; + + public: + bool for_future() const; + bool for_unbound_store() const; + const Store* store() const; + + public: + uint32_t requirement_index() const; + std::set requirement_indices() const; + std::set requirements() const; + + public: + void populate_layout_constraints(Legion::LayoutConstraintSet& layout_constraints) const; + + public: + static std::unique_ptr default_mapping(const Store* store, + StoreTarget target, + bool exact = false); + static std::unique_ptr create(const Store* store, InstanceMappingPolicy&& policy); +}; + +} // namespace legate::mapping::detail diff --git a/src/core/mapping/detail/operation.cc b/src/core/mapping/detail/operation.cc new file mode 100644 index 0000000000..7a06149a4a --- /dev/null +++ b/src/core/mapping/detail/operation.cc @@ -0,0 +1,90 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/mapping/detail/operation.h" +#include "core/runtime/detail/library.h" +#include "core/utilities/deserializer.h" + +namespace legate::mapping::detail { + +Mappable::Mappable() {} + +Mappable::Mappable(const Legion::Mappable* mappable) +{ + MapperDataDeserializer dez(mappable); + machine_ = dez.unpack(); + sharding_id_ = dez.unpack(); +} + +Task::Task(const Legion::Task* task, + const legate::detail::Library* library, + Legion::Mapping::MapperRuntime* runtime, + const Legion::Mapping::MapperContext context) + : Mappable(task), task_(task), library_(library) +{ + TaskDeserializer dez(task, runtime, context); + inputs_ = dez.unpack(); + outputs_ = dez.unpack(); + reductions_ = dez.unpack(); + scalars_ = dez.unpack>(); +} + +int64_t Task::task_id() const { return library_->get_local_task_id(task_->task_id); } + +TaskTarget Task::target() const +{ + switch (task_->target_proc.kind()) { + case Processor::LOC_PROC: return TaskTarget::CPU; + case Processor::TOC_PROC: return TaskTarget::GPU; + case Processor::OMP_PROC: return TaskTarget::OMP; + default: { + assert(false); + } + } + assert(false); + return TaskTarget::CPU; +} + +Copy::Copy(const Legion::Copy* copy, + Legion::Mapping::MapperRuntime* runtime, + const Legion::Mapping::MapperContext context) + : Mappable(), copy_(copy) +{ + CopyDeserializer dez(copy, + {copy->src_requirements, + copy->dst_requirements, + copy->src_indirect_requirements, + copy->dst_indirect_requirements}, + runtime, + context); + machine_ = dez.unpack(); + sharding_id_ = dez.unpack(); + inputs_ = dez.unpack(); + dez.next_requirement_list(); + outputs_ = dez.unpack(); + dez.next_requirement_list(); + input_indirections_ = dez.unpack(); + dez.next_requirement_list(); + output_indirections_ = dez.unpack(); +#ifdef DEBUG_LEGATE + for (auto& input : inputs_) assert(!input.is_future()); + for (auto& output : outputs_) assert(!output.is_future()); + for (auto& input_indirection : input_indirections_) assert(!input_indirection.is_future()); + for (auto& output_indirection : output_indirections_) assert(!output_indirection.is_future()); +#endif +} + +} // namespace legate::mapping::detail diff --git a/src/core/mapping/detail/operation.h b/src/core/mapping/detail/operation.h new file mode 100644 index 0000000000..b185fed29c --- /dev/null +++ b/src/core/mapping/detail/operation.h @@ -0,0 +1,146 @@ +/* Copyright 2021-2022 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include "core/data/scalar.h" +#include "core/mapping/detail/machine.h" +#include "core/mapping/detail/store.h" + +/** + * @file + * @brief Class definitions for operations and stores used in mapping + */ + +namespace legate::detail { +class Library; +} // namespace legate::detail + +namespace legate::mapping::detail { + +namespace { +using Stores = std::vector; +} // namespace + +class Mappable { + protected: + Mappable(); + + public: + Mappable(const Legion::Mappable* mappable); + + public: + const mapping::detail::Machine& machine() const { return machine_; } + uint32_t sharding_id() const { return sharding_id_; } + + protected: + mapping::detail::Machine machine_; + uint32_t sharding_id_; +}; + +/** + * @ingroup mapping + * @brief A metadata class for tasks + */ +class Task : public Mappable { + public: + Task(const Legion::Task* task, + const legate::detail::Library* library, + Legion::Mapping::MapperRuntime* runtime, + const Legion::Mapping::MapperContext context); + + public: + /** + * @brief Returns the task id + * + * @return Task id + */ + int64_t task_id() const; + + public: + /** + * @brief Returns metadata for the task's input stores + * + * @return Vector of store metadata objects + */ + const Stores& inputs() const { return inputs_; } + /** + * @brief Returns metadata for the task's output stores + * + * @return Vector of store metadata objects + */ + const Stores& outputs() const { return outputs_; } + /** + * @brief Returns metadata for the task's reduction stores + * + * @return Vector of store metadata objects + */ + const Stores& reductions() const { return reductions_; } + /** + * @brief Returns the vector of the task's by-value arguments. Unlike `mapping::Store` + * objects that have no access to data in the stores, the returned `Scalar` objects + * contain valid arguments to the task + * + * @return Vector of `Scalar` objects + */ + const std::vector& scalars() const { return scalars_; } + + public: + /** + * @brief Returns the point of the task + * + * @return The point of the task + */ + DomainPoint point() const { return task_->index_point; } + + public: + TaskTarget target() const; + + private: + const legate::detail::Library* library_; + const Legion::Task* task_; + + private: + Stores inputs_, outputs_, reductions_; + std::vector scalars_; +}; + +class Copy : public Mappable { + public: + Copy(const Legion::Copy* copy, + Legion::Mapping::MapperRuntime* runtime, + const Legion::Mapping::MapperContext context); + + public: + const Stores& inputs() const { return inputs_; } + const Stores& outputs() const { return outputs_; } + const Stores& input_indirections() const { return input_indirections_; } + const Stores& output_indirections() const { return output_indirections_; } + + public: + DomainPoint point() const { return copy_->index_point; } + + private: + const Legion::Copy* copy_; + + private: + Stores inputs_; + Stores outputs_; + Stores input_indirections_; + Stores output_indirections_; +}; + +} // namespace legate::mapping::detail diff --git a/src/core/mapping/operation.inl b/src/core/mapping/detail/operation.inl similarity index 100% rename from src/core/mapping/operation.inl rename to src/core/mapping/detail/operation.inl diff --git a/src/core/mapping/detail/store.cc b/src/core/mapping/detail/store.cc new file mode 100644 index 0000000000..81592703bf --- /dev/null +++ b/src/core/mapping/detail/store.cc @@ -0,0 +1,146 @@ +/* Copyright 2022 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/mapping/detail/store.h" + +#include "core/data/detail/transform.h" + +namespace legate::mapping::detail { + +RegionField::RegionField(const Legion::RegionRequirement* req, + int32_t dim, + uint32_t idx, + Legion::FieldID fid) + : req_(req), dim_(dim), idx_(idx), fid_(fid) +{ +} + +bool RegionField::can_colocate_with(const RegionField& other) const +{ + auto* my_req = get_requirement(); + auto* other_req = other.get_requirement(); + return my_req->region.get_tree_id() == other_req->region.get_tree_id() && fid_ == other.fid_ && + dim_ == other.dim_; +} + +Domain RegionField::domain(Legion::Mapping::MapperRuntime* runtime, + const Legion::Mapping::MapperContext context) const +{ + return runtime->get_index_space_domain(context, get_index_space()); +} + +Legion::IndexSpace RegionField::get_index_space() const { return req_->region.get_index_space(); } + +FutureWrapper::FutureWrapper(uint32_t idx, const Domain& domain) : idx_(idx), domain_(domain) {} + +Domain FutureWrapper::domain() const { return domain_; } + +Store::Store(int32_t dim, + std::shared_ptr type, + FutureWrapper future, + std::shared_ptr&& transform) + : is_future_(true), + is_unbound_store_(false), + dim_(dim), + type_(std::move(type)), + redop_id_(-1), + future_(future), + transform_(std::move(transform)) +{ +} + +Store::Store(Legion::Mapping::MapperRuntime* runtime, + const Legion::Mapping::MapperContext context, + int32_t dim, + std::shared_ptr type, + int32_t redop_id, + const RegionField& region_field, + bool is_unbound_store, + std::shared_ptr&& transform) + : is_future_(false), + is_unbound_store_(is_unbound_store), + dim_(dim), + type_(std::move(type)), + redop_id_(redop_id), + region_field_(region_field), + transform_(std::move(transform)), + runtime_(runtime), + context_(context) +{ +} + +Store::Store(Legion::Mapping::MapperRuntime* runtime, + const Legion::Mapping::MapperContext context, + const Legion::RegionRequirement* requirement) + : is_future_(false), + is_unbound_store_(false), + dim_(requirement->region.get_dim()), + redop_id_(-1), + runtime_(runtime), + context_(context) +{ + region_field_ = RegionField(requirement, dim_, 0, requirement->instance_fields.front()); +} + +bool Store::can_colocate_with(const Store& other) const +{ + if (is_future() || other.is_future()) + return false; + else if (unbound() || other.unbound()) + return false; + else if (is_reduction() || other.is_reduction()) + return redop() == other.redop() && region_field_.can_colocate_with(other.region_field_); + return region_field_.can_colocate_with(other.region_field_); +} + +const RegionField& Store::region_field() const +{ +#ifdef DEBUG_LEGATE + assert(!is_future()); +#endif + return region_field_; +} + +const FutureWrapper& Store::future() const +{ +#ifdef DEBUG_LEGATE + assert(is_future()); +#endif + return future_; +} + +RegionField::Id Store::unique_region_field_id() const { return region_field().unique_id(); } + +uint32_t Store::requirement_index() const { return region_field().index(); } + +uint32_t Store::future_index() const { return future().index(); } + +Domain Store::domain() const +{ + assert(!unbound()); + auto result = is_future_ ? future_.domain() : region_field_.domain(runtime_, context_); + if (nullptr != transform_) result = transform_->transform(result); + assert(result.dim == dim_); + return result; +} + +std::vector Store::find_imaginary_dims() const +{ + if (nullptr != transform_) return transform_->find_imaginary_dims(); + return std::vector(); +} + +} // namespace legate::mapping::detail diff --git a/src/core/mapping/detail/store.h b/src/core/mapping/detail/store.h new file mode 100644 index 0000000000..e6725fe368 --- /dev/null +++ b/src/core/mapping/detail/store.h @@ -0,0 +1,163 @@ +/* Copyright 2022 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include "core/type/detail/type_info.h" +#include "core/utilities/typedefs.h" + +namespace legate::detail { +class TransformStack; +} // namespace legate::detail + +namespace legate::mapping::detail { + +class RegionField { + public: + using Id = std::tuple; + + public: + RegionField() {} + RegionField(const Legion::RegionRequirement* req, int32_t dim, uint32_t idx, Legion::FieldID fid); + + public: + RegionField(const RegionField& other) = default; + RegionField& operator=(const RegionField& other) = default; + + public: + bool can_colocate_with(const RegionField& other) const; + + public: + Legion::Domain domain(Legion::Mapping::MapperRuntime* runtime, + const Legion::Mapping::MapperContext context) const; + + public: + bool operator==(const RegionField& other) const; + + public: + Id unique_id() const { return std::make_tuple(unbound(), idx_, fid_); } + + public: + int32_t dim() const { return dim_; } + uint32_t index() const { return idx_; } + Legion::FieldID field_id() const { return fid_; } + bool unbound() const { return dim_ < 0; } + + public: + const Legion::RegionRequirement* get_requirement() const { return req_; } + Legion::IndexSpace get_index_space() const; + + private: + const Legion::RegionRequirement* req_{nullptr}; + int32_t dim_{-1}; + uint32_t idx_{-1U}; + Legion::FieldID fid_{-1U}; +}; + +class FutureWrapper { + public: + FutureWrapper() {} + FutureWrapper(uint32_t idx, const Legion::Domain& domain); + + public: + FutureWrapper(const FutureWrapper& other) = default; + FutureWrapper& operator=(const FutureWrapper& other) = default; + + public: + int32_t dim() const { return domain_.dim; } + uint32_t index() const { return idx_; } + + public: + Legion::Domain domain() const; + + private: + uint32_t idx_{-1U}; + Legion::Domain domain_{}; +}; + +class Store { + public: + Store() {} + Store(int32_t dim, + std::shared_ptr type, + FutureWrapper future, + std::shared_ptr&& transform = nullptr); + Store(Legion::Mapping::MapperRuntime* runtime, + const Legion::Mapping::MapperContext context, + int32_t dim, + std::shared_ptr type, + int32_t redop_id, + const RegionField& region_field, + bool is_unbound_store = false, + std::shared_ptr&& transform = nullptr); + // A special constructor to create a mapper view of a store from a region requirement + Store(Legion::Mapping::MapperRuntime* runtime, + const Legion::Mapping::MapperContext context, + const Legion::RegionRequirement* requirement); + + public: + Store(const Store& other) = default; + Store& operator=(const Store& other) = default; + + public: + Store(Store&& other) = default; + Store& operator=(Store&& other) = default; + + public: + bool is_future() const { return is_future_; } + bool unbound() const { return is_unbound_store_; } + int32_t dim() const { return dim_; } + + public: + bool is_reduction() const { return redop_id_ > 0; } + int32_t redop() const { return redop_id_; } + + public: + bool can_colocate_with(const Store& other) const; + const RegionField& region_field() const; + const FutureWrapper& future() const; + + public: + RegionField::Id unique_region_field_id() const; + uint32_t requirement_index() const; + uint32_t future_index() const; + + public: + Domain domain() const; + + public: + std::vector find_imaginary_dims() const; + + private: + bool is_future_{false}; + bool is_unbound_store_{false}; + int32_t dim_{-1}; + std::shared_ptr type_{}; + int32_t redop_id_{-1}; + + private: + FutureWrapper future_; + RegionField region_field_; + + private: + std::shared_ptr transform_{nullptr}; + + private: + Legion::Mapping::MapperRuntime* runtime_{nullptr}; + Legion::Mapping::MapperContext context_{nullptr}; +}; + +} // namespace legate::mapping::detail diff --git a/src/core/mapping/machine.cc b/src/core/mapping/machine.cc index 0bfdec6051..83fc6ebe33 100644 --- a/src/core/mapping/machine.cc +++ b/src/core/mapping/machine.cc @@ -15,85 +15,72 @@ */ #include "core/mapping/machine.h" -#include "core/utilities/buffer_builder.h" - -#include "realm/network.h" +#include "core/mapping/detail/machine.h" +#include "core/utilities/detail/buffer_builder.h" namespace legate::mapping { -TaskTarget to_target(Processor::Kind kind) +///////////////////////////////////// +// legate::mapping::NodeRange +///////////////////////////////////// + +bool NodeRange::operator<(const NodeRange& other) const { - switch (kind) { - case Processor::Kind::TOC_PROC: return TaskTarget::GPU; - case Processor::Kind::OMP_PROC: return TaskTarget::OMP; - case Processor::Kind::LOC_PROC: return TaskTarget::CPU; - default: LEGATE_ABORT; - } - assert(false); - return TaskTarget::CPU; + return low < other.low || (low == other.low && high < other.high); } -Processor::Kind to_kind(TaskTarget target) +bool NodeRange::operator==(const NodeRange& other) const { - switch (target) { - case TaskTarget::GPU: return Processor::Kind::TOC_PROC; - case TaskTarget::OMP: return Processor::Kind::OMP_PROC; - case TaskTarget::CPU: return Processor::Kind::LOC_PROC; - default: LEGATE_ABORT; - } - assert(false); - return Processor::Kind::LOC_PROC; + return low == other.low && high == other.high; } -LegateVariantCode to_variant_code(TaskTarget target) +bool NodeRange::operator!=(const NodeRange& other) const { return !operator==(other); } + +///////////////////////////////////// +// legate::mapping::ProcessorRange +///////////////////////////////////// + +uint32_t ProcessorRange::count() const { return high - low; } + +bool ProcessorRange::empty() const { return high <= low; } + +ProcessorRange ProcessorRange::slice(uint32_t from, uint32_t to) const { - switch (target) { - case TaskTarget::GPU: return LEGATE_GPU_VARIANT; - case TaskTarget::OMP: return LEGATE_OMP_VARIANT; - case TaskTarget::CPU: return LEGATE_CPU_VARIANT; - default: LEGATE_ABORT; - } - assert(false); - return LEGATE_CPU_VARIANT; + uint32_t new_low = std::min(low + from, high); + uint32_t new_high = std::min(low + to, high); + return ProcessorRange(new_low, new_high, per_node_count); } -std::ostream& operator<<(std::ostream& stream, const TaskTarget& target) +NodeRange ProcessorRange::get_node_range() const { - switch (target) { - case TaskTarget::GPU: { - stream << "GPU"; - break; - } - case TaskTarget::OMP: { - stream << "OMP"; - break; - } - case TaskTarget::CPU: { - stream << "CPU"; - break; - } + if (empty()) { + throw std::invalid_argument("Illegal to get a node range of an empty processor range"); } - return stream; + return NodeRange{low / per_node_count, (high + per_node_count - 1) / per_node_count}; } -///////////////////////////////////// -// legate::mapping::ProcessorRange -///////////////////////////////////// +std::string ProcessorRange::to_string() const +{ + std::stringstream ss; + ss << "Proc([" << low << "," << high << "], " << per_node_count << " per node)"; + return ss.str(); +} + +ProcessorRange::ProcessorRange() {} ProcessorRange::ProcessorRange(uint32_t _low, uint32_t _high, uint32_t _per_node_count) - : low(_low), high(_high), per_node_count(_per_node_count) + : low(_low < _high ? _low : 0), + high(_low < _high ? _high : 0), + per_node_count(std::max(1, _per_node_count)) { - if (high < low) { - low = 0; - high = 0; - } } ProcessorRange ProcessorRange::operator&(const ProcessorRange& other) const { -#ifdef DEBUG_LEGATE - assert(other.per_node_count == per_node_count); -#endif + if (other.per_node_count != per_node_count) { + throw std::invalid_argument( + "Invalid to compute an intersection between processor ranges with different per-node counts"); + } return ProcessorRange(std::max(low, other.low), std::min(high, other.high), per_node_count); } @@ -117,37 +104,6 @@ bool ProcessorRange::operator<(const ProcessorRange& other) const return per_node_count < other.per_node_count; } -uint32_t ProcessorRange::count() const { return high - low; } - -bool ProcessorRange::empty() const { return high <= low; } - -std::string ProcessorRange::to_string() const -{ - std::stringstream ss; - ss << "Proc([" << low << "," << high << "], " << per_node_count << " per node)"; - return ss.str(); -} - -std::pair ProcessorRange::get_node_range() const -{ - if (empty()) throw std::runtime_error("Illegal to get a node range of an empty processor range"); - return std::make_pair(low / per_node_count, (high + per_node_count - 1) / per_node_count); -} - -ProcessorRange ProcessorRange::slice(uint32_t from, uint32_t to) const -{ - uint32_t new_low = std::min(low + from, high); - uint32_t new_high = std::min(low + to, high); - return ProcessorRange(new_low, new_high, per_node_count); -} - -void ProcessorRange::pack(BufferBuilder& buffer) const -{ - buffer.pack(low); - buffer.pack(high); - buffer.pack(per_node_count); -} - std::ostream& operator<<(std::ostream& stream, const ProcessorRange& range) { stream << range.to_string(); @@ -155,334 +111,89 @@ std::ostream& operator<<(std::ostream& stream, const ProcessorRange& range) } /////////////////////////////////////////// -// legate::mapping::MachineDesc +// legate::mapping::Machine ////////////////////////////////////////// -MachineDesc::MachineDesc(const std::map& ranges) - : processor_ranges(ranges) -{ - for (auto& [target, processor_range] : processor_ranges) - if (!processor_range.empty()) { - preferred_target = target; - return; - } -} +TaskTarget Machine::preferred_target() const { return impl_->preferred_target; } -MachineDesc::MachineDesc(std::map&& ranges) - : processor_ranges(std::move(ranges)) +ProcessorRange Machine::processor_range() const { return processor_range(impl_->preferred_target); } +ProcessorRange Machine::processor_range(TaskTarget target) const { - for (auto& [target, processor_range] : processor_ranges) - if (!processor_range.empty()) { - preferred_target = target; - return; - } + return impl_->processor_range(target); } -const ProcessorRange& MachineDesc::processor_range() const -{ - return processor_range(preferred_target); -} -static const ProcessorRange EMPTY_RANGE{}; +std::vector Machine::valid_targets() const { return impl_->valid_targets(); } -const ProcessorRange& MachineDesc::processor_range(TaskTarget target) const +std::vector Machine::valid_targets_except(const std::set& to_exclude) const { - auto finder = processor_ranges.find(target); - if (finder == processor_ranges.end()) { return EMPTY_RANGE; } - return finder->second; + return impl_->valid_targets_except(to_exclude); } -std::vector MachineDesc::valid_targets() const -{ - std::vector result; - for (auto& [target, _] : processor_ranges) result.push_back(target); - return result; -} +uint32_t Machine::count() const { return count(impl_->preferred_target); } -std::vector MachineDesc::valid_targets_except( - const std::set& to_exclude) const -{ - std::vector result; - for (auto& [target, _] : processor_ranges) - if (to_exclude.find(target) == to_exclude.end()) result.push_back(target); - return result; -} - -uint32_t MachineDesc::count() const { return count(preferred_target); } - -uint32_t MachineDesc::count(TaskTarget target) const { return processor_range(target).count(); } - -std::string MachineDesc::to_string() const -{ - std::stringstream ss; - ss << "Machine(preferred_target: " << preferred_target; - for (auto& [kind, range] : processor_ranges) ss << ", " << kind << ": " << range.to_string(); - ss << ")"; - return ss.str(); -} - -void MachineDesc::pack(BufferBuilder& buffer) const -{ - buffer.pack(static_cast(preferred_target)); - buffer.pack(processor_ranges.size()); - for (auto& [target, processor_range] : processor_ranges) { - buffer.pack(static_cast(target)); - processor_range.pack(buffer); - } -} - -MachineDesc MachineDesc::only(TaskTarget target) const { return only(std::vector({target})); } - -MachineDesc MachineDesc::only(const std::vector& targets) const -{ - std::map new_processor_ranges; - for (auto t : targets) new_processor_ranges.insert({t, processor_range(t)}); - - return MachineDesc(new_processor_ranges); -} - -MachineDesc MachineDesc::slice(uint32_t from, - uint32_t to, - TaskTarget target, - bool keep_others) const -{ - if (keep_others) { - std::map new_ranges(processor_ranges); - new_ranges[target] = processor_range(target).slice(from, to); - return MachineDesc(std::move(new_ranges)); - } else - return MachineDesc({{target, processor_range(target).slice(from, to)}}); -} +uint32_t Machine::count(TaskTarget target) const { return impl_->count(target); } -MachineDesc MachineDesc::slice(uint32_t from, uint32_t to, bool keep_others) const -{ - return slice(from, to, preferred_target, keep_others); -} +std::string Machine::to_string() const { return impl_->to_string(); } -MachineDesc MachineDesc::operator[](TaskTarget target) const { return only(target); } - -MachineDesc MachineDesc::operator[](const std::vector& targets) const -{ - return only(targets); -} +Machine Machine::only(TaskTarget target) const { return only(std::vector({target})); } -bool MachineDesc::operator==(const MachineDesc& other) const +Machine Machine::only(const std::vector& targets) const { - if (preferred_target != other.preferred_target) return false; - if (processor_ranges.size() != other.processor_ranges.size()) return false; - for (auto const& r : processor_ranges) { - auto finder = other.processor_ranges.find(r.first); - if (finder == other.processor_ranges.end() || r.second != finder->second) return false; - } - return true; + return Machine(new detail::Machine(impl_->only(targets))); } -bool MachineDesc::operator!=(const MachineDesc& other) const { return !(*this == other); } - -MachineDesc MachineDesc::operator&(const MachineDesc& other) const +Machine Machine::slice(uint32_t from, uint32_t to, TaskTarget target, bool keep_others) const { - std::map new_processor_ranges; - for (const auto& [target, range] : processor_ranges) { - auto finder = other.processor_ranges.find(target); - if (finder != other.processor_ranges.end()) { - new_processor_ranges[target] = finder->second & range; - } - } - return MachineDesc(new_processor_ranges); + return Machine(new detail::Machine(impl_->slice(from, to, target, keep_others))); } -bool MachineDesc::empty() const +Machine Machine::slice(uint32_t from, uint32_t to, bool keep_others) const { - for (const auto& r : processor_ranges) - if (!r.second.empty()) return false; - return true; + return slice(from, to, impl_->preferred_target, keep_others); } -//////////////////////////////////////// -// legate::mapping::LocalProcessorRange -///////////////////////////////////////// - -LocalProcessorRange::LocalProcessorRange() : offset_(0), total_proc_count_(0), procs_() {} - -LocalProcessorRange::LocalProcessorRange(const std::vector& procs) - : offset_(0), total_proc_count_(procs.size()), procs_(procs.data(), procs.size()) -{ -} +Machine Machine::operator[](TaskTarget target) const { return only(target); } -LocalProcessorRange::LocalProcessorRange(uint32_t offset, - uint32_t total_proc_count, - const Processor* local_procs, - size_t num_local_procs) - : offset_(offset), total_proc_count_(total_proc_count), procs_(local_procs, num_local_procs) -{ -} +Machine Machine::operator[](const std::vector& targets) const { return only(targets); } -const Processor& LocalProcessorRange::operator[](uint32_t idx) const -{ - auto local_idx = (idx % total_proc_count_) - offset_; -#ifdef DEBUG_LEGATE - assert(local_idx < procs_.size()); -#endif - return procs_[local_idx]; -} +bool Machine::operator==(const Machine& other) const { return *impl_ == *other.impl_; } -std::string LocalProcessorRange::to_string() const -{ - std::stringstream ss; - ss << "{offset: " << offset_ << ", total processor count: " << total_proc_count_ - << ", processors: "; - for (uint32_t idx = 0; idx < procs_.size(); ++idx) ss << procs_[idx] << ","; - ss << "}"; - return std::move(ss).str(); -} +bool Machine::operator!=(const Machine& other) const { return !(*impl_ == *other.impl_); } -std::ostream& operator<<(std::ostream& stream, const LocalProcessorRange& range) +Machine Machine::operator&(const Machine& other) const { - stream << range.to_string(); - return stream; + auto result = *impl_ & *other.impl_; + return Machine(new detail::Machine(std::move(result))); } -////////////////////////////// -// legate::mapping::Machine -////////////////////////////// -Machine::Machine() - : local_node(Realm::Network::my_node_id), - total_nodes(Legion::Machine::get_machine().get_address_space_count()) -{ - auto legion_machine = Legion::Machine::get_machine(); - Legion::Machine::ProcessorQuery procs(legion_machine); - // Query to find all our local processors - procs.local_address_space(); - for (auto proc : procs) { - switch (proc.kind()) { - case Processor::LOC_PROC: { - cpus_.push_back(proc); - continue; - } - case Processor::TOC_PROC: { - gpus_.push_back(proc); - continue; - } - case Processor::OMP_PROC: { - omps_.push_back(proc); - continue; - } - default: { - continue; - } - } - } +bool Machine::empty() const { return impl_->empty(); } - // Now do queries to find all our local memories - Legion::Machine::MemoryQuery sysmem(legion_machine); - sysmem.local_address_space().only_kind(Legion::Memory::SYSTEM_MEM); - assert(sysmem.count() > 0); - system_memory_ = sysmem.first(); - - if (!gpus_.empty()) { - Legion::Machine::MemoryQuery zcmem(legion_machine); - zcmem.local_address_space().only_kind(Legion::Memory::Z_COPY_MEM); - assert(zcmem.count() > 0); - zerocopy_memory_ = zcmem.first(); - } - for (auto& gpu : gpus_) { - Legion::Machine::MemoryQuery framebuffer(legion_machine); - framebuffer.local_address_space().only_kind(Legion::Memory::GPU_FB_MEM).best_affinity_to(gpu); - assert(framebuffer.count() > 0); - frame_buffers_[gpu] = framebuffer.first(); - } - for (auto& omp : omps_) { - Legion::Machine::MemoryQuery sockmem(legion_machine); - sockmem.local_address_space().only_kind(Legion::Memory::SOCKET_MEM).best_affinity_to(omp); - // If we have socket memories then use them - if (sockmem.count() > 0) socket_memories_[omp] = sockmem.first(); - // Otherwise we just use the local system memory - else - socket_memories_[omp] = system_memory_; - } -} +Machine::Machine(detail::Machine* impl) : impl_(impl) {} -const std::vector& Machine::procs(TaskTarget target) const -{ - switch (target) { - case TaskTarget::GPU: return gpus_; - case TaskTarget::OMP: return omps_; - case TaskTarget::CPU: return cpus_; - default: LEGATE_ABORT; - } - assert(false); - return cpus_; -} +Machine::Machine(const detail::Machine& impl) : impl_(new detail::Machine(impl)) {} -size_t Machine::total_frame_buffer_size() const -{ - // We assume that all memories of the same kind are symmetric in size - size_t per_node_size = frame_buffers_.size() * frame_buffers_.begin()->second.capacity(); - return per_node_size * total_nodes; -} +Machine::Machine(const Machine& other) : impl_(new detail::Machine(*other.impl_)) {} -size_t Machine::total_socket_memory_size() const +Machine& Machine::operator=(const Machine& other) { - // We assume that all memories of the same kind are symmetric in size - size_t per_node_size = socket_memories_.size() * socket_memories_.begin()->second.capacity(); - return per_node_size * total_nodes; + impl_ = new detail::Machine(*other.impl_); + return *this; } -bool Machine::has_socket_memory() const -{ - return !socket_memories_.empty() && - socket_memories_.begin()->second.kind() == Legion::Memory::SOCKET_MEM; -} +Machine::Machine(Machine&& other) : impl_(other.impl_) { other.impl_ = nullptr; } -LocalProcessorRange Machine::slice(TaskTarget target, - const MachineDesc& machine_desc, - bool fallback_to_global /*=false*/) const +Machine& Machine::operator=(Machine&& other) { - const auto& local_procs = procs(target); - - auto finder = machine_desc.processor_ranges.find(target); - if (machine_desc.processor_ranges.end() == finder) { - if (fallback_to_global) - return LocalProcessorRange(local_procs); - else - return LocalProcessorRange(); - } - - auto& global_range = finder->second; - - uint32_t num_local_procs = local_procs.size(); - uint32_t my_low = num_local_procs * local_node; - ProcessorRange my_range(my_low, my_low + num_local_procs, global_range.per_node_count); - - auto slice = global_range & my_range; - if (slice.empty()) { - if (fallback_to_global) - return LocalProcessorRange(local_procs); - else - return LocalProcessorRange(); - } - - return LocalProcessorRange(slice.low - global_range.low, - global_range.count(), - local_procs.data() + (slice.low - my_low), - slice.count()); + impl_ = other.impl_; + other.impl_ = nullptr; + return *this; } -Legion::Memory Machine::get_memory(Processor proc, StoreTarget target) const -{ - switch (target) { - case StoreTarget::SYSMEM: return system_memory_; - case StoreTarget::FBMEM: return frame_buffers_.at(proc); - case StoreTarget::ZCMEM: return zerocopy_memory_; - case StoreTarget::SOCKETMEM: return socket_memories_.at(proc); - default: LEGATE_ABORT; - } - assert(false); - return Legion::Memory::NO_MEMORY; -} +Machine::~Machine() { delete impl_; } -std::ostream& operator<<(std::ostream& stream, const MachineDesc& info) +std::ostream& operator<<(std::ostream& stream, const Machine& machine) { - stream << info.to_string(); + stream << machine.impl()->to_string(); return stream; } diff --git a/src/core/mapping/machine.h b/src/core/mapping/machine.h index 572d4214d1..9a85518513 100644 --- a/src/core/mapping/machine.h +++ b/src/core/mapping/machine.h @@ -1,4 +1,4 @@ -/* Copyright 2022 NVIDIA Corporation +/* Copyright 2023 NVIDIA Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,8 +20,6 @@ #include "core/mapping/mapping.h" #include "core/utilities/span.h" -#include "legate_defines.h" -#include "legion.h" /** * @file @@ -30,13 +28,20 @@ namespace legate::mapping { -TaskTarget to_target(Processor::Kind kind); - -Processor::Kind to_kind(TaskTarget target); - -LegateVariantCode to_variant_code(TaskTarget target); +/** + * @ingroup mapping + * @brief A class to represent a range of nodes. + * + * `NodeRange`s are half-open intervals of logical node IDs. + */ +struct NodeRange { + const uint32_t low; + const uint32_t high; -std::ostream& operator<<(std::ostream& stream, const TaskTarget& target); + bool operator<(const NodeRange&) const; + bool operator==(const NodeRange&) const; + bool operator!=(const NodeRange&) const; +}; /** * @ingroup mapping @@ -46,21 +51,17 @@ std::ostream& operator<<(std::ostream& stream, const TaskTarget& target); */ struct ProcessorRange { /** - * @brief Creates an empty processor range + * @brief Starting processor ID */ - ProcessorRange() {} + uint32_t low{0}; /** - * @brief Creates a processor range - * - * @param low Starting processor ID - * @param high End processor ID - * @param per_node_count Number of per-node processors + * @brief End processor ID */ - ProcessorRange(uint32_t low, uint32_t high, uint32_t per_node_count); - ProcessorRange operator&(const ProcessorRange&) const; - bool operator==(const ProcessorRange&) const; - bool operator!=(const ProcessorRange&) const; - bool operator<(const ProcessorRange&) const; + uint32_t high{0}; + /** + * @brief Number of per-node processors + */ + uint32_t per_node_count{1}; /** * @brief Returns the number of processors in the range * @@ -74,18 +75,6 @@ struct ProcessorRange { * @return false The range is not empty */ bool empty() const; - /** - * @brief Converts the range to a human-readable string - * - * @return Processor range in a string - */ - std::string to_string() const; - /** - * @brief Computes a range of node IDs for this processor range - * - * @return Node range in a pair - */ - std::pair get_node_range() const; /** * @brief Slices the processor range for a given sub-range * @@ -96,66 +85,61 @@ struct ProcessorRange { */ ProcessorRange slice(uint32_t from, uint32_t to) const; /** - * @brief Serializes the processor range to a buffer + * @brief Computes a range of node IDs for this processor range * - * @param buffer Buffer to which the processor range should be serialized + * @return Node range in a pair */ - void pack(BufferBuilder& buffer) const; - + NodeRange get_node_range() const; /** - * @brief Starting processor ID + * @brief Converts the range to a human-readable string + * + * @return Processor range in a string */ - uint32_t low{0}; + std::string to_string() const; /** - * @brief End processor ID + * @brief Creates an empty processor range */ - uint32_t high{0}; + ProcessorRange(); /** - * @brief Number of per-node processors + * @brief Creates a processor range + * + * @param low Starting processor ID + * @param high End processor ID + * @param per_node_count Number of per-node processors */ - uint32_t per_node_count{1}; + ProcessorRange(uint32_t low, uint32_t high, uint32_t per_node_count); + ProcessorRange operator&(const ProcessorRange&) const; + bool operator==(const ProcessorRange&) const; + bool operator!=(const ProcessorRange&) const; + bool operator<(const ProcessorRange&) const; }; std::ostream& operator<<(std::ostream& stream, const ProcessorRange& range); +namespace detail { +class Machine; +} // namespace detail + /** * @ingroup mapping * @brief Machine descriptor class * - * A machine descriptor describes the machine resource that should be used for a given scope of + * A `Machine` object describes the machine resource that should be used for a given scope of * execution. By default, the scope is given the entire machine resource configured for this * process. Then, the client can limit the resource by extracting a portion of the machine - * descriptor and setting it for the scope using `MachineTracker`. Configuring the scope with an - * empty machine descriptor raises a `std::runtime_error` exception. + * and setting it for the scope using `MachineTracker`. Configuring the scope with an + * empty machine raises a `std::runtime_error` exception. */ -struct MachineDesc { - /** - * @brief Creates an empty machine descriptor - */ - MachineDesc() {} - /** - * @brief Creates a machine descriptor with a given set of processor ranges - * - * @param processor_ranges Processor ranges - */ - MachineDesc(const std::map& processor_ranges); +class Machine { + public: /** - * @brief Creates a machine descriptor with a given set of processor ranges - * - * @param processor_ranges Processor ranges + * @brief Preferred processor type of this machine descriptor */ - MachineDesc(std::map&& processor_ranges); - - MachineDesc(const MachineDesc&) = default; - MachineDesc& operator=(const MachineDesc&) = default; - - MachineDesc(MachineDesc&&) = default; - MachineDesc& operator=(MachineDesc&&) = default; - + TaskTarget preferred_target() const; /** * @brief Returns the processor range for the preferred processor type in this descriptor */ - const ProcessorRange& processor_range() const; + ProcessorRange processor_range() const; /** * @brief Returns the processor range for a given processor type * @@ -163,8 +147,7 @@ struct MachineDesc { * * @target target Processor type to query */ - const ProcessorRange& processor_range(TaskTarget target) const; - + ProcessorRange processor_range(TaskTarget target) const; /** * @brief Returns the valid task targets within this machine descriptor * @@ -179,7 +162,6 @@ struct MachineDesc { * @return Task targets */ std::vector valid_targets_except(const std::set& to_exclude) const; - /** * @brief Returns the number of preferred processors * @@ -201,14 +183,6 @@ struct MachineDesc { * @return Machine descriptor in a string */ std::string to_string() const; - - /** - * @brief Serializes the machine descriptor to a buffer - * - * @param buffer Buffer to which the machine descriptor should be serialized - */ - void pack(BufferBuilder& buffer) const; - /** * @brief Extracts the processor range for a given processor type and creates a fresh machine * descriptor with it @@ -219,7 +193,7 @@ struct MachineDesc { * * @return Machine descriptor with the chosen processor range */ - MachineDesc only(TaskTarget target) const; + Machine only(TaskTarget target) const; /** * @brief Extracts the processor ranges for a given set of processor types and creates a fresh * machine descriptor with them @@ -231,7 +205,7 @@ struct MachineDesc { * * @return Machine descriptor with the chosen processor ranges */ - MachineDesc only(const std::vector& targets) const; + Machine only(const std::vector& targets) const; /** * @brief Slices the processor range for a given processor type * @@ -242,7 +216,7 @@ struct MachineDesc { * * @return Machine descriptor with the chosen procssor range sliced */ - MachineDesc slice(uint32_t from, uint32_t to, TaskTarget target, bool keep_others = false) const; + Machine slice(uint32_t from, uint32_t to, TaskTarget target, bool keep_others = false) const; /** * @brief Slices the processor range for the preferred processor type of this machine descriptor * @@ -252,8 +226,7 @@ struct MachineDesc { * * @return Machine descriptor with the preferred processor range sliced */ - MachineDesc slice(uint32_t from, uint32_t to, bool keep_others = false) const; - + Machine slice(uint32_t from, uint32_t to, bool keep_others = false) const; /** * @brief Selects the processor range for a given processor type and constructs a machine * descriptor with it. @@ -264,7 +237,7 @@ struct MachineDesc { * * @return Machine descriptor with the chosen processor range */ - MachineDesc operator[](TaskTarget target) const; + Machine operator[](TaskTarget target) const; /** * @brief Selects the processor ranges for a given set of processor types and constructs a machine * descriptor with them. @@ -275,9 +248,9 @@ struct MachineDesc { * * @return Machine descriptor with the chosen processor ranges */ - MachineDesc operator[](const std::vector& targets) const; - bool operator==(const MachineDesc& other) const; - bool operator!=(const MachineDesc& other) const; + Machine operator[](const std::vector& targets) const; + bool operator==(const Machine& other) const; + bool operator!=(const Machine& other) const; /** * @brief Computes an intersection between two machine descriptors * @@ -285,8 +258,7 @@ struct MachineDesc { * * @return Machine descriptor */ - MachineDesc operator&(const MachineDesc& other) const; - + Machine operator&(const Machine& other) const; /** * @brief Indicates whether the machine descriptor is empty. * @@ -297,100 +269,26 @@ struct MachineDesc { */ bool empty() const; - /** - * @brief Preferred processor type of this machine descriptor - */ - TaskTarget preferred_target{TaskTarget::CPU}; - /** - * @brief Processor ranges in this machine descriptor - */ - std::map processor_ranges{}; -}; - -std::ostream& operator<<(std::ostream& stream, const MachineDesc& info); - -class Machine; - -class LocalProcessorRange { - private: - friend class Machine; - LocalProcessorRange(); - LocalProcessorRange(const std::vector& procs); - LocalProcessorRange(uint32_t offset, - uint32_t total_proc_count, - const Processor* local_procs, - size_t num_local_procs); - - public: - const Processor& first() const { return *procs_.begin(); } - const Processor& operator[](uint32_t idx) const; - - public: - bool empty() const { return procs_.size() == 0; } - - public: - std::string to_string() const; - - private: - uint32_t offset_; - uint32_t total_proc_count_; - Span procs_; -}; - -std::ostream& operator<<(std::ostream& stream, const LocalProcessorRange& info); - -class Machine { - public: - Machine(); - - public: - const std::vector& cpus() const { return cpus_; } - const std::vector& gpus() const { return gpus_; } - const std::vector& omps() const { return omps_; } - const std::vector& procs(TaskTarget target) const; - - public: - size_t total_cpu_count() const { return total_nodes * cpus_.size(); } - size_t total_gpu_count() const { return total_nodes * gpus_.size(); } - size_t total_omp_count() const { return total_nodes * omps_.size(); } - public: - size_t total_frame_buffer_size() const; - size_t total_socket_memory_size() const; + Machine(detail::Machine* impl); + Machine(const detail::Machine& impl); public: - bool has_cpus() const { return !cpus_.empty(); } - bool has_gpus() const { return !gpus_.empty(); } - bool has_omps() const { return !omps_.empty(); } + Machine(const Machine&); + Machine& operator=(const Machine&); + Machine(Machine&&); + Machine& operator=(Machine&&); public: - bool has_socket_memory() const; + ~Machine(); public: - Memory get_memory(Processor proc, StoreTarget target) const; - Memory system_memory() const { return system_memory_; } - Memory zerocopy_memory() const { return zerocopy_memory_; } - const std::map& frame_buffers() const { return frame_buffers_; } - const std::map& socket_memories() const { return socket_memories_; } - - public: - LocalProcessorRange slice(TaskTarget target, - const MachineDesc& machine_desc, - bool fallback_to_global = false) const; - - public: - const uint32_t local_node; - const uint32_t total_nodes; + detail::Machine* impl() const { return impl_; } private: - std::vector cpus_; - std::vector gpus_; - std::vector omps_; - - private: - Memory system_memory_, zerocopy_memory_; - std::map frame_buffers_; - std::map socket_memories_; + detail::Machine* impl_{nullptr}; }; +std::ostream& operator<<(std::ostream& stream, const Machine& machine); + } // namespace legate::mapping diff --git a/src/core/mapping/mapping.cc b/src/core/mapping/mapping.cc index 6c1e17c292..e4efeef977 100644 --- a/src/core/mapping/mapping.cc +++ b/src/core/mapping/mapping.cc @@ -14,215 +14,282 @@ * */ -#include - -#include "legate.h" - #include "core/mapping/mapping.h" -#include "core/mapping/store.h" + +#include "core/mapping/detail/mapping.h" +#include "core/mapping/detail/store.h" namespace legate::mapping { -DimOrdering::DimOrdering(Kind _kind, std::vector&& _dims) - : kind(_kind), dims(std::move(_dims)) +std::ostream& operator<<(std::ostream& stream, const TaskTarget& target) { + switch (target) { + case TaskTarget::GPU: { + stream << "GPU"; + break; + } + case TaskTarget::OMP: { + stream << "OMP"; + break; + } + case TaskTarget::CPU: { + stream << "CPU"; + break; + } + } + return stream; } -/*static*/ DimOrdering DimOrdering::c_order() { return DimOrdering(Kind::C); } +std::ostream& operator<<(std::ostream& stream, const StoreTarget& target) +{ + switch (target) { + case StoreTarget::SYSMEM: { + stream << "SYSMEM"; + break; + } + case StoreTarget::FBMEM: { + stream << "FBMEM"; + break; + } + case StoreTarget::ZCMEM: { + stream << "ZCMEM"; + break; + } + case StoreTarget::SOCKETMEM: { + stream << "SOCKETMEM"; + break; + } + } + return stream; +} -/*static*/ DimOrdering DimOrdering::fortran_order() { return DimOrdering(Kind::FORTRAN); } +/*static*/ DimOrdering DimOrdering::c_order() +{ + static DimOrdering ordering(std::make_shared(Kind::C)); + return ordering; +} -/*static*/ DimOrdering DimOrdering::custom_order(std::vector&& dims) +/*static*/ DimOrdering DimOrdering::fortran_order() { - return DimOrdering(Kind::CUSTOM, std::move(dims)); + static DimOrdering ordering(std::make_shared(Kind::FORTRAN)); + return ordering; } -Memory::Kind get_memory_kind(StoreTarget target) +/*static*/ DimOrdering DimOrdering::custom_order(const std::vector& dims) { - switch (target) { - case StoreTarget::SYSMEM: return Memory::Kind::SYSTEM_MEM; - case StoreTarget::FBMEM: return Memory::Kind::GPU_FB_MEM; - case StoreTarget::ZCMEM: return Memory::Kind::Z_COPY_MEM; - case StoreTarget::SOCKETMEM: return Memory::Kind::SOCKET_MEM; - default: LEGATE_ABORT; - } - assert(false); - return Memory::Kind::SYSTEM_MEM; + return DimOrdering(std::make_shared(dims)); } -bool DimOrdering::operator==(const DimOrdering& other) const +void DimOrdering::set_c_order() { *this = c_order(); } + +void DimOrdering::set_fortran_order() { *this = fortran_order(); } + +void DimOrdering::set_custom_order(const std::vector& dims) { - return kind == other.kind && relative == other.relative && dims == other.dims; + impl_ = std::make_shared(dims); } -void DimOrdering::populate_dimension_ordering(const Store& store, - std::vector& ordering) const +DimOrdering::Kind DimOrdering::kind() const { return impl_->kind; } + +std::vector DimOrdering::dimensions() const { return impl_->dims; } + +bool DimOrdering::operator==(const DimOrdering& other) const { return *impl_ == *other.impl_; } + +DimOrdering::DimOrdering() { impl_ = std::move(c_order().impl_); } + +DimOrdering::DimOrdering(const DimOrdering&) = default; + +DimOrdering& DimOrdering::operator=(const DimOrdering&) = default; + +DimOrdering::DimOrdering(DimOrdering&&) = default; + +DimOrdering& DimOrdering::operator=(DimOrdering&&) = default; + +DimOrdering::DimOrdering(std::shared_ptr impl) : impl_(std::move(impl)) {} + +const detail::DimOrdering* DimOrdering::impl() const { return impl_.get(); } + +InstanceMappingPolicy& InstanceMappingPolicy::with_target(StoreTarget _target) & { - // TODO: We need to implement the relative dimension ordering - assert(!relative); - switch (kind) { - case Kind::C: { - auto dim = store.region_field().dim(); - for (int32_t idx = dim - 1; idx >= 0; --idx) - ordering.push_back(static_cast(DIM_X + idx)); - break; - } - case Kind::FORTRAN: { - auto dim = store.region_field().dim(); - for (int32_t idx = 0; idx < dim; ++idx) - ordering.push_back(static_cast(DIM_X + idx)); - break; - } - case Kind::CUSTOM: { - for (auto idx : dims) ordering.push_back(static_cast(DIM_X + idx)); - break; - } - } + set_target(_target); + return *this; } -void DimOrdering::set_c_order() { kind = Kind::C; } +InstanceMappingPolicy&& InstanceMappingPolicy::with_target(StoreTarget _target) const& +{ + return InstanceMappingPolicy(*this).with_target(_target); +} -void DimOrdering::set_fortran_order() { kind = Kind::FORTRAN; } +InstanceMappingPolicy&& InstanceMappingPolicy::with_target(StoreTarget _target) && +{ + return std::move(this->with_target(_target)); +} -void DimOrdering::set_custom_order(std::vector&& dims) +InstanceMappingPolicy& InstanceMappingPolicy::with_allocation_policy(AllocPolicy _allocation) & { - kind = Kind::CUSTOM; - dims = std::move(dims); + set_allocation_policy(_allocation); + return *this; } -bool InstanceMappingPolicy::operator==(const InstanceMappingPolicy& other) const +InstanceMappingPolicy&& InstanceMappingPolicy::with_allocation_policy( + AllocPolicy _allocation) const& { - return target == other.target && allocation == other.allocation && layout == other.layout && - exact == other.exact && ordering == other.ordering; + return InstanceMappingPolicy(*this).with_allocation_policy(_allocation); } -bool InstanceMappingPolicy::operator!=(const InstanceMappingPolicy& other) const +InstanceMappingPolicy&& InstanceMappingPolicy::with_allocation_policy(AllocPolicy _allocation) && { - return !operator==(other); + return std::move(this->with_allocation_policy(_allocation)); } -bool InstanceMappingPolicy::subsumes(const InstanceMappingPolicy& other) const +InstanceMappingPolicy& InstanceMappingPolicy::with_instance_layout(InstLayout _layout) & { - // the allocation policy doesn't concern the instance layout - return target == other.target && layout == other.layout && ordering == other.ordering && - (exact || !other.exact); + set_instance_layout(_layout); + return *this; } -void InstanceMappingPolicy::populate_layout_constraints( - const Store& store, Legion::LayoutConstraintSet& layout_constraints) const +InstanceMappingPolicy&& InstanceMappingPolicy::with_instance_layout(InstLayout _layout) const& { - std::vector dimension_ordering{}; + return InstanceMappingPolicy(*this).with_instance_layout(_layout); +} - if (layout == InstLayout::AOS) dimension_ordering.push_back(DIM_F); - ordering.populate_dimension_ordering(store, dimension_ordering); - if (layout == InstLayout::SOA) dimension_ordering.push_back(DIM_F); +InstanceMappingPolicy&& InstanceMappingPolicy::with_instance_layout(InstLayout _layout) && +{ + return std::move(this->with_instance_layout(_layout)); +} - layout_constraints.add_constraint( - Legion::OrderingConstraint(dimension_ordering, false /*contiguous*/)); +InstanceMappingPolicy& InstanceMappingPolicy::with_ordering(DimOrdering _ordering) & +{ + set_ordering(_ordering); + return *this; +} - layout_constraints.add_constraint(Legion::MemoryConstraint(get_memory_kind(target))); +InstanceMappingPolicy&& InstanceMappingPolicy::with_ordering(DimOrdering _ordering) const& +{ + return InstanceMappingPolicy(*this).with_ordering(_ordering); } -/*static*/ InstanceMappingPolicy InstanceMappingPolicy::default_policy(StoreTarget target, - bool exact) +InstanceMappingPolicy&& InstanceMappingPolicy::with_ordering(DimOrdering _ordering) && { - InstanceMappingPolicy policy{}; - policy.target = target; - policy.exact = exact; - return policy; + return std::move(this->with_ordering(_ordering)); } -bool StoreMapping::for_future() const +InstanceMappingPolicy& InstanceMappingPolicy::with_exact(bool _exact) & { - for (auto& store : stores) return store.get().is_future(); - assert(false); - return false; + set_exact(_exact); + return *this; } -bool StoreMapping::for_unbound_store() const +InstanceMappingPolicy&& InstanceMappingPolicy::with_exact(bool _exact) const& { - for (auto& store : stores) return store.get().unbound(); - assert(false); - return false; + return InstanceMappingPolicy(*this).with_exact(_exact); } -const Store& StoreMapping::store() const +InstanceMappingPolicy&& InstanceMappingPolicy::with_exact(bool _exact) && { -#ifdef DEBUG_LEGATE - assert(stores.size() == 1); -#endif - return stores.front(); + return std::move(this->with_exact(_exact)); } -uint32_t StoreMapping::requirement_index() const +void InstanceMappingPolicy::set_target(StoreTarget _target) { target = _target; } + +void InstanceMappingPolicy::set_allocation_policy(AllocPolicy _allocation) { -#ifdef DEBUG_LEGATE - assert(stores.size() > 0); - uint32_t result = -1U; - for (auto& store : stores) { - auto idx = store.get().requirement_index(); - assert(result == -1U || result == idx); - result = idx; - } - return result; -#else - static constexpr uint32_t invalid = -1U; - if (stores.empty()) return invalid; - return stores.front().get().requirement_index(); -#endif + allocation = _allocation; } -std::set StoreMapping::requirement_indices() const +void InstanceMappingPolicy::set_instance_layout(InstLayout _layout) { layout = _layout; } + +void InstanceMappingPolicy::set_ordering(DimOrdering _ordering) { ordering = _ordering; } + +void InstanceMappingPolicy::set_exact(bool _exact) { exact = _exact; } + +bool InstanceMappingPolicy::subsumes(const InstanceMappingPolicy& other) const { - std::set indices; - for (auto& store : stores) { - if (store.get().is_future()) continue; - indices.insert(store.get().region_field().index()); - } - return indices; + // the allocation policy doesn't concern the instance layout + return target == other.target && layout == other.layout && ordering == other.ordering && + (exact || !other.exact); } -std::set StoreMapping::requirements() const +InstanceMappingPolicy::InstanceMappingPolicy() {} + +InstanceMappingPolicy::~InstanceMappingPolicy() {} + +InstanceMappingPolicy::InstanceMappingPolicy(const InstanceMappingPolicy&) = default; + +InstanceMappingPolicy& InstanceMappingPolicy::operator=(const InstanceMappingPolicy&) = default; + +InstanceMappingPolicy::InstanceMappingPolicy(InstanceMappingPolicy&&) = default; + +InstanceMappingPolicy& InstanceMappingPolicy::operator=(InstanceMappingPolicy&&) = default; + +bool InstanceMappingPolicy::operator==(const InstanceMappingPolicy& other) const { - std::set reqs; - for (auto& store : stores) { - if (store.get().is_future()) continue; - auto* req = store.get().region_field().get_requirement(); - if (!req->region.exists()) continue; - reqs.insert(req); + return target == other.target && allocation == other.allocation && layout == other.layout && + exact == other.exact && ordering == other.ordering; +} + +bool InstanceMappingPolicy::operator!=(const InstanceMappingPolicy& other) const +{ + return !operator==(other); +} + +/*static*/ StoreMapping StoreMapping::default_mapping(Store store, StoreTarget target, bool exact) +{ + return create(store, InstanceMappingPolicy{}.with_target(target).with_exact(exact)); +} + +/*static*/ StoreMapping StoreMapping::create(Store store, InstanceMappingPolicy&& policy) +{ + return StoreMapping(detail::StoreMapping::create(store.impl(), std::move(policy)).release()); +} + +/*static*/ StoreMapping StoreMapping::create(const std::vector& stores, + InstanceMappingPolicy&& policy) +{ + if (stores.empty()) { + throw std::invalid_argument("Invalid to create a store mapping without any store"); } - return reqs; + auto mapping = new detail::StoreMapping(); + mapping->policy = std::move(policy); + for (auto& store : stores) { mapping->stores.push_back(store.impl()); } + return StoreMapping(mapping); } -void StoreMapping::populate_layout_constraints( - Legion::LayoutConstraintSet& layout_constraints) const +InstanceMappingPolicy& StoreMapping::policy() { return impl_->policy; } + +const InstanceMappingPolicy& StoreMapping::policy() const { return impl_->policy; } + +Store StoreMapping::store() const { return Store(impl_->stores.front()); } + +std::vector StoreMapping::stores() const { - policy.populate_layout_constraints(stores.front(), layout_constraints); + std::vector result; + for (auto& store : impl_->stores) { result.emplace_back(store); } + return std::move(result); +} - std::vector fields{}; - if (stores.size() > 1) { - std::set field_set{}; - for (auto& store : stores) { - auto field_id = store.get().region_field().field_id(); - if (field_set.find(field_id) == field_set.end()) { - fields.push_back(field_id); - field_set.insert(field_id); - } - } - } else - fields.push_back(stores.front().get().region_field().field_id()); - layout_constraints.add_constraint( - Legion::FieldConstraint(fields, false /*contiguous*/, false /*inorder*/)); +void StoreMapping::add_store(Store store) { impl_->stores.push_back(store.impl()); } + +StoreMapping::StoreMapping(detail::StoreMapping* impl) : impl_(impl) {} + +const detail::StoreMapping* StoreMapping::impl() const { return impl_; } + +StoreMapping::StoreMapping(StoreMapping&& other) : impl_(other.impl_) { other.impl_ = nullptr; } + +StoreMapping& StoreMapping::operator=(StoreMapping&& other) +{ + impl_ = other.impl_; + other.impl_ = nullptr; + return *this; } -/*static*/ StoreMapping StoreMapping::default_mapping(const Store& store, - StoreTarget target, - bool exact) +StoreMapping::~StoreMapping() { delete impl_; } + +detail::StoreMapping* StoreMapping::release() { - StoreMapping mapping{}; - mapping.policy = InstanceMappingPolicy::default_policy(target, exact); - mapping.stores.emplace_back(store); - return mapping; + auto result = impl_; + impl_ = nullptr; + return result; } } // namespace legate::mapping diff --git a/src/core/mapping/mapping.h b/src/core/mapping/mapping.h index 3889f0bbf8..4e51b54505 100644 --- a/src/core/mapping/mapping.h +++ b/src/core/mapping/mapping.h @@ -1,4 +1,4 @@ -/* Copyright 2021-2022 NVIDIA Corporation +/* Copyright 2023 NVIDIA Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,9 @@ #pragma once #include +#include #include "core/data/scalar.h" +#include "core/mapping/store.h" /** @defgroup mapping Mapping API */ @@ -29,7 +31,12 @@ namespace legate::mapping { -class Store; +namespace detail { +class BaseMapper; +class DimOrdering; +class StoreMapping; +} // namespace detail + class Task; // NOTE: codes are chosen to reflect the precedence between the processor kinds in choosing target @@ -57,6 +64,8 @@ enum class TaskTarget : int32_t { CPU = 3, }; +std::ostream& operator<<(std::ostream& stream, const TaskTarget& target); + /** * @ingroup mapping * @brief An enum class for store targets @@ -80,6 +89,8 @@ enum class StoreTarget : int32_t { SOCKETMEM = 4, }; +std::ostream& operator<<(std::ostream& stream, const StoreTarget& target); + /** * @ingroup mapping * @brief An enum class for instance allocation policies @@ -137,10 +148,6 @@ struct DimOrdering { CUSTOM = 3, }; - public: - DimOrdering() {} - DimOrdering(Kind kind, std::vector&& dims = {}); - public: /** * @brief Creates a C ordering object @@ -161,22 +168,7 @@ struct DimOrdering { * * @return A `DimOrdering` object */ - static DimOrdering custom_order(std::vector&& dims); - - public: - DimOrdering(const DimOrdering&) = default; - DimOrdering& operator=(const DimOrdering&) = default; - - public: - DimOrdering(DimOrdering&&) = default; - DimOrdering& operator=(DimOrdering&&) = default; - - public: - bool operator==(const DimOrdering&) const; - - public: - void populate_dimension_ordering(const Store& store, - std::vector& ordering) const; + static DimOrdering custom_order(const std::vector& dims); public: /** @@ -192,26 +184,36 @@ struct DimOrdering { * * @param dims A vector that stores the order of dimensions. */ - void set_custom_order(std::vector&& dims); + void set_custom_order(const std::vector& dims); public: /** * @brief Dimension ordering type */ - Kind kind{Kind::C}; - // When relative is true, 'dims' specifies the order of dimensions - // for the store's local coordinate space, which will be mapped - // back to the root store's original coordinate space. - /** - * @brief If true, the dimension ordering specifies the order of dimensions - * for the store's current domain, which will be transformed back to the root - * store's domain. - */ - bool relative{false}; + Kind kind() const; /** * @brief Dimension list. Used only when the `kind` is `CUSTOM`. */ - std::vector dims{}; + std::vector dimensions() const; + + public: + bool operator==(const DimOrdering&) const; + + private: + DimOrdering(std::shared_ptr impl); + + public: + const detail::DimOrdering* impl() const; + + public: + DimOrdering(); + DimOrdering(const DimOrdering&); + DimOrdering& operator=(const DimOrdering&); + DimOrdering(DimOrdering&&); + DimOrdering& operator=(DimOrdering&&); + + private: + std::shared_ptr impl_; }; /** @@ -233,7 +235,7 @@ struct InstanceMappingPolicy { */ InstLayout layout{InstLayout::SOA}; /** - * @brief Dimension ordering for the instance + * @brief Dimension ordering for the instance. C order by default. */ DimOrdering ordering{}; /** @@ -243,19 +245,88 @@ struct InstanceMappingPolicy { bool exact{false}; public: - InstanceMappingPolicy() {} - - public: - InstanceMappingPolicy(const InstanceMappingPolicy&) = default; - InstanceMappingPolicy& operator=(const InstanceMappingPolicy&) = default; - - public: - InstanceMappingPolicy(InstanceMappingPolicy&&) = default; - InstanceMappingPolicy& operator=(InstanceMappingPolicy&&) = default; + /** + * @brief Changes the store target + * + * @param `target` A new store target + * + * @return This instance mapping policy + */ + InstanceMappingPolicy& with_target(StoreTarget target) &; + InstanceMappingPolicy&& with_target(StoreTarget target) const&; + InstanceMappingPolicy&& with_target(StoreTarget target) &&; + /** + * @brief Changes the allocation policy + * + * @param `allocation` A new allocation policy + * + * @return This instance mapping policy + */ + InstanceMappingPolicy& with_allocation_policy(AllocPolicy allocation) &; + InstanceMappingPolicy&& with_allocation_policy(AllocPolicy allocation) const&; + InstanceMappingPolicy&& with_allocation_policy(AllocPolicy allocation) &&; + /** + * @brief Changes the instance layout + * + * @param `target` A new instance layout + * + * @return This instance mapping policy + */ + InstanceMappingPolicy& with_instance_layout(InstLayout layout) &; + InstanceMappingPolicy&& with_instance_layout(InstLayout layout) const&; + InstanceMappingPolicy&& with_instance_layout(InstLayout layout) &&; + /** + * @brief Changes the dimension ordering + * + * @param `target` A new dimension ordering + * + * @return This instance mapping policy + */ + InstanceMappingPolicy& with_ordering(DimOrdering ordering) &; + InstanceMappingPolicy&& with_ordering(DimOrdering ordering) const&; + InstanceMappingPolicy&& with_ordering(DimOrdering ordering) &&; + /** + * @brief Changes the value of `exact` + * + * @param `target` A new value for the `exact` field + * + * @return This instance mapping policy + */ + InstanceMappingPolicy& with_exact(bool exact) &; + InstanceMappingPolicy&& with_exact(bool exact) const&; + InstanceMappingPolicy&& with_exact(bool exact) &&; public: - bool operator==(const InstanceMappingPolicy&) const; - bool operator!=(const InstanceMappingPolicy&) const; + /** + * @brief Changes the store target + * + * @param `target` A new store target + */ + void set_target(StoreTarget target); + /** + * @brief Changes the allocation policy + * + * @param `allocation` A new allocation policy + */ + void set_allocation_policy(AllocPolicy allocation); + /** + * @brief Changes the instance layout + * + * @param `target` A new instance layout + */ + void set_instance_layout(InstLayout layout); + /** + * @brief Changes the dimension ordering + * + * @param `target` A new dimension ordering + */ + void set_ordering(DimOrdering ordering); + /** + * @brief Changes the value of `exact` + * + * @param `target` A new value for the `exact` field + */ + void set_exact(bool exact); public: /** @@ -270,13 +341,19 @@ struct InstanceMappingPolicy { */ bool subsumes(const InstanceMappingPolicy& other) const; - private: - friend class StoreMapping; - void populate_layout_constraints(const Store& store, - Legion::LayoutConstraintSet& layout_constraints) const; + public: + InstanceMappingPolicy(); + ~InstanceMappingPolicy(); public: - static InstanceMappingPolicy default_policy(StoreTarget target, bool exact = false); + InstanceMappingPolicy(const InstanceMappingPolicy&); + InstanceMappingPolicy& operator=(const InstanceMappingPolicy&); + InstanceMappingPolicy(InstanceMappingPolicy&&); + InstanceMappingPolicy& operator=(InstanceMappingPolicy&&); + + public: + bool operator==(const InstanceMappingPolicy&) const; + bool operator!=(const InstanceMappingPolicy&) const; }; /** @@ -286,68 +363,96 @@ struct InstanceMappingPolicy { struct StoreMapping { public: /** - * @brief Stores to which the `policy` should be applied + * @brief Creates a mapping policy for the given store following the default mapping poicy + * + * @param store Target store + * @param target Kind of the memory to which the store should be mapped + * @param exact Indicates whether the instance should be exact + * + * @return A store mapping */ - std::vector> stores{}; + static StoreMapping default_mapping(Store store, StoreTarget target, bool exact = false); /** - * @brief Instance mapping policy + * @brief Creates a mapping policy for the given store using the instance mapping policy + * + * @param store Target store for the mapping policy + * @param policy Instance mapping policy to apply + * + * @return A store mapping */ - InstanceMappingPolicy policy; - - public: - StoreMapping() {} + static StoreMapping create(Store store, InstanceMappingPolicy&& policy); - public: - StoreMapping(const StoreMapping&) = default; - StoreMapping& operator=(const StoreMapping&) = default; - - public: - StoreMapping(StoreMapping&&) = default; - StoreMapping& operator=(StoreMapping&&) = default; + /** + * @brief Creates a mapping policy for the given set of stores using the instance mapping policy + * + * @param store Target store for the mapping policy + * @param policy Instance mapping policy to apply + * + * @return A store mapping + */ + static StoreMapping create(const std::vector& stores, InstanceMappingPolicy&& policy); public: - bool for_future() const; - bool for_unbound_store() const; - const Store& store() const; + /** + * @brief Returns the instance mapping policy of this `StoreMapping` object + * + * @return A reference to the `InstanceMappingPolicy` object + */ + InstanceMappingPolicy& policy(); + /** + * @brief Returns the instance mapping policy of this `StoreMapping` object + * + * @return A reference to the `InstanceMappingPolicy` object + */ + const InstanceMappingPolicy& policy() const; public: /** - * @brief Returns a region requirement index for the stores. + * @brief Returns the store for which this `StoreMapping` object describes a mapping policy. * - * Returns an undefined value if the store mapping has more than one store and the stores are - * mapped to different region requirements. + * If the policy is for multiple stores, the first store added to this policy will be returned; * - * @return Region requirement index + * @return A `Store` object */ - uint32_t requirement_index() const; + Store store() const; /** - * @brief Returns a set of region requirement indices for the stores. + * @brief Returns all the stores for which this `StoreMapping` object describes a mapping policy * - * @return A set of region requirement indices + * @return A vector of `Store` objects */ - std::set requirement_indices() const; + std::vector stores() const; + + public: /** - * @brief Returns the stores' region requirements + * @brief Adds a store to this `StoreMapping` object * - * @return A set of region requirements + * @param store Store to add */ - std::set requirements() const; + void add_store(Store store); private: - friend class BaseMapper; - void populate_layout_constraints(Legion::LayoutConstraintSet& layout_constraints) const; + StoreMapping(detail::StoreMapping* impl); public: - /** - * @brief Creates a `StoreMapping` object following the default mapping poicy - * - * @param store Target store for the mapping policy - * @param target Target memory type for the store - * @param exact Indicates whether the policy should request an exact instance - * - * @return A `StoreMapping` object - */ - static StoreMapping default_mapping(const Store& store, StoreTarget target, bool exact = false); + const detail::StoreMapping* impl() const; + + private: + StoreMapping(const StoreMapping&) = delete; + StoreMapping& operator=(const StoreMapping&) = delete; + + public: + StoreMapping(StoreMapping&&); + StoreMapping& operator=(StoreMapping&&); + + public: + ~StoreMapping(); + + private: + friend class detail::BaseMapper; + detail::StoreMapping* release(); + + private: + detail::StoreMapping* impl_{nullptr}; }; /** diff --git a/src/core/mapping/operation.cc b/src/core/mapping/operation.cc index 279e28481c..af2b5cc76f 100644 --- a/src/core/mapping/operation.cc +++ b/src/core/mapping/operation.cc @@ -1,4 +1,4 @@ -/* Copyright 2021-2022 NVIDIA Corporation +/* Copyright 2023 NVIDIA Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,76 +15,47 @@ */ #include "core/mapping/operation.h" -#include "core/runtime/context.h" -#include "core/utilities/deserializer.h" +#include "core/mapping/detail/operation.h" namespace legate::mapping { -Mappable::Mappable() {} +int64_t Task::task_id() const { return impl_->task_id(); } -Mappable::Mappable(const Legion::Mappable* mappable) -{ - MapperDataDeserializer dez(mappable); - machine_desc_ = dez.unpack(); - sharding_id_ = dez.unpack(); -} +namespace { -Task::Task(const Legion::Task* task, - const LibraryContext* library, - Legion::Mapping::MapperRuntime* runtime, - const Legion::Mapping::MapperContext context) - : Mappable(task), task_(task), library_(library) +template +std::vector convert_stores(const Stores& stores) { - TaskDeserializer dez(task, runtime, context); - inputs_ = dez.unpack>(); - outputs_ = dez.unpack>(); - reductions_ = dez.unpack>(); - scalars_ = dez.unpack>(); + std::vector result; + for (auto& store : stores) { result.emplace_back(&store); } + return std::move(result); } -int64_t Task::task_id() const { return library_->get_local_task_id(task_->task_id); } +} // namespace -TaskTarget Task::target() const -{ - switch (task_->target_proc.kind()) { - case Processor::LOC_PROC: return TaskTarget::CPU; - case Processor::TOC_PROC: return TaskTarget::GPU; - case Processor::OMP_PROC: return TaskTarget::OMP; - default: { - assert(false); - } - } - assert(false); - return TaskTarget::CPU; -} +std::vector Task::inputs() const { return convert_stores(impl_->inputs()); } -Copy::Copy(const Legion::Copy* copy, - Legion::Mapping::MapperRuntime* runtime, - const Legion::Mapping::MapperContext context) - : Mappable(), copy_(copy) -{ - CopyDeserializer dez(copy, - {copy->src_requirements, - copy->dst_requirements, - copy->src_indirect_requirements, - copy->dst_indirect_requirements}, - runtime, - context); - machine_desc_ = dez.unpack(); - sharding_id_ = dez.unpack(); - inputs_ = dez.unpack>(); - dez.next_requirement_list(); - outputs_ = dez.unpack>(); - dez.next_requirement_list(); - input_indirections_ = dez.unpack>(); - dez.next_requirement_list(); - output_indirections_ = dez.unpack>(); -#ifdef DEBUG_LEGATE - for (auto& input : inputs_) assert(!input.is_future()); - for (auto& output : outputs_) assert(!output.is_future()); - for (auto& input_indirection : input_indirections_) assert(!input_indirection.is_future()); - for (auto& output_indirection : output_indirections_) assert(!output_indirection.is_future()); -#endif -} +std::vector Task::outputs() const { return convert_stores(impl_->outputs()); } + +std::vector Task::reductions() const { return convert_stores(impl_->reductions()); } + +const std::vector& Task::scalars() const { return impl_->scalars(); } + +Store Task::input(uint32_t index) const { return Store(&impl_->inputs().at(index)); } + +Store Task::output(uint32_t index) const { return Store(&impl_->outputs().at(index)); } + +Store Task::reduction(uint32_t index) const { return Store(&impl_->reductions().at(index)); } + +size_t Task::num_inputs() const { return impl_->inputs().size(); } + +size_t Task::num_outputs() const { return impl_->outputs().size(); } + +size_t Task::num_reductions() const { return impl_->reductions().size(); } + +Task::Task(detail::Task* impl) : impl_(impl) {} + +// The impl is owned by the caller, so we don't need to deallocate it +Task::~Task() {} } // namespace legate::mapping diff --git a/src/core/mapping/operation.h b/src/core/mapping/operation.h index 9f39e0d377..71b325d32e 100644 --- a/src/core/mapping/operation.h +++ b/src/core/mapping/operation.h @@ -1,4 +1,4 @@ -/* Copyright 2021-2022 NVIDIA Corporation +/* Copyright 2023 NVIDIA Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,10 +17,8 @@ #pragma once #include -#include #include "core/data/scalar.h" -#include "core/mapping/machine.h" #include "core/mapping/store.h" /** @@ -28,40 +26,17 @@ * @brief Class definitions for operations and stores used in mapping */ -namespace legate { -class LibraryContext; -class TransformStack; -} // namespace legate - namespace legate::mapping { -class Mappable { - protected: - Mappable(); - - public: - Mappable(const Legion::Mappable* mappable); - - public: - const mapping::MachineDesc& machine_desc() const { return machine_desc_; } - uint32_t sharding_id() const { return sharding_id_; } - - protected: - mapping::MachineDesc machine_desc_; - uint32_t sharding_id_; -}; +namespace detail { +class Task; +} // namespace detail /** * @ingroup mapping * @brief A metadata class for tasks */ -class Task : public Mappable { - public: - Task(const Legion::Task* task, - const LibraryContext* library, - Legion::Mapping::MapperRuntime* runtime, - const Legion::Mapping::MapperContext context); - +class Task { public: /** * @brief Returns the task id @@ -76,19 +51,19 @@ class Task : public Mappable { * * @return Vector of store metadata objects */ - const std::vector& inputs() const { return inputs_; } + std::vector inputs() const; /** * @brief Returns metadata for the task's output stores * * @return Vector of store metadata objects */ - const std::vector& outputs() const { return outputs_; } + std::vector outputs() const; /** * @brief Returns metadata for the task's reduction stores * * @return Vector of store metadata objects */ - const std::vector& reductions() const { return reductions_; } + std::vector reductions() const; /** * @brief Returns the vector of the task's by-value arguments. Unlike `mapping::Store` * objects that have no access to data in the stores, the returned `Scalar` objects @@ -96,53 +71,66 @@ class Task : public Mappable { * * @return Vector of `Scalar` objects */ - const std::vector& scalars() const { return scalars_; } + const std::vector& scalars() const; public: /** - * @brief Returns the point of the task + * @brief Returns metadata for the task's input store + * + * @param index Index of the input store * - * @return The point of the task + * @return Store metadata object */ - DomainPoint point() const { return task_->index_point; } - - public: - TaskTarget target() const; - - private: - const LibraryContext* library_; - const Legion::Task* task_; - - private: - std::vector inputs_, outputs_, reductions_; - std::vector scalars_; -}; - -class Copy : public Mappable { - public: - Copy(const Legion::Copy* copy, - Legion::Mapping::MapperRuntime* runtime, - const Legion::Mapping::MapperContext context); + Store input(uint32_t index) const; + /** + * @brief Returns metadata for the task's output store + * + * @param index Index of the output store + * + * @return Store metadata object + */ + Store output(uint32_t index) const; + /** + * @brief Returns metadata for the task's reduction store + * + * @param index Index of the reduction store + * + * @return Store metadata object + */ + Store reduction(uint32_t index) const; public: - const std::vector& inputs() const { return inputs_; } - const std::vector& outputs() const { return outputs_; } - const std::vector& input_indirections() const { return input_indirections_; } - const std::vector& output_indirections() const { return output_indirections_; } + /** + * @brief Returns the number of task's inputs + * + * @return Number of stores + */ + size_t num_inputs() const; + /** + * @brief Returns the number of task's outputs + * + * @return Number of stores + */ + size_t num_outputs() const; + /** + * @brief Returns the number of task's reductions + * + * @return Number of stores + */ + size_t num_reductions() const; public: - DomainPoint point() const { return copy_->index_point; } + Task(detail::Task* impl_); + ~Task(); private: - const Legion::Copy* copy_; + Task(const Task&) = delete; + Task& operator=(const Task&) = delete; + Task(Task&&) = delete; + Task& operator=(Task&&) = delete; private: - std::vector inputs_; - std::vector outputs_; - std::vector input_indirections_; - std::vector output_indirections_; + detail::Task* impl_{nullptr}; }; } // namespace legate::mapping - -#include "core/mapping/operation.inl" diff --git a/src/core/mapping/store.cc b/src/core/mapping/store.cc index 0b5adcfb6b..ce39ac28c9 100644 --- a/src/core/mapping/store.cc +++ b/src/core/mapping/store.cc @@ -1,4 +1,4 @@ -/* Copyright 2022 NVIDIA Corporation +/* Copyright 2023 NVIDIA Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,130 +15,38 @@ */ #include "core/mapping/store.h" +#include "core/mapping/detail/store.h" namespace legate::mapping { -RegionField::RegionField(const Legion::RegionRequirement* req, - int32_t dim, - uint32_t idx, - Legion::FieldID fid) - : req_(req), dim_(dim), idx_(idx), fid_(fid) -{ -} - -bool RegionField::can_colocate_with(const RegionField& other) const -{ - auto* my_req = get_requirement(); - auto* other_req = other.get_requirement(); - return my_req->region.get_tree_id() == other_req->region.get_tree_id(); -} - -Domain RegionField::domain(Legion::Mapping::MapperRuntime* runtime, - const Legion::Mapping::MapperContext context) const -{ - return runtime->get_index_space_domain(context, get_index_space()); -} - -Legion::IndexSpace RegionField::get_index_space() const { return req_->region.get_index_space(); } +bool Store::is_future() const { return impl_->is_future(); } -FutureWrapper::FutureWrapper(uint32_t idx, const Domain& domain) : idx_(idx), domain_(domain) {} +bool Store::unbound() const { return impl_->unbound(); } -Domain FutureWrapper::domain() const { return domain_; } - -Store::Store(int32_t dim, - std::unique_ptr type, - FutureWrapper future, - std::shared_ptr&& transform) - : is_future_(true), - is_unbound_store_(false), - dim_(dim), - type_(std::move(type)), - redop_id_(-1), - future_(future), - transform_(std::move(transform)) -{ -} +int32_t Store::dim() const { return impl_->dim(); } -Store::Store(Legion::Mapping::MapperRuntime* runtime, - const Legion::Mapping::MapperContext context, - int32_t dim, - std::unique_ptr type, - int32_t redop_id, - const RegionField& region_field, - bool is_unbound_store, - std::shared_ptr&& transform) - : is_future_(false), - is_unbound_store_(is_unbound_store), - dim_(dim), - type_(std::move(type)), - redop_id_(redop_id), - region_field_(region_field), - transform_(std::move(transform)), - runtime_(runtime), - context_(context) -{ -} +bool Store::is_reduction() const { return impl_->is_reduction(); } -Store::Store(Legion::Mapping::MapperRuntime* runtime, - const Legion::Mapping::MapperContext context, - const Legion::RegionRequirement* requirement) - : is_future_(false), - is_unbound_store_(false), - dim_(requirement->region.get_dim()), - type_(nullptr), - redop_id_(-1), - runtime_(runtime), - context_(context) -{ - region_field_ = RegionField(requirement, dim_, 0, requirement->instance_fields.front()); -} +int32_t Store::redop() const { return impl_->redop(); } bool Store::can_colocate_with(const Store& other) const { - if (is_future() || other.is_future()) - return false; - else if (unbound() || other.unbound()) - return false; - else if (is_reduction() || other.is_reduction()) - return redop() == other.redop() && region_field_.can_colocate_with(other.region_field_); - return region_field_.can_colocate_with(other.region_field_); + return impl_->can_colocate_with(*other.impl_); } -const RegionField& Store::region_field() const -{ -#ifdef DEBUG_LEGATE - assert(!is_future()); -#endif - return region_field_; -} +Domain Store::domain() const { return impl_->domain(); } -const FutureWrapper& Store::future() const -{ -#ifdef DEBUG_LEGATE - assert(is_future()); -#endif - return future_; -} +Store::Store(const detail::Store* impl) : impl_(impl) {} -RegionField::Id Store::unique_region_field_id() const { return region_field().unique_id(); } +Store::Store(const Store& other) = default; -uint32_t Store::requirement_index() const { return region_field().index(); } +Store& Store::operator=(const Store& other) = default; -uint32_t Store::future_index() const { return future().index(); } +Store::Store(Store&& other) = default; -Domain Store::domain() const -{ - assert(!unbound()); - auto result = is_future_ ? future_.domain() : region_field_.domain(runtime_, context_); - if (nullptr != transform_) result = transform_->transform(result); - assert(result.dim == dim_); - return result; -} +Store& Store::operator=(Store&& other) = default; -std::vector Store::find_imaginary_dims() const -{ - if (nullptr != transform_) return transform_->find_imaginary_dims(); - return std::vector(); -} +// Don't need to release impl as it's held by the operation +Store::~Store() {} } // namespace legate::mapping diff --git a/src/core/mapping/store.h b/src/core/mapping/store.h index 0d976f1790..31c211eb00 100644 --- a/src/core/mapping/store.h +++ b/src/core/mapping/store.h @@ -1,4 +1,4 @@ -/* Copyright 2022 NVIDIA Corporation +/* Copyright 2023 NVIDIA Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,80 +16,14 @@ #pragma once -#include "core/data/transform.h" #include "core/type/type_info.h" +#include "core/utilities/typedefs.h" -namespace legate::mapping { - -class RegionField { - public: - using Id = std::tuple; - - public: - RegionField() {} - RegionField(const Legion::RegionRequirement* req, int32_t dim, uint32_t idx, Legion::FieldID fid); - - public: - RegionField(const RegionField& other) = default; - RegionField& operator=(const RegionField& other) = default; - - public: - bool can_colocate_with(const RegionField& other) const; - - public: - template - Legion::Rect shape(Legion::Mapping::MapperRuntime* runtime, - const Legion::Mapping::MapperContext context) const; - - public: - Legion::Domain domain(Legion::Mapping::MapperRuntime* runtime, - const Legion::Mapping::MapperContext context) const; - - public: - bool operator==(const RegionField& other) const; - - public: - Id unique_id() const { return std::make_tuple(unbound(), idx_, fid_); } - - public: - int32_t dim() const { return dim_; } - uint32_t index() const { return idx_; } - Legion::FieldID field_id() const { return fid_; } - bool unbound() const { return dim_ < 0; } +namespace legate::mapping::detail { +class Store; +} // namespace legate::mapping::detail - public: - const Legion::RegionRequirement* get_requirement() const { return req_; } - Legion::IndexSpace get_index_space() const; - - private: - const Legion::RegionRequirement* req_{nullptr}; - int32_t dim_{-1}; - uint32_t idx_{-1U}; - Legion::FieldID fid_{-1U}; -}; - -class FutureWrapper { - public: - FutureWrapper() {} - FutureWrapper(uint32_t idx, const Legion::Domain& domain); - - public: - FutureWrapper(const FutureWrapper& other) = default; - FutureWrapper& operator=(const FutureWrapper& other) = default; - - public: - int32_t dim() const { return domain_.dim; } - uint32_t index() const { return idx_; } - - public: - template - Legion::Rect shape() const; - Legion::Domain domain() const; - - private: - uint32_t idx_{-1U}; - Legion::Domain domain_{}; -}; +namespace legate::mapping { /** * @ingroup mapping @@ -97,33 +31,6 @@ class FutureWrapper { * only the data relevant to mapping */ class Store { - public: - Store() {} - Store(int32_t dim, - std::unique_ptr type, - FutureWrapper future, - std::shared_ptr&& transform = nullptr); - Store(Legion::Mapping::MapperRuntime* runtime, - const Legion::Mapping::MapperContext context, - int32_t dim, - std::unique_ptr type, - int32_t redop_id, - const RegionField& region_field, - bool is_unbound_store = false, - std::shared_ptr&& transform = nullptr); - // A special constructor to create a mapper view of a store from a region requirement - Store(Legion::Mapping::MapperRuntime* runtime, - const Legion::Mapping::MapperContext context, - const Legion::RegionRequirement* requirement); - - public: - Store(const Store& other) = delete; - Store& operator=(const Store& other) = delete; - - public: - Store(Store&& other) = default; - Store& operator=(Store&& other) = default; - public: /** * @brief Indicates whether the store is backed by a future @@ -131,20 +38,20 @@ class Store { * @return true The store is backed by a future * @return false The store is backed by a region field */ - bool is_future() const { return is_future_; } + bool is_future() const; /** * @brief Indicates whether the store is unbound * * @return true The store is unbound * @return false The store is a normal store */ - bool unbound() const { return is_unbound_store_; } + bool unbound() const; /** * @brief Returns the store's dimension * * @return Store's dimension */ - int32_t dim() const { return dim_; } + int32_t dim() const; public: /** @@ -153,13 +60,13 @@ class Store { * @return true The store is a reduction store * @return false The store is either an input or output store */ - bool is_reduction() const { return redop_id_ > 0; } + bool is_reduction() const; /** * @brief Returns the reduction operator id for the store * * @return Reduction oeprator id */ - int32_t redop() const { return redop_id_; } + int32_t redop() const; public: /** @@ -171,13 +78,6 @@ class Store { * @return false The store cannot colocate with the input */ bool can_colocate_with(const Store& other) const; - const RegionField& region_field() const; - const FutureWrapper& future() const; - - public: - RegionField::Id unique_region_field_id() const; - uint32_t requirement_index() const; - uint32_t future_index() const; public: /** @@ -186,7 +86,10 @@ class Store { * @return Store's domain */ template - Rect shape() const; + Rect shape() const + { + return Rect(domain()); + } /** * @brief Returns the store's domain in a dimension-erased domain type * @@ -195,25 +98,20 @@ class Store { Domain domain() const; public: - std::vector find_imaginary_dims() const; + Store(const detail::Store* impl); + const detail::Store* impl() const { return impl_; } - private: - bool is_future_{false}; - bool is_unbound_store_{false}; - int32_t dim_{-1}; - std::unique_ptr type_{nullptr}; - int32_t redop_id_{-1}; - - private: - FutureWrapper future_; - RegionField region_field_; + public: + Store(const Store& other); + Store& operator=(const Store& other); + Store(Store&& other); + Store& operator=(Store&& other); - private: - std::shared_ptr transform_{nullptr}; + public: + ~Store(); private: - Legion::Mapping::MapperRuntime* runtime_{nullptr}; - Legion::Mapping::MapperContext context_{nullptr}; + const detail::Store* impl_{nullptr}; }; } // namespace legate::mapping diff --git a/src/core/operation/detail/copy.cc b/src/core/operation/detail/copy.cc index 794d31fa0a..585f7140c3 100644 --- a/src/core/operation/detail/copy.cc +++ b/src/core/operation/detail/copy.cc @@ -17,21 +17,21 @@ #include "core/operation/detail/copy.h" #include "core/operation/detail/copy_launcher.h" -#include "core/partitioning/constraint.h" -#include "core/partitioning/constraint_solver.h" +#include "core/partitioning/detail/constraint.h" +#include "core/partitioning/detail/constraint_solver.h" +#include "core/partitioning/detail/partitioner.h" #include "core/partitioning/partition.h" -#include "core/partitioning/partitioner.h" namespace legate::detail { Copy::Copy(std::shared_ptr target, std::shared_ptr source, int64_t unique_id, - mapping::MachineDesc&& machine) + mapping::detail::Machine&& machine) : Operation(unique_id, std::move(machine)), target_{target.get(), declare_partition()}, source_{source.get(), declare_partition()}, - constraint_(legate::align(target_.variable, source_.variable)) + constraint_(align(target_.variable, source_.variable)) { record_partition(target_.variable, std::move(target)); record_partition(source_.variable, std::move(source)); diff --git a/src/core/operation/detail/copy.h b/src/core/operation/detail/copy.h index a70dc2016c..e04086cb2d 100644 --- a/src/core/operation/detail/copy.h +++ b/src/core/operation/detail/copy.h @@ -31,7 +31,7 @@ class Copy : public Operation { Copy(std::shared_ptr target, std::shared_ptr source, int64_t unique_id, - mapping::MachineDesc&& machine); + mapping::detail::Machine&& machine); public: void validate() override; diff --git a/src/core/operation/detail/copy_launcher.cc b/src/core/operation/detail/copy_launcher.cc index 3b7e78c289..c9747b89be 100644 --- a/src/core/operation/detail/copy_launcher.cc +++ b/src/core/operation/detail/copy_launcher.cc @@ -1,4 +1,4 @@ -/* Copyright 2021 NVIDIA Corporation +/* Copyright 2023 NVIDIA Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,13 +15,13 @@ */ #include "core/operation/detail/copy_launcher.h" + #include "core/data/detail/logical_store.h" -#include "core/mapping/machine.h" #include "core/operation/detail/launcher_arg.h" #include "core/operation/detail/projection.h" -#include "core/runtime/context.h" +#include "core/runtime/detail/library.h" #include "core/runtime/detail/runtime.h" -#include "core/utilities/buffer_builder.h" +#include "core/utilities/detail/buffer_builder.h" namespace legate::detail { @@ -80,7 +80,7 @@ void CopyArg::pack(BufferBuilder& buffer) const buffer.pack(field_id_); } -CopyLauncher::CopyLauncher(const mapping::MachineDesc& machine, int64_t tag) +CopyLauncher::CopyLauncher(const mapping::detail::Machine& machine, int64_t tag) : machine_(machine), tag_(tag) { mapper_arg_ = new BufferBuilder(); @@ -221,7 +221,7 @@ std::unique_ptr CopyLauncher::build_index_copy( auto index_copy = std::make_unique(launch_domain, Legion::Predicate::TRUE_PRED, - runtime->core_context()->get_mapper_id(), + runtime->core_library()->get_mapper_id(), tag_, mapper_arg_->to_legion_buffer(), provenance.c_str()); @@ -237,7 +237,7 @@ std::unique_ptr CopyLauncher::build_single_copy() auto& provenance = runtime->provenance_manager()->get_provenance(); auto single_copy = std::make_unique(Legion::Predicate::TRUE_PRED, - runtime->core_context()->get_mapper_id(), + runtime->core_library()->get_mapper_id(), tag_, mapper_arg_->to_legion_buffer(), provenance.c_str()); diff --git a/src/core/operation/detail/copy_launcher.h b/src/core/operation/detail/copy_launcher.h index ce9ef85140..c4a298a76b 100644 --- a/src/core/operation/detail/copy_launcher.h +++ b/src/core/operation/detail/copy_launcher.h @@ -19,18 +19,13 @@ #include #include -#include "core/mapping/machine.h" +#include "core/mapping/detail/machine.h" namespace legate { class BufferBuilder; -class LibraryContext; class Scalar; } // namespace legate -namespace legate::mapping { -class MachineDesc; -} // namespace legate::mapping - namespace legate::detail { class LogicalStore; @@ -41,7 +36,7 @@ class RequirementAnalyzer; class CopyLauncher { public: - CopyLauncher(const mapping::MachineDesc& machine, int64_t tag = 0); + CopyLauncher(const mapping::detail::Machine& machine, int64_t tag = 0); ~CopyLauncher(); public: @@ -77,7 +72,7 @@ class CopyLauncher { void populate_copy(Launcher* launcher); private: - mapping::MachineDesc machine_; + mapping::detail::Machine machine_; int64_t tag_; Legion::ProjectionID key_proj_id_{0}; diff --git a/src/core/operation/detail/fill.cc b/src/core/operation/detail/fill.cc index 43cae2156b..0a14326c7e 100644 --- a/src/core/operation/detail/fill.cc +++ b/src/core/operation/detail/fill.cc @@ -19,15 +19,15 @@ #include "core/data/detail/logical_store.h" #include "core/operation/detail/fill_launcher.h" #include "core/operation/detail/projection.h" -#include "core/partitioning/constraint_solver.h" -#include "core/partitioning/partitioner.h" +#include "core/partitioning/detail/constraint_solver.h" +#include "core/partitioning/detail/partitioner.h" namespace legate::detail { Fill::Fill(std::shared_ptr&& lhs, std::shared_ptr&& value, int64_t unique_id, - mapping::MachineDesc&& machine) + mapping::detail::Machine&& machine) : Operation(unique_id, std::move(machine)), lhs_var_(declare_partition()), lhs_(std::move(lhs)), diff --git a/src/core/operation/detail/fill.h b/src/core/operation/detail/fill.h index da6ea51c47..17956ec794 100644 --- a/src/core/operation/detail/fill.h +++ b/src/core/operation/detail/fill.h @@ -26,7 +26,7 @@ class Fill : public Operation { Fill(std::shared_ptr&& lhs, std::shared_ptr&& value, int64_t unique_id, - mapping::MachineDesc&& machine); + mapping::detail::Machine&& machine); public: void validate() override; diff --git a/src/core/operation/detail/fill_launcher.cc b/src/core/operation/detail/fill_launcher.cc index a525cc5f48..e154138c6e 100644 --- a/src/core/operation/detail/fill_launcher.cc +++ b/src/core/operation/detail/fill_launcher.cc @@ -15,16 +15,17 @@ */ #include "core/operation/detail/fill_launcher.h" + #include "core/data/detail/logical_store.h" #include "core/mapping/machine.h" #include "core/operation/detail/projection.h" -#include "core/runtime/context.h" +#include "core/runtime/detail/library.h" #include "core/runtime/detail/runtime.h" -#include "core/utilities/buffer_builder.h" +#include "core/utilities/detail/buffer_builder.h" namespace legate::detail { -FillLauncher::FillLauncher(const mapping::MachineDesc& machine, int64_t tag) +FillLauncher::FillLauncher(const mapping::detail::Machine& machine, int64_t tag) : machine_(machine), tag_(tag), mapper_arg_(new BufferBuilder()) { } @@ -75,7 +76,7 @@ std::unique_ptr FillLauncher::build_index_fill( future_value, lhs_proj.proj_id, Legion::Predicate::TRUE_PRED, - runtime->core_context()->get_mapper_id(), + runtime->core_library()->get_mapper_id(), lhs_proj.tag, mapper_arg_->to_legion_buffer(), provenance.c_str()); @@ -100,7 +101,7 @@ std::unique_ptr FillLauncher::build_single_fill( lhs_parent, future_value, Legion::Predicate::TRUE_PRED, - runtime->core_context()->get_mapper_id(), + runtime->core_library()->get_mapper_id(), lhs_proj.tag, mapper_arg_->to_legion_buffer(), provenance.c_str()); diff --git a/src/core/operation/detail/fill_launcher.h b/src/core/operation/detail/fill_launcher.h index 585ef698d7..c77118970e 100644 --- a/src/core/operation/detail/fill_launcher.h +++ b/src/core/operation/detail/fill_launcher.h @@ -18,17 +18,12 @@ #include -#include "core/mapping/machine.h" +#include "core/mapping/detail/machine.h" namespace legate { class BufferBuilder; -class LibraryContext; } // namespace legate -namespace legate::mapping { -class MachineDesc; -} // namespace legate::mapping - namespace legate::detail { class LogicalStore; @@ -36,7 +31,7 @@ class ProjectionInfo; class FillLauncher { public: - FillLauncher(const mapping::MachineDesc& machine, int64_t tag = 0); + FillLauncher(const mapping::detail::Machine& machine, int64_t tag = 0); ~FillLauncher(); public: @@ -57,7 +52,7 @@ class FillLauncher { LogicalStore* value); private: - mapping::MachineDesc machine_; + mapping::detail::Machine machine_; int64_t tag_; private: diff --git a/src/core/operation/detail/gather.cc b/src/core/operation/detail/gather.cc index 4c306f92e0..8b658859a2 100644 --- a/src/core/operation/detail/gather.cc +++ b/src/core/operation/detail/gather.cc @@ -17,10 +17,10 @@ #include "core/operation/detail/gather.h" #include "core/operation/detail/copy_launcher.h" -#include "core/partitioning/constraint.h" -#include "core/partitioning/constraint_solver.h" +#include "core/partitioning/detail/constraint.h" +#include "core/partitioning/detail/constraint_solver.h" +#include "core/partitioning/detail/partitioner.h" #include "core/partitioning/partition.h" -#include "core/partitioning/partitioner.h" namespace legate::detail { @@ -28,12 +28,12 @@ Gather::Gather(std::shared_ptr target, std::shared_ptr source, std::shared_ptr source_indirect, int64_t unique_id, - mapping::MachineDesc&& machine) + mapping::detail::Machine&& machine) : Operation(unique_id, std::move(machine)), target_{target.get(), declare_partition()}, source_{source.get(), declare_partition()}, source_indirect_{source_indirect.get(), declare_partition()}, - constraint_(legate::align(target_.variable, source_indirect_.variable)) + constraint_(align(target_.variable, source_indirect_.variable)) { record_partition(target_.variable, std::move(target)); record_partition(source_.variable, std::move(source)); diff --git a/src/core/operation/detail/gather.h b/src/core/operation/detail/gather.h index 83930aef7b..c584c92b8f 100644 --- a/src/core/operation/detail/gather.h +++ b/src/core/operation/detail/gather.h @@ -32,7 +32,7 @@ class Gather : public Operation { std::shared_ptr source, std::shared_ptr source_indirect, int64_t unique_id, - mapping::MachineDesc&& machine); + mapping::detail::Machine&& machine); public: void set_indirect_out_of_range(bool flag) { out_of_range_ = flag; } diff --git a/src/core/operation/detail/launcher_arg.cc b/src/core/operation/detail/launcher_arg.cc index c317a2bd70..ad992a2ba6 100644 --- a/src/core/operation/detail/launcher_arg.cc +++ b/src/core/operation/detail/launcher_arg.cc @@ -18,6 +18,8 @@ #include "core/data/detail/logical_region_field.h" #include "core/operation/detail/req_analyzer.h" +#include "core/type/detail/type_info.h" +#include "core/utilities/detail/buffer_builder.h" namespace legate::detail { @@ -82,7 +84,7 @@ void FutureStoreArg::pack(BufferBuilder& buffer) const buffer.pack(redop_); buffer.pack(read_only_); buffer.pack(has_storage_); - buffer.pack(store_->type().size()); + buffer.pack(store_->type()->size()); buffer.pack(store_->get_storage()->extents().data()); } diff --git a/src/core/operation/detail/launcher_arg.h b/src/core/operation/detail/launcher_arg.h index 9a8287a1fa..2ec7121540 100644 --- a/src/core/operation/detail/launcher_arg.h +++ b/src/core/operation/detail/launcher_arg.h @@ -17,11 +17,12 @@ #pragma once #include "core/data/detail/logical_store.h" +#include "core/data/detail/scalar.h" #include "core/data/scalar.h" -#include "core/utilities/buffer_builder.h" namespace legate::detail { +class BufferBuilder; class OutputRequirementAnalyzer; class ProjectionInfo; class RequirementAnalyzer; @@ -48,7 +49,7 @@ struct ScalarArg : public ArgWrapper { struct UntypedScalarArg : public ArgWrapper { public: - UntypedScalarArg(const Scalar& scalar) : scalar_(scalar) {} + UntypedScalarArg(Scalar&& scalar) : scalar_(std::move(scalar)) {} public: ~UntypedScalarArg() {} diff --git a/src/core/operation/detail/operation.cc b/src/core/operation/detail/operation.cc index b68432d4f8..8abc7fcf6f 100644 --- a/src/core/operation/detail/operation.cc +++ b/src/core/operation/detail/operation.cc @@ -16,13 +16,13 @@ #include "core/operation/detail/operation.h" -#include "core/partitioning/constraint.h" -#include "core/partitioning/partitioner.h" +#include "core/partitioning/detail/constraint.h" +#include "core/partitioning/detail/partitioner.h" #include "core/runtime/detail/runtime.h" namespace legate::detail { -Operation::Operation(uint64_t unique_id, mapping::MachineDesc&& machine) +Operation::Operation(uint64_t unique_id, mapping::detail::Machine&& machine) : unique_id_(unique_id), machine_(std::move(machine)), provenance_(Runtime::get_runtime()->provenance_manager()->get_provenance()) diff --git a/src/core/operation/detail/operation.h b/src/core/operation/detail/operation.h index b776ce3786..6257a56261 100644 --- a/src/core/operation/detail/operation.h +++ b/src/core/operation/detail/operation.h @@ -19,17 +19,14 @@ #include #include "core/data/detail/logical_store.h" -#include "core/mapping/machine.h" +#include "core/mapping/detail/machine.h" #include "core/operation/detail/projection.h" -namespace legate { -class Variable; -} // namespace legate - namespace legate::detail { class ConstraintSolver; class LogicalStore; class Strategy; +class Variable; class Operation { protected: @@ -37,7 +34,7 @@ class Operation { LogicalStore* store; const Variable* variable; }; - Operation(uint64_t unique_id, mapping::MachineDesc&& machine); + Operation(uint64_t unique_id, mapping::detail::Machine&& machine); public: virtual ~Operation() {} @@ -54,7 +51,7 @@ class Operation { std::shared_ptr find_store(const Variable* variable) const; public: - const mapping::MachineDesc& machine() const { return machine_; } + const mapping::detail::Machine& machine() const { return machine_; } const std::string& provenance() const { return provenance_; } protected: @@ -80,7 +77,7 @@ class Operation { std::map> store_mappings_{}; std::map, const Variable*> part_mappings_{}; std::string provenance_; - mapping::MachineDesc machine_; + mapping::detail::Machine machine_; }; } // namespace legate::detail diff --git a/src/core/operation/detail/scatter.cc b/src/core/operation/detail/scatter.cc index 99aeb5491d..e2df1c2dc7 100644 --- a/src/core/operation/detail/scatter.cc +++ b/src/core/operation/detail/scatter.cc @@ -17,10 +17,10 @@ #include "core/operation/detail/scatter.h" #include "core/operation/detail/copy_launcher.h" -#include "core/partitioning/constraint.h" -#include "core/partitioning/constraint_solver.h" +#include "core/partitioning/detail/constraint.h" +#include "core/partitioning/detail/constraint_solver.h" +#include "core/partitioning/detail/partitioner.h" #include "core/partitioning/partition.h" -#include "core/partitioning/partitioner.h" namespace legate::detail { @@ -28,12 +28,12 @@ Scatter::Scatter(std::shared_ptr target, std::shared_ptr target_indirect, std::shared_ptr source, int64_t unique_id, - mapping::MachineDesc&& machine) + mapping::detail::Machine&& machine) : Operation(unique_id, std::move(machine)), target_{target.get(), declare_partition()}, target_indirect_{target_indirect.get(), declare_partition()}, source_{source.get(), declare_partition()}, - constraint_(legate::align(source_.variable, target_indirect_.variable)) + constraint_(align(source_.variable, target_indirect_.variable)) { record_partition(target_.variable, std::move(target)); record_partition(target_indirect_.variable, std::move(target_indirect)); diff --git a/src/core/operation/detail/scatter.h b/src/core/operation/detail/scatter.h index 23f94a058e..1880a6fdeb 100644 --- a/src/core/operation/detail/scatter.h +++ b/src/core/operation/detail/scatter.h @@ -32,7 +32,7 @@ class Scatter : public Operation { std::shared_ptr target_indirect, std::shared_ptr source, int64_t unique_id, - mapping::MachineDesc&& machine); + mapping::detail::Machine&& machine); public: void set_indirect_out_of_range(bool flag) { out_of_range_ = flag; } diff --git a/src/core/operation/detail/scatter_gather.cc b/src/core/operation/detail/scatter_gather.cc index 2c3cfdcf1f..41cb1cc203 100644 --- a/src/core/operation/detail/scatter_gather.cc +++ b/src/core/operation/detail/scatter_gather.cc @@ -17,10 +17,10 @@ #include "core/operation/detail/scatter_gather.h" #include "core/operation/detail/copy_launcher.h" -#include "core/partitioning/constraint.h" -#include "core/partitioning/constraint_solver.h" +#include "core/partitioning/detail/constraint.h" +#include "core/partitioning/detail/constraint_solver.h" +#include "core/partitioning/detail/partitioner.h" #include "core/partitioning/partition.h" -#include "core/partitioning/partitioner.h" namespace legate::detail { @@ -29,13 +29,13 @@ ScatterGather::ScatterGather(std::shared_ptr target, std::shared_ptr source, std::shared_ptr source_indirect, int64_t unique_id, - mapping::MachineDesc&& machine) + mapping::detail::Machine&& machine) : Operation(unique_id, std::move(machine)), target_{target.get(), declare_partition()}, target_indirect_{target_indirect.get(), declare_partition()}, source_{source.get(), declare_partition()}, source_indirect_{source_indirect.get(), declare_partition()}, - constraint_(legate::align(source_indirect_.variable, target_indirect_.variable)) + constraint_(align(source_indirect_.variable, target_indirect_.variable)) { record_partition(target_.variable, std::move(target)); record_partition(target_indirect_.variable, std::move(target_indirect)); diff --git a/src/core/operation/detail/scatter_gather.h b/src/core/operation/detail/scatter_gather.h index 0a9c0f5459..b3677573c8 100644 --- a/src/core/operation/detail/scatter_gather.h +++ b/src/core/operation/detail/scatter_gather.h @@ -33,7 +33,7 @@ class ScatterGather : public Operation { std::shared_ptr source, std::shared_ptr source_indirect, int64_t unique_id, - mapping::MachineDesc&& machine); + mapping::detail::Machine&& machine); public: void set_source_indirect_out_of_range(bool flag); diff --git a/src/core/operation/detail/task.cc b/src/core/operation/detail/task.cc index da68a6dbff..36e898f972 100644 --- a/src/core/operation/detail/task.cc +++ b/src/core/operation/detail/task.cc @@ -21,13 +21,14 @@ #include "core/data/scalar.h" #include "core/operation/detail/projection.h" #include "core/operation/detail/task_launcher.h" -#include "core/partitioning/constraint_solver.h" +#include "core/partitioning/detail/constraint_solver.h" +#include "core/partitioning/detail/partitioner.h" #include "core/partitioning/partition.h" -#include "core/partitioning/partitioner.h" -#include "core/runtime/context.h" #include "core/runtime/detail/communicator_manager.h" +#include "core/runtime/detail/library.h" #include "core/runtime/detail/provenance_manager.h" #include "core/runtime/detail/runtime.h" +#include "core/type/detail/type_info.h" namespace legate::detail { @@ -35,17 +36,17 @@ namespace legate::detail { // legate::Task //////////////////////////////////////////////////// -Task::Task(const LibraryContext* library, +Task::Task(const Library* library, int64_t task_id, uint64_t unique_id, - mapping::MachineDesc&& machine) + mapping::detail::Machine&& machine) : Operation(unique_id, std::move(machine)), library_(library), task_id_(task_id) { } void Task::add_scalar_arg(const Scalar& scalar) { scalars_.push_back(scalar); } -void Task::add_scalar_arg(Scalar&& scalar) { scalars_.emplace_back(std::move(scalar)); } +void Task::add_scalar_arg(Scalar&& scalar) { scalars_.push_back(std::move(scalar)); } void Task::set_concurrent(bool concurrent) { concurrent_ = concurrent; } @@ -102,13 +103,13 @@ void Task::launch(Strategy* p_strategy) if (!store->unbound()) continue; auto field_space = strategy.find_field_space(var); // TODO: We should reuse field ids here - auto field_size = store->type().size(); + auto field_size = store->type()->size(); auto field_id = runtime->allocate_field(field_space, field_size); launcher.add_unbound_output(store, field_space, field_id); } // Add by-value scalars - for (auto& scalar : scalars_) launcher.add_scalar(scalar); + for (auto& scalar : scalars_) launcher.add_scalar(std::move(scalar)); // Add communicators if (launch_domain != nullptr) @@ -227,10 +228,10 @@ std::string Task::to_string() const // legate::AutoTask //////////////////////////////////////////////////// -AutoTask::AutoTask(const LibraryContext* library, +AutoTask::AutoTask(const Library* library, int64_t task_id, uint64_t unique_id, - mapping::MachineDesc&& machine) + mapping::detail::Machine&& machine) : Task(library, task_id, unique_id, std::move(machine)) { } @@ -253,7 +254,7 @@ void AutoTask::add_reduction(std::shared_ptr store, int32_t redop, const Variable* partition_symbol) { - auto legion_redop_id = store->type().find_reduction_operator(redop); + auto legion_redop_id = store->type()->find_reduction_operator(redop); if (store->has_scalar_storage()) scalar_reductions_.push_back(reductions_.size()); add_store(reductions_, std::move(store), partition_symbol); reduction_ops_.push_back(legion_redop_id); @@ -291,11 +292,11 @@ void AutoTask::validate() ManualTask::~ManualTask() {} -ManualTask::ManualTask(const LibraryContext* library, +ManualTask::ManualTask(const Library* library, int64_t task_id, const Shape& launch_shape, uint64_t unique_id, - mapping::MachineDesc&& machine) + mapping::detail::Machine&& machine) : Task(library, task_id, unique_id, std::move(machine)), strategy_(std::make_unique()) { @@ -333,7 +334,7 @@ void ManualTask::add_output(std::shared_ptr store_partiti void ManualTask::add_reduction(std::shared_ptr store, int32_t redop) { - auto legion_redop_id = store->type().find_reduction_operator(redop); + auto legion_redop_id = store->type()->find_reduction_operator(redop); if (store->has_scalar_storage()) scalar_reductions_.push_back(reductions_.size()); add_store(reductions_, std::move(store), create_no_partition()); reduction_ops_.push_back(legion_redop_id); @@ -342,7 +343,7 @@ void ManualTask::add_reduction(std::shared_ptr store, int32_t redo void ManualTask::add_reduction(std::shared_ptr store_partition, int32_t redop) { - auto legion_redop_id = store_partition->store()->type().find_reduction_operator(redop); + auto legion_redop_id = store_partition->store()->type()->find_reduction_operator(redop); if (store_partition->store()->has_scalar_storage()) scalar_reductions_.push_back(reductions_.size()); add_store(reductions_, store_partition->store(), store_partition->partition()); diff --git a/src/core/operation/detail/task.h b/src/core/operation/detail/task.h index 89d22db6cf..a10c457a94 100644 --- a/src/core/operation/detail/task.h +++ b/src/core/operation/detail/task.h @@ -18,18 +18,19 @@ #include +#include "core/data/detail/scalar.h" #include "core/operation/detail/operation.h" #include "core/partitioning/constraint.h" namespace legate { class Constraint; -class LibraryContext; class Scalar; } // namespace legate namespace legate::detail { class CommunicatorFactory; class ConstraintSolver; +class Library; class LogicalStore; class LogicalStorePartition; class Strategy; @@ -37,10 +38,10 @@ class Runtime; class Task : public Operation { protected: - Task(const LibraryContext* library, + Task(const Library* library, int64_t task_id, uint64_t unique_id, - mapping::MachineDesc&& machine); + mapping::detail::Machine&& machine); public: virtual ~Task() {} @@ -64,7 +65,7 @@ class Task : public Operation { std::string to_string() const override; protected: - const LibraryContext* library_; + const Library* library_; int64_t task_id_; bool concurrent_{false}; bool has_side_effect_{false}; @@ -79,10 +80,10 @@ class Task : public Operation { class AutoTask : public Task { private: friend class Runtime; - AutoTask(const LibraryContext* library, + AutoTask(const Library* library, int64_t task_id, uint64_t unique_id, - mapping::MachineDesc&& machine); + mapping::detail::Machine&& machine); public: ~AutoTask() {} @@ -113,11 +114,11 @@ class AutoTask : public Task { class ManualTask : public Task { private: friend class Runtime; - ManualTask(const LibraryContext* library, + ManualTask(const Library* library, int64_t task_id, const Shape& launch_shape, uint64_t unique_id, - mapping::MachineDesc&& machine); + mapping::detail::Machine&& machine); public: ~ManualTask(); diff --git a/src/core/operation/detail/task_launcher.cc b/src/core/operation/detail/task_launcher.cc index 41f926baf8..f4970f3d28 100644 --- a/src/core/operation/detail/task_launcher.cc +++ b/src/core/operation/detail/task_launcher.cc @@ -18,19 +18,19 @@ #include "core/data/detail/logical_region_field.h" #include "core/data/detail/logical_store.h" #include "core/data/scalar.h" -#include "core/mapping/machine.h" #include "core/operation/detail/launcher_arg.h" #include "core/operation/detail/req_analyzer.h" -#include "core/runtime/context.h" +#include "core/runtime/detail/library.h" #include "core/runtime/detail/partition_manager.h" #include "core/runtime/detail/runtime.h" -#include "core/runtime/shard.h" -#include "core/utilities/buffer_builder.h" +#include "core/runtime/detail/shard.h" +#include "core/type/detail/type_info.h" +#include "core/utilities/detail/buffer_builder.h" namespace legate::detail { -TaskLauncher::TaskLauncher(const LibraryContext* library, - const mapping::MachineDesc& machine, +TaskLauncher::TaskLauncher(const Library* library, + const mapping::detail::Machine& machine, int64_t task_id, int64_t tag /*= 0*/) : library_(library), task_id_(task_id), tag_(tag), machine_(machine), provenance_("") @@ -38,8 +38,8 @@ TaskLauncher::TaskLauncher(const LibraryContext* library, initialize(); } -TaskLauncher::TaskLauncher(const LibraryContext* library, - const mapping::MachineDesc& machine, +TaskLauncher::TaskLauncher(const Library* library, + const mapping::detail::Machine& machine, const std::string& provenance, int64_t task_id, int64_t tag /*= 0*/) @@ -73,9 +73,9 @@ int64_t TaskLauncher::legion_task_id() const { return library_->get_task_id(task int64_t TaskLauncher::legion_mapper_id() const { return library_->get_mapper_id(); } -void TaskLauncher::add_scalar(const Scalar& scalar) +void TaskLauncher::add_scalar(Scalar&& scalar) { - scalars_.push_back(new UntypedScalarArg(scalar)); + scalars_.push_back(new UntypedScalarArg(std::move(scalar))); } void TaskLauncher::add_input(LogicalStore* store, std::unique_ptr proj_info) @@ -283,7 +283,7 @@ void TaskLauncher::post_process_unbound_stores() auto* store = arg->store(); auto& req = output_requirements_[arg->requirement_index()]; auto region_field = - runtime->import_region_field(req.parent, arg->field_id(), store->type().size()); + runtime->import_region_field(req.parent, arg->field_id(), store->type()->size()); store->set_region_field(std::move(region_field)); store->set_key_partition(machine_, no_part.get()); } @@ -302,7 +302,7 @@ void TaskLauncher::post_process_unbound_stores(const Legion::FutureMap& result, auto*& arg, const auto& req, const auto& weights, const auto& machine) { auto* store = arg->store(); auto region_field = - runtime->import_region_field(req.parent, arg->field_id(), store->type().size()); + runtime->import_region_field(req.parent, arg->field_id(), store->type()->size()); store->set_region_field(std::move(region_field)); // TODO: Need to handle key partitions for multi-dimensional unbound stores diff --git a/src/core/operation/detail/task_launcher.h b/src/core/operation/detail/task_launcher.h index bdcd2ff50e..9bbff0a159 100644 --- a/src/core/operation/detail/task_launcher.h +++ b/src/core/operation/detail/task_launcher.h @@ -20,31 +20,27 @@ #include "legion.h" -#include "core/mapping/machine.h" - -namespace legate { -class BufferBuilder; -class LibraryContext; -class Scalar; -} // namespace legate +#include "core/mapping/detail/machine.h" namespace legate::detail { class ArgWrapper; +class Library; class LogicalStore; class ProjectionInfo; class OutputRegionArg; class OutputRequirementAnalyzer; class RequirementAnalyzer; +class Scalar; class TaskLauncher { public: - TaskLauncher(const LibraryContext* library, - const mapping::MachineDesc& machine, + TaskLauncher(const Library* library, + const mapping::detail::Machine& machine, int64_t task_id, int64_t tag = 0); - TaskLauncher(const LibraryContext* library, - const mapping::MachineDesc& machine, + TaskLauncher(const Library* library, + const mapping::detail::Machine& machine, const std::string& provenance, int64_t task_id, int64_t tag = 0); @@ -58,7 +54,7 @@ class TaskLauncher { int64_t legion_mapper_id() const; public: - void add_scalar(const Scalar& scalar); + void add_scalar(Scalar&& scalar); void add_input(LogicalStore* store, std::unique_ptr proj_info); void add_output(LogicalStore* store, std::unique_ptr proj_info); void add_reduction(LogicalStore* store, @@ -100,10 +96,10 @@ class TaskLauncher { const Legion::Domain& launch_domain); private: - const LibraryContext* library_; + const Library* library_; int64_t task_id_; int64_t tag_; - mapping::MachineDesc machine_; + mapping::detail::Machine machine_; std::string provenance_; Legion::ProjectionID key_proj_id_{0}; diff --git a/src/core/operation/task.cc b/src/core/operation/task.cc index b3a5931f21..56c028be2f 100644 --- a/src/core/operation/task.cc +++ b/src/core/operation/task.cc @@ -16,7 +16,9 @@ #include "core/operation/task.h" +#include "core/data/detail/scalar.h" #include "core/operation/detail/task.h" +#include "core/partitioning/detail/constraint.h" namespace legate { @@ -24,45 +26,41 @@ namespace legate { // legate::AutoTask //////////////////////////////////////////////////// -void AutoTask::add_input(LogicalStore store, const Variable* partition_symbol) +void AutoTask::add_input(LogicalStore store, Variable partition_symbol) { - impl_->add_input(store.impl(), partition_symbol); + impl_->add_input(store.impl(), partition_symbol.impl()); } -void AutoTask::add_output(LogicalStore store, const Variable* partition_symbol) +void AutoTask::add_output(LogicalStore store, Variable partition_symbol) { - impl_->add_output(store.impl(), partition_symbol); + impl_->add_output(store.impl(), partition_symbol.impl()); } -void AutoTask::add_reduction(LogicalStore store, - ReductionOpKind redop, - const Variable* partition_symbol) +void AutoTask::add_reduction(LogicalStore store, ReductionOpKind redop, Variable partition_symbol) { - impl_->add_reduction(store.impl(), static_cast(redop), partition_symbol); + impl_->add_reduction(store.impl(), static_cast(redop), partition_symbol.impl()); } -void AutoTask::add_reduction(LogicalStore store, int32_t redop, const Variable* partition_symbol) +void AutoTask::add_reduction(LogicalStore store, int32_t redop, Variable partition_symbol) { - impl_->add_reduction(store.impl(), redop, partition_symbol); + impl_->add_reduction(store.impl(), redop, partition_symbol.impl()); } -void AutoTask::add_scalar_arg(const Scalar& scalar) { impl_->add_scalar_arg(scalar); } +void AutoTask::add_scalar_arg(const Scalar& scalar) { impl_->add_scalar_arg(*scalar.impl_); } -void AutoTask::add_scalar_arg(Scalar&& scalar) { impl_->add_scalar_arg(scalar); } +void AutoTask::add_scalar_arg(Scalar&& scalar) { impl_->add_scalar_arg(std::move(*scalar.impl_)); } -void AutoTask::add_constraint(std::unique_ptr constraint) +void AutoTask::add_constraint(Constraint&& constraint) { - impl_->add_constraint(std::move(constraint)); + impl_->add_constraint(std::unique_ptr(constraint.release())); } -const Variable* AutoTask::find_or_declare_partition(LogicalStore store) +Variable AutoTask::find_or_declare_partition(LogicalStore store) { - return impl_->find_or_declare_partition(store.impl()); + return Variable(impl_->find_or_declare_partition(store.impl())); } -const Variable* AutoTask::declare_partition() { return impl_->declare_partition(); } - -const mapping::MachineDesc& AutoTask::machine() const { return impl_->machine(); } +Variable AutoTask::declare_partition() { return Variable(impl_->declare_partition()); } const std::string& AutoTask::provenance() const { return impl_->provenance(); } @@ -123,11 +121,12 @@ void ManualTask::add_reduction(LogicalStorePartition store_partition, int32_t re impl_->add_reduction(store_partition.impl(), redop); } -void ManualTask::add_scalar_arg(const Scalar& scalar) { impl_->add_scalar_arg(scalar); } - -void ManualTask::add_scalar_arg(Scalar&& scalar) { impl_->add_scalar_arg(scalar); } +void ManualTask::add_scalar_arg(const Scalar& scalar) { impl_->add_scalar_arg(*scalar.impl_); } -const mapping::MachineDesc& ManualTask::machine() const { return impl_->machine(); } +void ManualTask::add_scalar_arg(Scalar&& scalar) +{ + impl_->add_scalar_arg(std::move(*scalar.impl_)); +} const std::string& ManualTask::provenance() const { return impl_->provenance(); } diff --git a/src/core/operation/task.h b/src/core/operation/task.h index fde6df8a8c..d5527aca35 100644 --- a/src/core/operation/task.h +++ b/src/core/operation/task.h @@ -19,7 +19,9 @@ #include #include "core/data/logical_store.h" +#include "core/data/scalar.h" #include "core/mapping/machine.h" +#include "core/partitioning/constraint.h" /** * @file @@ -33,10 +35,6 @@ class ManualTask; namespace legate { -class Constraint; -class Scalar; -class Variable; - /** * @ingroup op * @brief A class for auto-parallelized task desciptors @@ -52,7 +50,7 @@ class AutoTask { * @param store A store to add to the task as input * @param partition_symbol A partition symbol for the store */ - void add_input(LogicalStore store, const Variable* partition_symbol); + void add_input(LogicalStore store, Variable partition_symbol); /** * @brief Adds a store to the task as output * @@ -62,7 +60,7 @@ class AutoTask { * @param store A store to add to the task as output * @param partition_symbol A partition symbol for the store */ - void add_output(LogicalStore store, const Variable* partition_symbol); + void add_output(LogicalStore store, Variable partition_symbol); /** * @brief Adds a store to the task for reductions * @@ -73,7 +71,7 @@ class AutoTask { * @param redop ID of the reduction operator to use. The store's type must support the operator. * @param partition_symbol A partition symbol for the store */ - void add_reduction(LogicalStore store, ReductionOpKind redop, const Variable* partition_symbol); + void add_reduction(LogicalStore store, ReductionOpKind redop, Variable partition_symbol); /** * @brief Adds a store to the task for reductions * @@ -84,7 +82,7 @@ class AutoTask { * @param redop ID of the reduction operator to use. The store's type must support the operator. * @param partition_symbol A partition symbol for the store */ - void add_reduction(LogicalStore store, int32_t redop, const Variable* partition_symbol); + void add_reduction(LogicalStore store, int32_t redop, Variable partition_symbol); /** * @brief Adds a by-value scalar argument to the task * @@ -102,7 +100,7 @@ class AutoTask { * * @param constraint A partitioning constraint */ - void add_constraint(std::unique_ptr constraint); + void add_constraint(Constraint&& constraint); public: /** @@ -112,19 +110,13 @@ class AutoTask { * * @return The existing symbol if there is one for the store, a fresh symbol otherwise */ - const Variable* find_or_declare_partition(LogicalStore store); + Variable find_or_declare_partition(LogicalStore store); /** * @brief Declares partition symbol * * @return A new symbol that can be used when passing a store to an operation */ - const Variable* declare_partition(); - /** - * @brief Returns the machine of the scope in which this operation is issued - * - * @return The machine of the scope - */ - const mapping::MachineDesc& machine() const; + Variable declare_partition(); /** * @brief Returns the provenance information of this operation * @@ -261,12 +253,6 @@ class ManualTask { void add_scalar_arg(Scalar&& scalar); public: - /** - * @brief Returns the machine of the scope in which this operation is issued - * - * @return The machine of the scope - */ - const mapping::MachineDesc& machine() const; /** * @brief Returns the provenance information of this operation * diff --git a/src/core/partitioning/constraint.cc b/src/core/partitioning/constraint.cc index 53a9b899a8..2fc94ff6c2 100644 --- a/src/core/partitioning/constraint.cc +++ b/src/core/partitioning/constraint.cc @@ -1,4 +1,4 @@ -/* Copyright 2021 NVIDIA Corporation +/* Copyright 2023 NVIDIA Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,158 +14,56 @@ * */ -#include - -#include "core/data/scalar.h" -#include "core/operation/detail/operation.h" #include "core/partitioning/constraint.h" -#include "core/partitioning/partition.h" -#include "core/partitioning/partitioner.h" +#include "core/partitioning/detail/constraint.h" namespace legate { -Literal::Literal(const std::shared_ptr& partition) : partition_(partition) {} - -std::string Literal::to_string() const { return partition_->to_string(); } - -void Literal::find_partition_symbols(std::vector& partition_symbols) const {} +std::string Variable::to_string() const { return impl_->to_string(); } -Variable::Variable(const detail::Operation* op, int32_t id) : op_(op), id_(id) {} - -bool operator==(const Variable& lhs, const Variable& rhs) -{ - return lhs.op_ == rhs.op_ && lhs.id_ == rhs.id_; -} +Variable::Variable(const detail::Variable* impl) : impl_(impl) {} -bool operator<(const Variable& lhs, const Variable& rhs) { return lhs.id_ < rhs.id_; } - -void Variable::find_partition_symbols(std::vector& partition_symbols) const -{ - partition_symbols.push_back(this); -} - -std::string Variable::to_string() const -{ - std::stringstream ss; - ss << "X" << id_ << "{" << op_->to_string() << "}"; - return std::move(ss).str(); -} +Variable::~Variable() {} -Alignment::Alignment(std::unique_ptr&& lhs, std::unique_ptr&& rhs) - : lhs_(std::move(lhs)), rhs_(std::move(rhs)) -{ -} +Variable::Variable(const Variable&) = default; -void Alignment::find_partition_symbols(std::vector& partition_symbols) const -{ - lhs_->find_partition_symbols(partition_symbols); - rhs_->find_partition_symbols(partition_symbols); -} +Variable& Variable::operator=(const Variable&) = default; -void Alignment::validate() const -{ - if (*lhs_ == *rhs_) return; - auto lhs_store = lhs_->operation()->find_store(lhs_.get()); - auto rhs_store = rhs_->operation()->find_store(rhs_.get()); - if (lhs_store->extents() != rhs_store->extents()) - throw std::invalid_argument("Alignment requires the stores to have the same shape, but found " + - lhs_store->extents().to_string() + " and " + - rhs_store->extents().to_string()); -} +std::string Constraint::to_string() const { return impl_->to_string(); } -std::string Alignment::to_string() const -{ - std::stringstream ss; - ss << "Align(" << lhs_->to_string() << ", " << rhs_->to_string() << ")"; - return std::move(ss).str(); -} +Constraint::Constraint(detail::Constraint* impl) : impl_(impl) {} -Broadcast::Broadcast(std::unique_ptr variable, tuple&& axes) - : variable_(std::move(variable)), axes_(std::move(axes)) -{ -} +Constraint::~Constraint() { delete impl_; } -void Broadcast::find_partition_symbols(std::vector& partition_symbols) const -{ - partition_symbols.push_back(variable_.get()); -} - -void Broadcast::validate() const -{ - auto store = variable_->operation()->find_store(variable_.get()); - for (auto axis : axes_.data()) { - if (axis < 0 || axis >= store->dim()) - throw std::invalid_argument("Invalid broadcasting dimension " + std::to_string(axis) + - " for a " + std::to_string(store->dim()) + "-D store"); - } -} - -std::string Broadcast::to_string() const -{ - std::stringstream ss; - ss << "Broadcast(" << variable_->to_string() << ", " << axes_.to_string() << ")"; - return std::move(ss).str(); -} - -ImageConstraint::ImageConstraint(std::unique_ptr var_function, - std::unique_ptr var_range) - : var_function_(std::move(var_function)), var_range_(std::move(var_range)) -{ -} - -void ImageConstraint::find_partition_symbols(std::vector& partition_symbols) const -{ - partition_symbols.push_back(var_function_.get()); - partition_symbols.push_back(var_range_.get()); -} - -void ImageConstraint::validate() const -{ - auto func = var_function_->operation()->find_store(var_function_.get()); - auto range = var_range_->operation()->find_store(var_range_.get()); - - if (!(is_point_type(func->type(), range->dim()) || is_rect_type(func->type(), range->dim()))) - throw std::invalid_argument("Store from which the image partition is derived should have " + - std::to_string(range->dim()) + "-D points or rects"); -} - -std::string ImageConstraint::to_string() const -{ - std::stringstream ss; - ss << "ImageConstraint(" << var_function_->to_string() << ", " << var_range_->to_string() << ")"; - return std::move(ss).str(); -} +Constraint::Constraint(Constraint&& other) : impl_(other.impl_) { other.impl_ = nullptr; } -std::unique_ptr ImageConstraint::resolve(const detail::Strategy& strategy) const +Constraint& Constraint::operator=(Constraint&& other) { - const auto* src = var_function(); - auto src_part = strategy[src]; - if (src_part->has_launch_domain()) - return create_image(src->operation()->find_store(src), src_part); - else - return create_no_partition(); + impl_ = other.impl_; + other.impl_ = nullptr; + return *this; } -std::unique_ptr align(const Variable* lhs, const Variable* rhs) +detail::Constraint* Constraint::release() { - return std::make_unique(std::make_unique(*lhs), - std::make_unique(*rhs)); + auto result = impl_; + impl_ = nullptr; + return result; } -std::unique_ptr broadcast(const Variable* variable, const tuple& axes) +Constraint align(Variable lhs, Variable rhs) { - return broadcast(variable, tuple(axes)); + return Constraint(detail::align(lhs.impl(), rhs.impl()).release()); } -std::unique_ptr broadcast(const Variable* variable, tuple&& axes) +Constraint broadcast(Variable variable, const tuple& axes) { - return std::make_unique(std::make_unique(*variable), std::move(axes)); + return Constraint(detail::broadcast(variable.impl(), tuple(axes)).release()); } -std::unique_ptr image(const Variable* var_function, const Variable* var_range) +Constraint image(Variable var_function, Variable var_range) { - return std::make_unique(std::make_unique(*var_function), - std::make_unique(*var_range)); + return Constraint(detail::image(var_function.impl(), var_range.impl()).release()); } } // namespace legate diff --git a/src/core/partitioning/constraint.h b/src/core/partitioning/constraint.h index b5b0df3948..6b6000c900 100644 --- a/src/core/partitioning/constraint.h +++ b/src/core/partitioning/constraint.h @@ -1,4 +1,4 @@ -/* Copyright 2021 NVIDIA Corporation +/* Copyright 2023 NVIDIA Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,9 +16,7 @@ #pragma once -#include #include -#include #include "core/utilities/tuple.h" @@ -29,302 +27,68 @@ * @file * @brief Class definitions for partitioning constraint language */ - -namespace legate::detail { -class Operation; -class Strategy; -} // namespace legate::detail - namespace legate { -class Alignment; -class Broadcast; +class AutoTask; + +namespace detail { class Constraint; -class ImageConstraint; -class Literal; -class Partition; class Variable; - -/** - * @ingroup partitioning - * @brief A base class for expressions - */ -struct Expr { - enum class Kind : int32_t { - LITERAL = 0, - VARIABLE = 1, - }; - - virtual ~Expr() {} - - virtual void find_partition_symbols(std::vector& partition_symbols) const = 0; - - /** - * @brief Indicates whether the expression is 'closed', i.e., free of any variables - * - * @return true Expression is closed - * @return false Expression is not closed - */ - virtual bool closed() const = 0; - /** - * @brief Converts the expressions to a human-readable string - * - * @return Expression in a string - */ - virtual std::string to_string() const = 0; - - /** - * @brief Returns the expression kind - * - * @return Expression kind - */ - virtual Kind kind() const = 0; - virtual const Literal* as_literal() const = 0; - virtual const Variable* as_variable() const = 0; -}; - -/** - * @ingroup partitioning - * @brief A class for literals - */ -class Literal : public Expr { - public: - Literal(const std::shared_ptr& partition); - - public: - Literal(const Literal&) = default; - Literal& operator=(const Literal&) = default; - - public: - void find_partition_symbols(std::vector& partition_symbols) const override; - - public: - bool closed() const override { return true; } - std::string to_string() const override; - - public: - Kind kind() const override { return Kind::LITERAL; } - const Literal* as_literal() const override { return this; } - const Variable* as_variable() const override { return nullptr; } - - public: - std::shared_ptr partition() const { return partition_; } - - private: - std::shared_ptr partition_; -}; +} // namespace detail /** * @ingroup partitioning * @brief Class for partition symbols */ -class Variable : public Expr { - public: - Variable(const detail::Operation* op, int32_t id); - - public: - Variable(const Variable&) = default; - Variable& operator=(const Variable&) = default; - +class Variable { public: - friend bool operator==(const Variable& lhs, const Variable& rhs); - friend bool operator<(const Variable& lhs, const Variable& rhs); + std::string to_string() const; public: - void find_partition_symbols(std::vector& partition_symbols) const override; + Variable(const detail::Variable* impl); + ~Variable(); public: - bool closed() const override { return false; } - std::string to_string() const override; + Variable(const Variable&); + Variable& operator=(const Variable&); - public: - Kind kind() const override { return Kind::VARIABLE; } - const Literal* as_literal() const override { return nullptr; } - const Variable* as_variable() const override { return this; } + private: + Variable(Variable&&) = delete; + Variable& operator=(Variable&&) = delete; public: - const detail::Operation* operation() const { return op_; } + const detail::Variable* impl() const { return impl_; } private: - const detail::Operation* op_; - int32_t id_; + const detail::Variable* impl_; }; /** * @ingroup partitioning * @brief A base class for partitioning constraints */ -struct Constraint { - enum class Kind : int32_t { - ALIGNMENT = 0, - BROADCAST = 1, - IMAGE = 2, - }; - - virtual ~Constraint() {} - - virtual void find_partition_symbols(std::vector& partition_symbols) const = 0; - - /** - * @brief Converts the constraint to a human-readable string - * - * @return Constraint in a string - */ - virtual std::string to_string() const = 0; - - /** - * @brief Returns the constraint kind - * - * @return Constraint kind - */ - virtual Kind kind() const = 0; - - virtual void validate() const = 0; - - virtual const Alignment* as_alignment() const = 0; - virtual const Broadcast* as_broadcast() const = 0; - virtual const ImageConstraint* as_image_constraint() const = 0; -}; - -/** - * @ingroup partitioning - * @brief A class for alignment constraints - * - * An alignment constraint on stores indicates that the stores should be partitioned in the same - * way. If the stores referred to by an alignment constraint have different shapes, an - * `std::invalid_argument` exception will be raised when the partitioner solves the constraint. - */ -class Alignment : public Constraint { - public: - Alignment(std::unique_ptr&& lhs, std::unique_ptr&& rhs); - - public: - Kind kind() const override { return Kind::ALIGNMENT; } - - public: - void find_partition_symbols(std::vector& partition_symbols) const override; - +class Constraint { public: - void validate() const; + std::string to_string() const; public: - std::string to_string() const override; + Constraint(detail::Constraint* impl); + ~Constraint(); public: - const Alignment* as_alignment() const override { return this; } - const Broadcast* as_broadcast() const override { return nullptr; } - const ImageConstraint* as_image_constraint() const override { return nullptr; } - - public: - /** - * @brief Returns the LHS of the alignment constraint - * - * @return Variable - */ - const Variable* lhs() const { return lhs_.get(); } - /** - * @brief Returns the RHS of the alignment constraint - * - * @return Variable - */ - const Variable* rhs() const { return rhs_.get(); } - - public: - bool is_trivial() const { return *lhs_ == *rhs_; } + Constraint(Constraint&&); + Constraint& operator=(Constraint&&); private: - std::unique_ptr lhs_; - std::unique_ptr rhs_; -}; - -/** - * @ingroup partitioning - * @brief A class for broadcast constraints - * - * A broadcast constraint on a store indicates that some or all dimensions of the store should not - * be partitioned. The dimensions to broadcast must be specified by the constraint as well. If any - * of the dimension names is invalid, an `std::invalid_argument` exception will be raised when the - * auto-partitioner solves the constraint. - */ -class Broadcast : public Constraint { - public: - Broadcast(std::unique_ptr variable, tuple&& axes); - - public: - Kind kind() const override { return Kind::BROADCAST; } - - public: - void find_partition_symbols(std::vector& partition_symbols) const override; - - public: - void validate() const; - - public: - std::string to_string() const override; - - public: - const Alignment* as_alignment() const override { return nullptr; } - const Broadcast* as_broadcast() const override { return this; } - const ImageConstraint* as_image_constraint() const override { return nullptr; } - - public: - /** - * @brief Returns the partition symbol to which this constraint is applied - * - * @return Partition symbol - */ - const Variable* variable() const { return variable_.get(); } - /** - * @brief Returns the list of axes to broadcast - * - * @return Tuple of integers - */ - const tuple& axes() const { return axes_; } + Constraint(const Constraint&) = delete; + Constraint& operator=(const Constraint&) = delete; private: - std::unique_ptr variable_; - tuple axes_; -}; - -/** - * @ingroup partitioning - * @brief A class for image constraints - * - * This constraint tells the partitioner that `var_range_` should be derived by collecting the image - * of a "function", a store that contains either points or rects. `var_function_` is the partition - * of the function that the partitioner should use in the image partitioning. - */ -class ImageConstraint : public Constraint { - public: - ImageConstraint(std::unique_ptr var_function, std::unique_ptr var_range); - - public: - Kind kind() const override { return Kind::IMAGE; } - - public: - void find_partition_symbols(std::vector& partition_symbols) const override; - - public: - void validate() const; - - public: - std::string to_string() const override; - - public: - const Alignment* as_alignment() const override { return nullptr; } - const Broadcast* as_broadcast() const override { return nullptr; } - const ImageConstraint* as_image_constraint() const override { return this; } - - public: - const Variable* var_function() const { return var_function_.get(); } - const Variable* var_range() const { return var_range_.get(); } - - public: - std::unique_ptr resolve(const detail::Strategy& strategy) const; + friend class AutoTask; + detail::Constraint* release(); private: - std::unique_ptr var_function_; - std::unique_ptr var_range_; + detail::Constraint* impl_{nullptr}; }; /** @@ -336,18 +100,7 @@ class ImageConstraint : public Constraint { * * @return Alignment constraint */ -std::unique_ptr align(const Variable* lhs, const Variable* rhs); - -/** - * @ingroup partitioning - * @brief Creates a broadcast constraint on a variable - * - * @param variable Partition symbol to constrain - * @param axes List of dimensions to broadcast - * - * @return Broadcast constraint - */ -std::unique_ptr broadcast(const Variable* variable, const tuple& axes); +Constraint align(Variable lhs, Variable rhs); /** * @ingroup partitioning @@ -358,7 +111,7 @@ std::unique_ptr broadcast(const Variable* variable, const tuple broadcast(const Variable* variable, tuple&& axes); +Constraint broadcast(Variable variable, const tuple& axes); /** * @ingroup partitioning @@ -369,6 +122,6 @@ std::unique_ptr broadcast(const Variable* variable, tuple&& * * @return Broadcast constraint */ -std::unique_ptr image(const Variable* var_function, const Variable* var_range); +Constraint image(Variable var_function, Variable var_range); } // namespace legate diff --git a/src/core/partitioning/detail/constraint.cc b/src/core/partitioning/detail/constraint.cc new file mode 100644 index 0000000000..e24f3a0287 --- /dev/null +++ b/src/core/partitioning/detail/constraint.cc @@ -0,0 +1,162 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/partitioning/detail/constraint.h" + +#include + +#include "core/data/scalar.h" +#include "core/operation/detail/operation.h" +#include "core/partitioning/detail/partitioner.h" +#include "core/partitioning/partition.h" + +namespace legate::detail { + +Literal::Literal(const std::shared_ptr& partition) : partition_(partition) {} + +std::string Literal::to_string() const { return partition_->to_string(); } + +void Literal::find_partition_symbols(std::vector& partition_symbols) const {} + +Variable::Variable(const detail::Operation* op, int32_t id) : op_(op), id_(id) {} + +bool operator==(const Variable& lhs, const Variable& rhs) +{ + return lhs.op_ == rhs.op_ && lhs.id_ == rhs.id_; +} + +bool operator<(const Variable& lhs, const Variable& rhs) { return lhs.id_ < rhs.id_; } + +void Variable::find_partition_symbols(std::vector& partition_symbols) const +{ + partition_symbols.push_back(this); +} + +std::string Variable::to_string() const +{ + std::stringstream ss; + ss << "X" << id_ << "{" << op_->to_string() << "}"; + return std::move(ss).str(); +} + +Alignment::Alignment(const Variable* lhs, const Variable* rhs) : lhs_(lhs), rhs_(rhs) {} + +void Alignment::find_partition_symbols(std::vector& partition_symbols) const +{ + partition_symbols.push_back(lhs_); + partition_symbols.push_back(rhs_); +} + +void Alignment::validate() const +{ + if (*lhs_ == *rhs_) return; + auto lhs_store = lhs_->operation()->find_store(lhs_); + auto rhs_store = rhs_->operation()->find_store(rhs_); + if (lhs_store->extents() != rhs_store->extents()) + throw std::invalid_argument("Alignment requires the stores to have the same shape, but found " + + lhs_store->extents().to_string() + " and " + + rhs_store->extents().to_string()); +} + +std::string Alignment::to_string() const +{ + std::stringstream ss; + ss << "Align(" << lhs_->to_string() << ", " << rhs_->to_string() << ")"; + return std::move(ss).str(); +} + +Broadcast::Broadcast(const Variable* variable, const tuple& axes) + : variable_(variable), axes_(axes) +{ +} + +void Broadcast::find_partition_symbols(std::vector& partition_symbols) const +{ + partition_symbols.push_back(variable_); +} + +void Broadcast::validate() const +{ + auto store = variable_->operation()->find_store(variable_); + for (auto axis : axes_.data()) { + if (axis < 0 || axis >= store->dim()) + throw std::invalid_argument("Invalid broadcasting dimension " + std::to_string(axis) + + " for a " + std::to_string(store->dim()) + "-D store"); + } +} + +std::string Broadcast::to_string() const +{ + std::stringstream ss; + ss << "Broadcast(" << variable_->to_string() << ", " << axes_.to_string() << ")"; + return std::move(ss).str(); +} + +ImageConstraint::ImageConstraint(const Variable* var_function, const Variable* var_range) + : var_function_(var_function), var_range_(var_range) +{ +} + +void ImageConstraint::find_partition_symbols(std::vector& partition_symbols) const +{ + partition_symbols.push_back(var_function_); + partition_symbols.push_back(var_range_); +} + +void ImageConstraint::validate() const +{ + auto func = var_function_->operation()->find_store(var_function_); + auto range = var_range_->operation()->find_store(var_range_); + + if (!(is_point_type(func->type(), range->dim()) || is_rect_type(func->type(), range->dim()))) + throw std::invalid_argument("Store from which the image partition is derived should have " + + std::to_string(range->dim()) + "-D points or rects"); +} + +std::string ImageConstraint::to_string() const +{ + std::stringstream ss; + ss << "ImageConstraint(" << var_function_->to_string() << ", " << var_range_->to_string() << ")"; + return std::move(ss).str(); +} + +std::unique_ptr ImageConstraint::resolve(const detail::Strategy& strategy) const +{ + const auto* src = var_function(); + auto src_part = strategy[src]; + if (src_part->has_launch_domain()) + return create_image(src->operation()->find_store(src), src_part); + else + return create_no_partition(); +} + +std::unique_ptr align(const Variable* lhs, const Variable* rhs) +{ + return std::make_unique(lhs, rhs); +} + +std::unique_ptr broadcast(const Variable* variable, const tuple& axes) + +{ + return std::make_unique(variable, axes); +} + +std::unique_ptr image(const Variable* var_function, const Variable* var_range) +{ + return std::make_unique(var_function, var_range); +} + +} // namespace legate::detail diff --git a/src/core/partitioning/detail/constraint.h b/src/core/partitioning/detail/constraint.h new file mode 100644 index 0000000000..2c35a86dde --- /dev/null +++ b/src/core/partitioning/detail/constraint.h @@ -0,0 +1,231 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include +#include +#include + +#include "core/utilities/tuple.h" + +namespace legate { +class Partition; +} // namespace legate + +namespace legate::detail { +class Operation; +class Strategy; + +class Alignment; +class Broadcast; +class Constraint; +class ImageConstraint; +class Literal; +class Variable; + +struct Expr { + enum class Kind : int32_t { + LITERAL = 0, + VARIABLE = 1, + }; + virtual ~Expr() {} + virtual void find_partition_symbols(std::vector& partition_symbols) const = 0; + virtual bool closed() const = 0; + virtual std::string to_string() const = 0; + virtual Kind kind() const = 0; + virtual const Literal* as_literal() const = 0; + virtual const Variable* as_variable() const = 0; +}; + +class Literal : public Expr { + public: + Literal(const std::shared_ptr& partition); + + public: + Literal(const Literal&) = default; + Literal& operator=(const Literal&) = default; + + public: + void find_partition_symbols(std::vector& partition_symbols) const override; + + public: + bool closed() const override { return true; } + std::string to_string() const override; + + public: + Kind kind() const override { return Kind::LITERAL; } + const Literal* as_literal() const override { return this; } + const Variable* as_variable() const override { return nullptr; } + + public: + std::shared_ptr partition() const { return partition_; } + + private: + std::shared_ptr partition_; +}; + +class Variable : public Expr { + public: + Variable(const Operation* op, int32_t id); + + public: + Variable(const Variable&) = default; + Variable& operator=(const Variable&) = default; + + public: + friend bool operator==(const Variable& lhs, const Variable& rhs); + friend bool operator<(const Variable& lhs, const Variable& rhs); + + public: + void find_partition_symbols(std::vector& partition_symbols) const override; + + public: + bool closed() const override { return false; } + std::string to_string() const override; + + public: + Kind kind() const override { return Kind::VARIABLE; } + const Literal* as_literal() const override { return nullptr; } + const Variable* as_variable() const override { return this; } + + public: + const Operation* operation() const { return op_; } + + private: + const Operation* op_; + int32_t id_; +}; + +struct Constraint { + enum class Kind : int32_t { + ALIGNMENT = 0, + BROADCAST = 1, + IMAGE = 2, + }; + virtual ~Constraint() {} + virtual void find_partition_symbols(std::vector& partition_symbols) const = 0; + virtual std::string to_string() const = 0; + virtual Kind kind() const = 0; + virtual void validate() const = 0; + virtual const Alignment* as_alignment() const = 0; + virtual const Broadcast* as_broadcast() const = 0; + virtual const ImageConstraint* as_image_constraint() const = 0; +}; + +class Alignment : public Constraint { + public: + Alignment(const Variable* lhs, const Variable* rhs); + + public: + Kind kind() const override { return Kind::ALIGNMENT; } + + public: + void find_partition_symbols(std::vector& partition_symbols) const override; + + public: + void validate() const; + + public: + std::string to_string() const override; + + public: + const Alignment* as_alignment() const override { return this; } + const Broadcast* as_broadcast() const override { return nullptr; } + const ImageConstraint* as_image_constraint() const override { return nullptr; } + + public: + const Variable* lhs() const { return lhs_; } + const Variable* rhs() const { return rhs_; } + + public: + bool is_trivial() const { return *lhs_ == *rhs_; } + + private: + const Variable* lhs_; + const Variable* rhs_; +}; + +class Broadcast : public Constraint { + public: + Broadcast(const Variable* variable, const tuple& axes); + + public: + Kind kind() const override { return Kind::BROADCAST; } + + public: + void find_partition_symbols(std::vector& partition_symbols) const override; + + public: + void validate() const; + + public: + std::string to_string() const override; + + public: + const Alignment* as_alignment() const override { return nullptr; } + const Broadcast* as_broadcast() const override { return this; } + const ImageConstraint* as_image_constraint() const override { return nullptr; } + + public: + const Variable* variable() const { return variable_; } + const tuple& axes() const { return axes_; } + + private: + const Variable* variable_; + tuple axes_; +}; + +class ImageConstraint : public Constraint { + public: + ImageConstraint(const Variable* var_function, const Variable* var_range); + + public: + Kind kind() const override { return Kind::IMAGE; } + + public: + void find_partition_symbols(std::vector& partition_symbols) const override; + + public: + void validate() const; + + public: + std::string to_string() const override; + + public: + const Alignment* as_alignment() const override { return nullptr; } + const Broadcast* as_broadcast() const override { return nullptr; } + const ImageConstraint* as_image_constraint() const override { return this; } + + public: + const Variable* var_function() const { return var_function_; } + const Variable* var_range() const { return var_range_; } + + public: + std::unique_ptr resolve(const Strategy& strategy) const; + + private: + const Variable* var_function_; + const Variable* var_range_; +}; + +std::unique_ptr align(const Variable* lhs, const Variable* rhs); + +std::unique_ptr broadcast(const Variable* variable, const tuple& axes); + +std::unique_ptr image(const Variable* var_function, const Variable* var_range); + +} // namespace legate::detail diff --git a/src/core/partitioning/constraint_solver.cc b/src/core/partitioning/detail/constraint_solver.cc similarity index 97% rename from src/core/partitioning/constraint_solver.cc rename to src/core/partitioning/detail/constraint_solver.cc index 8ce81d33b9..149c55db0a 100644 --- a/src/core/partitioning/constraint_solver.cc +++ b/src/core/partitioning/detail/constraint_solver.cc @@ -1,4 +1,4 @@ -/* Copyright 2021 NVIDIA Corporation +/* Copyright 2023 NVIDIA Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,15 +14,15 @@ * */ +#include "core/partitioning/detail/constraint_solver.h" + #include #include "legion.h" #include "core/data/detail/logical_store.h" #include "core/operation/detail/operation.h" -#include "core/partitioning/constraint.h" -#include "core/partitioning/constraint_solver.h" -#include "core/partitioning/partitioner.h" +#include "core/partitioning/detail/partitioner.h" namespace legate { extern Legion::Logger log_legate; diff --git a/src/core/partitioning/constraint_solver.h b/src/core/partitioning/detail/constraint_solver.h similarity index 97% rename from src/core/partitioning/constraint_solver.h rename to src/core/partitioning/detail/constraint_solver.h index 95175bb313..67109fdd5e 100644 --- a/src/core/partitioning/constraint_solver.h +++ b/src/core/partitioning/detail/constraint_solver.h @@ -20,7 +20,7 @@ #include #include #include -#include "core/partitioning/constraint.h" +#include "core/partitioning/detail/constraint.h" #include "core/partitioning/restriction.h" #include "core/utilities/ordered_set.h" diff --git a/src/core/partitioning/partitioner.cc b/src/core/partitioning/detail/partitioner.cc similarity index 98% rename from src/core/partitioning/partitioner.cc rename to src/core/partitioning/detail/partitioner.cc index 0c9940c6b2..ff746b6b27 100644 --- a/src/core/partitioning/partitioner.cc +++ b/src/core/partitioning/detail/partitioner.cc @@ -14,13 +14,13 @@ * */ -#include "core/partitioning/partitioner.h" +#include "core/partitioning/detail/partitioner.h" + #include "core/data/detail/logical_store.h" #include "core/data/logical_store.h" #include "core/data/scalar.h" #include "core/operation/detail/operation.h" -#include "core/partitioning/constraint.h" -#include "core/partitioning/constraint_solver.h" +#include "core/partitioning/detail/constraint_solver.h" #include "core/partitioning/partition.h" #include "core/runtime/detail/runtime.h" diff --git a/src/core/partitioning/partitioner.h b/src/core/partitioning/detail/partitioner.h similarity index 98% rename from src/core/partitioning/partitioner.h rename to src/core/partitioning/detail/partitioner.h index 6382160b1d..9576b11283 100644 --- a/src/core/partitioning/partitioner.h +++ b/src/core/partitioning/detail/partitioner.h @@ -21,7 +21,7 @@ #include #include "core/data/shape.h" -#include "core/partitioning/constraint.h" +#include "core/partitioning/detail/constraint.h" #include "core/partitioning/restriction.h" namespace legate { diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index 29feabefca..cfd3cada26 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -20,6 +20,7 @@ #include "core/partitioning/partition.h" #include "core/runtime/detail/partition_manager.h" #include "core/runtime/detail/runtime.h" +#include "core/type/detail/type_info.h" namespace legate { @@ -302,7 +303,7 @@ Legion::LogicalPartition Image::construct(Legion::LogicalRegion region, bool com func_partition_->construct(func_region, func_partition_->is_complete_for(func_->get_storage())); auto runtime = detail::Runtime::get_runtime(); - bool is_range = func_->type().code == Type::Code::STRUCT; + bool is_range = func_->type()->code == Type::Code::STRUCT; auto color_space = runtime->find_or_create_index_space(to_domain(color_shape())); auto index_partition = runtime->create_image_partition(region.get_index_space(), diff --git a/src/core/runtime/context.cc b/src/core/runtime/context.cc deleted file mode 100644 index d83ad9d2b4..0000000000 --- a/src/core/runtime/context.cc +++ /dev/null @@ -1,315 +0,0 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#include "legion.h" - -#include "core/data/buffer.h" -#include "core/data/scalar.h" -#include "core/data/store.h" -#include "core/mapping/base_mapper.h" -#include "core/runtime/context.h" -#include "core/runtime/runtime.h" -#include "core/utilities/deserializer.h" - -#ifdef LEGATE_USE_CUDA -#include "core/cuda/cuda_help.h" -#endif - -#include "mappers/logging_wrapper.h" - -namespace legate { - -LibraryContext::LibraryContext(const std::string& library_name, - const ResourceConfig& config, - std::unique_ptr mapper) - : runtime_(Legion::Runtime::get_runtime()), library_name_(library_name) -{ - task_scope_ = ResourceIdScope( - runtime_->generate_library_task_ids(library_name.c_str(), config.max_tasks), config.max_tasks); - redop_scope_ = ResourceIdScope( - runtime_->generate_library_reduction_ids(library_name.c_str(), config.max_reduction_ops), - config.max_reduction_ops); - proj_scope_ = ResourceIdScope( - runtime_->generate_library_projection_ids(library_name.c_str(), config.max_projections), - config.max_projections); - shard_scope_ = ResourceIdScope( - runtime_->generate_library_sharding_ids(library_name.c_str(), config.max_shardings), - config.max_shardings); - mapper_id_ = runtime_->generate_library_mapper_ids(library_name.c_str(), 1); - - register_mapper(std::move(mapper)); -} - -const std::string& LibraryContext::get_library_name() const { return library_name_; } - -Legion::TaskID LibraryContext::get_task_id(int64_t local_task_id) const -{ - assert(task_scope_.valid()); - return task_scope_.translate(local_task_id); -} - -Legion::ReductionOpID LibraryContext::get_reduction_op_id(int64_t local_redop_id) const -{ - assert(redop_scope_.valid()); - return redop_scope_.translate(local_redop_id); -} - -Legion::ProjectionID LibraryContext::get_projection_id(int64_t local_proj_id) const -{ - if (local_proj_id == 0) - return 0; - else { - assert(proj_scope_.valid()); - return proj_scope_.translate(local_proj_id); - } -} - -Legion::ShardingID LibraryContext::get_sharding_id(int64_t local_shard_id) const -{ - assert(shard_scope_.valid()); - return shard_scope_.translate(local_shard_id); -} - -int64_t LibraryContext::get_local_task_id(Legion::TaskID task_id) const -{ - assert(task_scope_.valid()); - return task_scope_.invert(task_id); -} - -int64_t LibraryContext::get_local_reduction_op_id(Legion::ReductionOpID redop_id) const -{ - assert(redop_scope_.valid()); - return redop_scope_.invert(redop_id); -} - -int64_t LibraryContext::get_local_projection_id(Legion::ProjectionID proj_id) const -{ - if (proj_id == 0) - return 0; - else { - assert(proj_scope_.valid()); - return proj_scope_.invert(proj_id); - } -} - -int64_t LibraryContext::get_local_sharding_id(Legion::ShardingID shard_id) const -{ - assert(shard_scope_.valid()); - return shard_scope_.invert(shard_id); -} - -bool LibraryContext::valid_task_id(Legion::TaskID task_id) const -{ - return task_scope_.in_scope(task_id); -} - -bool LibraryContext::valid_reduction_op_id(Legion::ReductionOpID redop_id) const -{ - return redop_scope_.in_scope(redop_id); -} - -bool LibraryContext::valid_projection_id(Legion::ProjectionID proj_id) const -{ - return proj_scope_.in_scope(proj_id); -} - -bool LibraryContext::valid_sharding_id(Legion::ShardingID shard_id) const -{ - return shard_scope_.in_scope(shard_id); -} - -const std::string& LibraryContext::get_task_name(int64_t local_task_id) const -{ - return find_task(local_task_id)->name(); -} - -void LibraryContext::register_mapper(std::unique_ptr mapper) -{ - // Hold the pointer to the mapper to keep it alive - mapper_ = std::move(mapper); - - auto base_mapper = new mapping::BaseMapper(mapper_.get(), runtime_->get_mapper_runtime(), this); - Legion::Mapping::Mapper* legion_mapper = base_mapper; - if (Core::log_mapping_decisions) - legion_mapper = new Legion::Mapping::LoggingWrapper(base_mapper, &base_mapper->logger); - runtime_->add_mapper(mapper_id_, legion_mapper); -} - -void LibraryContext::register_task(int64_t local_task_id, std::unique_ptr task_info) -{ - auto task_id = get_task_id(local_task_id); - if (!task_scope_.in_scope(task_id)) { - std::stringstream ss; - ss << "Task " << local_task_id << " is invalid for library '" << library_name_ - << "' (max local task id: " << (task_scope_.size() - 1) << ")"; - throw std::out_of_range(std::move(ss).str()); - } - -#ifdef DEBUG_LEGATE - log_legate.debug() << "[" << library_name_ << "] task " << local_task_id - << " (global id: " << task_id << "), " << *task_info; -#endif - if (tasks_.find(local_task_id) != tasks_.end()) - throw std::invalid_argument("Task " + std::to_string(local_task_id) + - " already exists in library " + library_name_); - task_info->register_task(task_id); - tasks_.emplace(std::make_pair(local_task_id, std::move(task_info))); -} - -const TaskInfo* LibraryContext::find_task(int64_t local_task_id) const -{ - auto finder = tasks_.find(local_task_id); - if (tasks_.end() == finder) { - throw std::out_of_range("Library " + get_library_name() + " does not have task " + - std::to_string(local_task_id)); - } - return finder->second.get(); -} - -void LibraryContext::perform_callback(Legion::RegistrationWithArgsCallbackFnptr callback, - Legion::UntypedBuffer buffer) -{ - Legion::Runtime::perform_registration_callback(callback, buffer, true /*global*/); -} - -TaskContext::TaskContext(const Legion::Task* task, - const std::vector& regions, - Legion::Context context, - Legion::Runtime* runtime) - : task_(task), regions_(regions), context_(context), runtime_(runtime) -{ - { - mapping::MapperDataDeserializer dez(task); - machine_desc_ = dez.unpack(); - } - - TaskDeserializer dez(task, regions); - inputs_ = dez.unpack>(); - outputs_ = dez.unpack>(); - reductions_ = dez.unpack>(); - scalars_ = dez.unpack>(); - - can_raise_exception_ = dez.unpack(); - - bool insert_barrier = false; - Legion::PhaseBarrier arrival, wait; - if (task->is_index_space) { - insert_barrier = dez.unpack(); - if (insert_barrier) { - arrival = dez.unpack(); - wait = dez.unpack(); - } - comms_ = dez.unpack>(); - } - - // For reduction tree cases, some input stores may be mapped to NO_REGION - // when the number of subregions isn't a multiple of the chosen radix. - // To simplify the programming mode, we filter out those "invalid" stores out. - if (task_->tag == LEGATE_CORE_TREE_REDUCE_TAG) { - std::vector inputs; - for (auto& input : inputs_) - if (input.valid()) inputs.push_back(std::move(input)); - inputs_.swap(inputs); - } - - // CUDA drivers < 520 have a bug that causes deadlock under certain circumstances - // if the application has multiple threads that launch blocking kernels, such as - // NCCL all-reduce kernels. This barrier prevents such deadlock by making sure - // all CUDA driver calls from Realm are done before any of the GPU tasks starts - // making progress. - if (insert_barrier) { - arrival.arrive(); - wait.wait(); - } -#ifdef LEGATE_USE_CUDA - // If the task is running on a GPU and there is at least one scalar store for reduction, - // we need to wait for all the host-to-device copies for initialization to finish - if (Processor::get_executing_processor().kind() == Processor::Kind::TOC_PROC) - for (auto& reduction : reductions_) - if (reduction.is_future()) { - CHECK_CUDA(cudaDeviceSynchronize()); - break; - } -#endif -} - -bool TaskContext::is_single_task() const { return !task_->is_index_space; } - -DomainPoint TaskContext::get_task_index() const { return task_->index_point; } - -Domain TaskContext::get_launch_domain() const { return task_->index_domain; } - -void TaskContext::make_all_unbound_stores_empty() -{ - for (auto& output : outputs_) - if (output.is_unbound_store()) output.bind_empty_data(); -} - -ReturnValues TaskContext::pack_return_values() const -{ - auto return_values = get_return_values(); - if (can_raise_exception_) { - ReturnedException exn{}; - return_values.push_back(exn.pack()); - } - return ReturnValues(std::move(return_values)); -} - -ReturnValues TaskContext::pack_return_values_with_exception(int32_t index, - const std::string& error_message) const -{ - auto return_values = get_return_values(); - if (can_raise_exception_) { - ReturnedException exn(index, error_message); - return_values.push_back(exn.pack()); - } - return ReturnValues(std::move(return_values)); -} - -std::vector TaskContext::get_return_values() const -{ - size_t num_unbound_outputs = 0; - std::vector return_values; - - for (auto& output : outputs_) { - if (!output.is_unbound_store()) continue; - return_values.push_back(output.pack_weight()); - ++num_unbound_outputs; - } - for (auto& output : outputs_) { - if (!output.is_future()) continue; - return_values.push_back(output.pack()); - } - for (auto& reduction : reductions_) { - if (!reduction.is_future()) continue; - return_values.push_back(reduction.pack()); - } - - // If this is a reduction task, we do sanity checks on the invariants - // the Python code relies on. - if (task_->tag == LEGATE_CORE_TREE_REDUCE_TAG) { - if (return_values.size() != 1 || num_unbound_outputs != 1) { - legate::log_legate.error("Reduction tasks must have only one unbound output and no others"); - LEGATE_ABORT; - } - } - - return return_values; -} - -const std::string& TaskContext::get_provenance() const { return task_->get_provenance_string(); } - -} // namespace legate diff --git a/src/core/runtime/detail/communicator_manager.h b/src/core/runtime/detail/communicator_manager.h index ec6d3498f1..77258e5a46 100644 --- a/src/core/runtime/detail/communicator_manager.h +++ b/src/core/runtime/detail/communicator_manager.h @@ -19,7 +19,7 @@ #include #include -#include "core/mapping/machine.h" +#include "core/mapping/detail/machine.h" #include "core/utilities/typedefs.h" namespace legate::detail { @@ -49,10 +49,11 @@ class CommunicatorFactory { virtual bool is_supported_target(mapping::TaskTarget target) const = 0; protected: - virtual Legion::FutureMap initialize(const mapping::MachineDesc& machine, uint32_t num_tasks) = 0; - virtual void finalize(const mapping::MachineDesc& machine, + virtual Legion::FutureMap initialize(const mapping::detail::Machine& machine, + uint32_t num_tasks) = 0; + virtual void finalize(const mapping::detail::Machine& machine, uint32_t num_tasks, - const Legion::FutureMap& communicator) = 0; + const Legion::FutureMap& communicator) = 0; private: template @@ -60,7 +61,10 @@ class CommunicatorFactory { Desc desc; mapping::TaskTarget target; mapping::ProcessorRange range; - mapping::MachineDesc get_machine() const { return mapping::MachineDesc({{target, range}}); } + mapping::detail::Machine get_machine() const + { + return mapping::detail::Machine({{target, range}}); + } bool operator==(const CacheKey& other) const { return desc == other.desc && target == other.target && range == other.range; diff --git a/src/core/runtime/detail/library.cc b/src/core/runtime/detail/library.cc new file mode 100644 index 0000000000..fbfdf07fe0 --- /dev/null +++ b/src/core/runtime/detail/library.cc @@ -0,0 +1,173 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/runtime/detail/library.h" + +#include "core/mapping/detail/base_mapper.h" +#include "core/mapping/machine.h" +#include "core/mapping/mapping.h" +#include "core/runtime/runtime.h" + +#include "mappers/logging_wrapper.h" + +namespace legate::detail { + +Library::Library(const std::string& library_name, + const ResourceConfig& config, + std::unique_ptr mapper) + : runtime_(Legion::Runtime::get_runtime()), library_name_(library_name) +{ + task_scope_ = ResourceIdScope( + runtime_->generate_library_task_ids(library_name.c_str(), config.max_tasks), config.max_tasks); + redop_scope_ = ResourceIdScope( + runtime_->generate_library_reduction_ids(library_name.c_str(), config.max_reduction_ops), + config.max_reduction_ops); + proj_scope_ = ResourceIdScope( + runtime_->generate_library_projection_ids(library_name.c_str(), config.max_projections), + config.max_projections); + shard_scope_ = ResourceIdScope( + runtime_->generate_library_sharding_ids(library_name.c_str(), config.max_shardings), + config.max_shardings); + mapper_id_ = runtime_->generate_library_mapper_ids(library_name.c_str(), 1); + + register_mapper(std::move(mapper)); +} + +const std::string& Library::get_library_name() const { return library_name_; } + +Legion::TaskID Library::get_task_id(int64_t local_task_id) const +{ + assert(task_scope_.valid()); + return task_scope_.translate(local_task_id); +} + +Legion::ReductionOpID Library::get_reduction_op_id(int64_t local_redop_id) const +{ + assert(redop_scope_.valid()); + return redop_scope_.translate(local_redop_id); +} + +Legion::ProjectionID Library::get_projection_id(int64_t local_proj_id) const +{ + if (local_proj_id == 0) + return 0; + else { + assert(proj_scope_.valid()); + return proj_scope_.translate(local_proj_id); + } +} + +Legion::ShardingID Library::get_sharding_id(int64_t local_shard_id) const +{ + assert(shard_scope_.valid()); + return shard_scope_.translate(local_shard_id); +} + +int64_t Library::get_local_task_id(Legion::TaskID task_id) const +{ + assert(task_scope_.valid()); + return task_scope_.invert(task_id); +} + +int64_t Library::get_local_reduction_op_id(Legion::ReductionOpID redop_id) const +{ + assert(redop_scope_.valid()); + return redop_scope_.invert(redop_id); +} + +int64_t Library::get_local_projection_id(Legion::ProjectionID proj_id) const +{ + if (proj_id == 0) + return 0; + else { + assert(proj_scope_.valid()); + return proj_scope_.invert(proj_id); + } +} + +int64_t Library::get_local_sharding_id(Legion::ShardingID shard_id) const +{ + assert(shard_scope_.valid()); + return shard_scope_.invert(shard_id); +} + +bool Library::valid_task_id(Legion::TaskID task_id) const { return task_scope_.in_scope(task_id); } + +bool Library::valid_reduction_op_id(Legion::ReductionOpID redop_id) const +{ + return redop_scope_.in_scope(redop_id); +} + +bool Library::valid_projection_id(Legion::ProjectionID proj_id) const +{ + return proj_scope_.in_scope(proj_id); +} + +bool Library::valid_sharding_id(Legion::ShardingID shard_id) const +{ + return shard_scope_.in_scope(shard_id); +} + +const std::string& Library::get_task_name(int64_t local_task_id) const +{ + return find_task(local_task_id)->name(); +} + +void Library::register_mapper(std::unique_ptr mapper) +{ + // Hold the pointer to the mapper to keep it alive + mapper_ = std::move(mapper); + + auto base_mapper = + new mapping::detail::BaseMapper(mapper_.get(), runtime_->get_mapper_runtime(), this); + Legion::Mapping::Mapper* legion_mapper = base_mapper; + if (Core::log_mapping_decisions) + legion_mapper = new Legion::Mapping::LoggingWrapper(base_mapper, &base_mapper->logger); + runtime_->add_mapper(mapper_id_, legion_mapper); +} + +void Library::register_task(int64_t local_task_id, std::unique_ptr task_info) +{ + auto task_id = get_task_id(local_task_id); + if (!task_scope_.in_scope(task_id)) { + std::stringstream ss; + ss << "Task " << local_task_id << " is invalid for library '" << library_name_ + << "' (max local task id: " << (task_scope_.size() - 1) << ")"; + throw std::out_of_range(std::move(ss).str()); + } + +#ifdef DEBUG_LEGATE + log_legate.debug() << "[" << library_name_ << "] task " << local_task_id + << " (global id: " << task_id << "), " << *task_info; +#endif + if (tasks_.find(local_task_id) != tasks_.end()) + throw std::invalid_argument("Task " + std::to_string(local_task_id) + + " already exists in library " + library_name_); + task_info->register_task(task_id); + tasks_.emplace(std::make_pair(local_task_id, std::move(task_info))); +} + +const TaskInfo* Library::find_task(int64_t local_task_id) const noexcept(false) +{ + auto finder = tasks_.find(local_task_id); + if (tasks_.end() == finder) { + throw std::out_of_range("Library " + get_library_name() + " does not have task " + + std::to_string(local_task_id)); + } + return finder->second.get(); +} + +} // namespace legate::detail diff --git a/src/core/runtime/detail/library.h b/src/core/runtime/detail/library.h new file mode 100644 index 0000000000..c89818721d --- /dev/null +++ b/src/core/runtime/detail/library.h @@ -0,0 +1,135 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include +#include + +#include "core/mapping/mapping.h" +#include "core/runtime/resource.h" +#include "core/task/task_info.h" +#include "core/utilities/typedefs.h" + +namespace legate::detail { + +class Runtime; + +class Library { + private: + class ResourceIdScope { + public: + ResourceIdScope() = default; + ResourceIdScope(int64_t base, int64_t size) : base_(base), size_(size) {} + + public: + ResourceIdScope(const ResourceIdScope&) = default; + + public: + int64_t translate(int64_t local_resource_id) const { return base_ + local_resource_id; } + int64_t invert(int64_t resource_id) const + { + assert(in_scope(resource_id)); + return resource_id - base_; + } + int64_t generate_id() + { + if (next_ == size_) throw std::overflow_error("The scope ran out of IDs"); + return next_++; + } + + public: + bool valid() const { return base_ != -1; } + bool in_scope(int64_t resource_id) const + { + return base_ <= resource_id && resource_id < base_ + size_; + } + int64_t size() const { return size_; } + + private: + int64_t base_{-1}; + int64_t size_{-1}; + int64_t next_{0}; + }; + + private: + friend class Runtime; + Library(const std::string& library_name, + const ResourceConfig& config, + std::unique_ptr mapper); + + public: + Library(const Library&) = delete; + Library(Library&&) = delete; + + public: + const std::string& get_library_name() const; + + public: + Legion::TaskID get_task_id(int64_t local_task_id) const; + Legion::MapperID get_mapper_id() const { return mapper_id_; } + Legion::ReductionOpID get_reduction_op_id(int64_t local_redop_id) const; + Legion::ProjectionID get_projection_id(int64_t local_proj_id) const; + Legion::ShardingID get_sharding_id(int64_t local_shard_id) const; + + public: + int64_t get_local_task_id(Legion::TaskID task_id) const; + int64_t get_local_reduction_op_id(Legion::ReductionOpID redop_id) const; + int64_t get_local_projection_id(Legion::ProjectionID proj_id) const; + int64_t get_local_sharding_id(Legion::ShardingID shard_id) const; + + public: + bool valid_task_id(Legion::TaskID task_id) const; + bool valid_reduction_op_id(Legion::ReductionOpID redop_id) const; + bool valid_projection_id(Legion::ProjectionID proj_id) const; + bool valid_sharding_id(Legion::ShardingID shard_id) const; + + public: + int64_t get_new_task_id() { return task_scope_.generate_id(); } + + public: + /** + * @brief Returns the name of a task + * + * @param local_task_id Task id + * @return Name of the task + */ + const std::string& get_task_name(int64_t local_task_id) const; + void register_mapper(std::unique_ptr mapper); + + public: + void register_task(int64_t local_task_id, std::unique_ptr task_info); + const TaskInfo* find_task(int64_t local_task_id) const noexcept(false); + + private: + void perform_callback(Legion::RegistrationWithArgsCallbackFnptr callback, + Legion::UntypedBuffer buffer); + + private: + Legion::Runtime* runtime_; + const std::string library_name_; + ResourceIdScope task_scope_; + ResourceIdScope redop_scope_; + ResourceIdScope proj_scope_; + ResourceIdScope shard_scope_; + + private: + Legion::MapperID mapper_id_; + std::unique_ptr mapper_; + std::unordered_map> tasks_; +}; + +} // namespace legate::detail diff --git a/src/core/runtime/detail/machine_manager.cc b/src/core/runtime/detail/machine_manager.cc index 3ed0f00272..e41f7529be 100644 --- a/src/core/runtime/detail/machine_manager.cc +++ b/src/core/runtime/detail/machine_manager.cc @@ -19,9 +19,9 @@ namespace legate::detail { //////////////////////////////////////////// -// legate::MachineManager +// legate::detail::MachineManager //////////////////////////////////////////// -const mapping::MachineDesc& MachineManager::get_machine() const +const mapping::detail::Machine& MachineManager::get_machine() const { #ifdef DEBUG_LEGATE assert(machines_.size() > 0); @@ -29,14 +29,9 @@ const mapping::MachineDesc& MachineManager::get_machine() const return machines_.back(); } -void MachineManager::push_machine(const mapping::MachineDesc& machine) +void MachineManager::push_machine(mapping::detail::Machine&& machine) { - machines_.push_back(machine); -} - -void MachineManager::push_machine(mapping::MachineDesc&& machine) -{ - machines_.emplace_back(machine); + machines_.push_back(std::move(machine)); } void MachineManager::pop_machine() diff --git a/src/core/runtime/detail/machine_manager.h b/src/core/runtime/detail/machine_manager.h index 8294e04988..b7ccad662d 100644 --- a/src/core/runtime/detail/machine_manager.h +++ b/src/core/runtime/detail/machine_manager.h @@ -16,25 +16,24 @@ #pragma once -#include "core/mapping/machine.h" +#include "core/mapping/detail/machine.h" -namespace legate::mapping { -class MachineDesc; -} // namespace legate::mapping +namespace legate::mapping::detail { +class Machine; +} // namespace legate::mapping::detail namespace legate::detail { class MachineManager { public: - const mapping::MachineDesc& get_machine() const; + const mapping::detail::Machine& get_machine() const; - void push_machine(const mapping::MachineDesc& machine); - void push_machine(mapping::MachineDesc&& machine); + void push_machine(mapping::detail::Machine&& machine); void pop_machine(); private: - std::vector machines_; + std::vector machines_; }; } // namespace legate::detail diff --git a/src/core/runtime/detail/partition_manager.cc b/src/core/runtime/detail/partition_manager.cc index f468832e9a..a511f5474c 100644 --- a/src/core/runtime/detail/partition_manager.cc +++ b/src/core/runtime/detail/partition_manager.cc @@ -19,14 +19,14 @@ #include "core/legate_c.h" #include "core/mapping/machine.h" #include "core/partitioning/partition.h" -#include "core/runtime/context.h" +#include "core/runtime/detail/library.h" #include "core/runtime/detail/runtime.h" namespace legate::detail { -PartitionManager::PartitionManager(Runtime* runtime, const LibraryContext* context) +PartitionManager::PartitionManager(Runtime* runtime) { - auto mapper_id = context->get_mapper_id(); + auto mapper_id = runtime->core_library()->get_mapper_id(); min_shard_volume_ = runtime->get_tunable(mapper_id, LEGATE_CORE_TUNABLE_MIN_SHARD_VOLUME); @@ -35,7 +35,7 @@ PartitionManager::PartitionManager(Runtime* runtime, const LibraryContext* conte #endif } -const std::vector& PartitionManager::get_factors(const mapping::MachineDesc& machine) +const std::vector& PartitionManager::get_factors(const mapping::Machine& machine) { uint32_t curr_num_pieces = machine.count(); @@ -57,7 +57,7 @@ const std::vector& PartitionManager::get_factors(const mapping::Machin return finder->second; } -Shape PartitionManager::compute_launch_shape(const mapping::MachineDesc& machine, +Shape PartitionManager::compute_launch_shape(const mapping::Machine& machine, const Restrictions& restrictions, const Shape& shape) { diff --git a/src/core/runtime/detail/partition_manager.h b/src/core/runtime/detail/partition_manager.h index 06cbad6749..94e6872d58 100644 --- a/src/core/runtime/detail/partition_manager.h +++ b/src/core/runtime/detail/partition_manager.h @@ -18,19 +18,16 @@ #include -#include "legion.h" - #include "core/data/shape.h" #include "core/partitioning/restriction.h" namespace legate { -class LibraryContext; class Tiling; class Weighted; } // namespace legate namespace legate::mapping { -class MachineDesc; +class Machine; } // namespace legate::mapping namespace legate::detail { @@ -39,13 +36,13 @@ class Runtime; class PartitionManager { public: - PartitionManager(Runtime* runtime, const LibraryContext* context); + PartitionManager(Runtime* runtime); public: - const std::vector& get_factors(const mapping::MachineDesc& machine); + const std::vector& get_factors(const mapping::Machine& machine); public: - Shape compute_launch_shape(const mapping::MachineDesc& machine, + Shape compute_launch_shape(const mapping::Machine& machine, const Restrictions& restrictions, const Shape& shape); Shape compute_tile_shape(const Shape& extents, const Shape& launch_shape); diff --git a/src/core/runtime/projection.cc b/src/core/runtime/detail/projection.cc similarity index 96% rename from src/core/runtime/projection.cc rename to src/core/runtime/detail/projection.cc index ff378cbc21..17440f113e 100644 --- a/src/core/runtime/projection.cc +++ b/src/core/runtime/detail/projection.cc @@ -20,12 +20,11 @@ #include #include -#include "core/runtime/projection.h" +#include "core/runtime/detail/projection.h" -#include "core/runtime/context.h" +#include "core/runtime/detail/library.h" #include "core/utilities/dispatch.h" #include "core/utilities/typedefs.h" -#include "legate_defines.h" extern Legion::Logger log_legate; @@ -107,7 +106,7 @@ bool is_identity(int32_t src_ndim, const SymbolicPoint& point) } // namespace legate::proj -namespace legate { +namespace legate::detail { // This special functor overrides the default projection implementation because it needs // to know the the target color space for delinearization. Also note that this functor's @@ -303,9 +302,9 @@ struct create_affine_functor_fn { }; void register_legate_core_projection_functors(Legion::Runtime* runtime, - const LibraryContext* context) + const detail::Library* core_library) { - auto proj_id = context->get_projection_id(LEGATE_CORE_DELINEARIZE_PROJ_ID); + auto proj_id = core_library->get_projection_id(LEGATE_CORE_DELINEARIZE_PROJ_ID); auto functor = new DelinearizationFunctor(runtime); log_legate.debug("Register delinearizing functor: functor: %p, id: %d", functor, proj_id); runtime->register_projection_functor(proj_id, functor, true /*silence warnings*/); @@ -355,7 +354,7 @@ struct LinearizingPointTransformFunctor : public Legion::PointTransformFunctor { static auto* linearizing_point_transform_functor = new LinearizingPointTransformFunctor(); -} // namespace legate +} // namespace legate::detail extern "C" { @@ -369,7 +368,7 @@ void legate_register_affine_projection_functor(int32_t src_ndim, auto runtime = Legion::Runtime::get_runtime(); legate::double_dispatch(src_ndim, tgt_ndim, - legate::create_affine_functor_fn{}, + legate::detail::create_affine_functor_fn{}, runtime, dims, weights, @@ -379,6 +378,6 @@ void legate_register_affine_projection_functor(int32_t src_ndim, void* legate_linearizing_point_transform_functor() { - return legate::linearizing_point_transform_functor; + return legate::detail::linearizing_point_transform_functor; } } diff --git a/src/core/runtime/projection.h b/src/core/runtime/detail/projection.h similarity index 94% rename from src/core/runtime/projection.h rename to src/core/runtime/detail/projection.h index d60bad31ca..389b3403d1 100644 --- a/src/core/runtime/projection.h +++ b/src/core/runtime/detail/projection.h @@ -60,9 +60,9 @@ bool is_identity(int32_t ndim, const SymbolicPoint& point); } // namespace legate::proj -namespace legate { +namespace legate::detail { -class LibraryContext; +class Library; // Interface for Legate projection functors class LegateProjectionFunctor : public Legion::ProjectionFunctor { @@ -87,9 +87,9 @@ class LegateProjectionFunctor : public Legion::ProjectionFunctor { }; void register_legate_core_projection_functors(Legion::Runtime* runtime, - const LibraryContext* context); + const detail::Library* core_library); LegateProjectionFunctor* find_legate_projection_functor(Legion::ProjectionID proj_id, bool allow_missing = false); -} // namespace legate +} // namespace legate::detail diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index ae0799994a..a0e532db36 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -19,8 +19,9 @@ #include "core/comm/comm.h" #include "core/data/detail/logical_region_field.h" #include "core/data/detail/logical_store.h" -#include "core/mapping/core_mapper.h" -#include "core/mapping/default_mapper.h" +#include "core/mapping/detail/core_mapper.h" +#include "core/mapping/detail/default_mapper.h" +#include "core/mapping/detail/machine.h" #include "core/operation/detail/copy.h" #include "core/operation/detail/fill.h" #include "core/operation/detail/gather.h" @@ -28,9 +29,11 @@ #include "core/operation/detail/scatter_gather.h" #include "core/operation/detail/task.h" #include "core/operation/detail/task_launcher.h" -#include "core/partitioning/partitioner.h" -#include "core/runtime/projection.h" -#include "core/runtime/shard.h" +#include "core/partitioning/detail/partitioner.h" +#include "core/runtime/detail/library.h" +#include "core/runtime/detail/shard.h" +#include "core/runtime/runtime.h" +#include "core/task/detail/task_context.h" #include "env_defaults.h" #include "realm/network.h" @@ -39,6 +42,8 @@ namespace legate { Logger log_legate("legate"); +// This is the unique string name for our library which can be used from both C++ and Python to +// generate IDs const char* const core_library_name = "legate.core"; } // namespace legate @@ -62,22 +67,21 @@ Runtime::Runtime() Runtime::~Runtime() {} -LibraryContext* Runtime::create_library(const std::string& library_name, - const ResourceConfig& config, - std::unique_ptr mapper) +Library* Runtime::create_library(const std::string& library_name, + const ResourceConfig& config, + std::unique_ptr mapper) { if (libraries_.find(library_name) != libraries_.end()) throw std::invalid_argument("Library " + library_name + " already exists"); log_legate.debug("Library %s is created", library_name.c_str()); - if (nullptr == mapper) mapper = std::make_unique(); - auto context = new LibraryContext(library_name, config, std::move(mapper)); + if (nullptr == mapper) mapper = std::make_unique(); + auto context = new Library(library_name, config, std::move(mapper)); libraries_[library_name] = context; return context; } -LibraryContext* Runtime::find_library(const std::string& library_name, - bool can_fail /*=false*/) const +Library* Runtime::find_library(const std::string& library_name, bool can_fail /*=false*/) const { auto finder = libraries_.find(library_name); if (libraries_.end() == finder) { @@ -87,12 +91,12 @@ LibraryContext* Runtime::find_library(const std::string& library_name, return finder->second; } -LibraryContext* Runtime::find_or_create_library(const std::string& library_name, - const ResourceConfig& config, - std::unique_ptr mapper, - bool* created) +Library* Runtime::find_or_create_library(const std::string& library_name, + const ResourceConfig& config, + std::unique_ptr mapper, + bool* created) { - LibraryContext* result = find_library(library_name, true /*can_fail*/); + Library* result = find_library(library_name, true /*can_fail*/); if (result != nullptr) { if (created != nullptr) *created = false; return result; @@ -146,24 +150,25 @@ void Runtime::initialize(Legion::Context legion_context) if (initialized_) throw std::runtime_error("Legate runtime has already been initialized"); initialized_ = true; legion_context_ = legion_context; - core_context_ = find_library(core_library_name, false /*can_fail*/); + core_library_ = find_library(core_library_name, false /*can_fail*/); communicator_manager_ = new CommunicatorManager(); - partition_manager_ = new PartitionManager(this, core_context_); + partition_manager_ = new PartitionManager(this); machine_manager_ = new MachineManager(); provenance_manager_ = new ProvenanceManager(); - Core::retrieve_tunable(legion_context_, legion_runtime_, core_context_); + Core::has_socket_mem = + get_tunable(core_library_->get_mapper_id(), LEGATE_CORE_TUNABLE_HAS_SOCKET_MEM); initialize_toplevel_machine(); - comm::register_builtin_communicator_factories(core_context_); + comm::register_builtin_communicator_factories(core_library_); } -mapping::MachineDesc Runtime::slice_machine_for_task(LibraryContext* library, int64_t task_id) +mapping::detail::Machine Runtime::slice_machine_for_task(Library* library, int64_t task_id) { auto* task_info = library->find_task(task_id); std::vector task_targets; auto& machine = machine_manager_->get_machine(); for (const auto& t : machine.valid_targets()) { - if (task_info->has_variant(mapping::to_variant_code(t))) task_targets.push_back(t); + if (task_info->has_variant(mapping::detail::to_variant_code(t))) task_targets.push_back(t); } auto sliced = machine.only(task_targets); @@ -178,14 +183,14 @@ mapping::MachineDesc Runtime::slice_machine_for_task(LibraryContext* library, in } // This function should be moved to the library context -std::unique_ptr Runtime::create_task(LibraryContext* library, int64_t task_id) +std::unique_ptr Runtime::create_task(Library* library, int64_t task_id) { auto machine = slice_machine_for_task(library, task_id); auto task = new AutoTask(library, task_id, next_unique_id_++, std::move(machine)); return std::unique_ptr(task); } -std::unique_ptr Runtime::create_task(LibraryContext* library, +std::unique_ptr Runtime::create_task(Library* library, int64_t task_id, const Shape& launch_shape) { @@ -274,14 +279,14 @@ void Runtime::schedule(std::vector> operations) for (auto& op : operations) op->launch(strategy.get()); } -std::shared_ptr Runtime::create_store(std::unique_ptr type, int32_t dim) +std::shared_ptr Runtime::create_store(std::shared_ptr type, int32_t dim) { auto storage = std::make_shared(dim, std::move(type)); return std::make_shared(std::move(storage)); } std::shared_ptr Runtime::create_store(const Shape& extents, - std::unique_ptr type, + std::shared_ptr type, bool optimize_scalar /*=false*/) { auto storage = std::make_shared(extents, std::move(type), optimize_scalar); @@ -291,8 +296,8 @@ std::shared_ptr Runtime::create_store(const Shape& extents, std::shared_ptr Runtime::create_store(const Scalar& scalar) { Shape extents{1}; - auto future = create_future(scalar.ptr(), scalar.size()); - auto storage = std::make_shared(extents, scalar.type().clone(), future); + auto future = create_future(scalar.data(), scalar.size()); + auto storage = std::make_shared(extents, scalar.type(), future); return std::make_shared(std::move(storage)); } @@ -379,7 +384,7 @@ RegionField Runtime::map_region_field(const LogicalRegionField* rf) Legion::RegionRequirement req(root_region, READ_WRITE, EXCLUSIVE, root_region); req.add_field(field_id); - auto mapper_id = core_context_->get_mapper_id(); + auto mapper_id = core_library_->get_mapper_id(); // TODO: We need to pass the metadata about logical store Legion::InlineLauncher launcher(req, mapper_id); pr = legion_runtime_->map_region(legion_context_, launcher); @@ -485,7 +490,7 @@ Legion::IndexPartition Runtime::create_image_partition( color_space, LEGION_COMPUTE_KIND, LEGION_AUTO_GENERATE_ID, - core_context_->get_mapper_id()); + core_library_->get_mapper_id()); else return legion_runtime_->create_partition_by_image(legion_context_, index_space, @@ -495,7 +500,7 @@ Legion::IndexPartition Runtime::create_image_partition( color_space, LEGION_COMPUTE_KIND, LEGION_AUTO_GENERATE_ID, - core_context_->get_mapper_id()); + core_library_->get_mapper_id()); } Legion::FieldSpace Runtime::create_field_space() @@ -657,9 +662,9 @@ Legion::Future Runtime::extract_scalar(const Legion::Future& result, uint32_t id { auto& machine = get_machine(); auto& provenance = provenance_manager()->get_provenance(); - auto variant = mapping::to_variant_code(machine.preferred_target); + auto variant = mapping::detail::to_variant_code(machine.preferred_target); TaskLauncher launcher( - core_context_, machine, provenance, LEGATE_CORE_EXTRACT_SCALAR_TASK_ID, variant); + core_library_, machine, provenance, LEGATE_CORE_EXTRACT_SCALAR_TASK_ID, variant); launcher.add_future(result); launcher.add_scalar(Scalar(idx)); return launcher.execute_single(); @@ -671,9 +676,9 @@ Legion::FutureMap Runtime::extract_scalar(const Legion::FutureMap& result, { auto& machine = get_machine(); auto& provenance = provenance_manager()->get_provenance(); - auto variant = mapping::to_variant_code(machine.preferred_target); + auto variant = mapping::detail::to_variant_code(machine.preferred_target); TaskLauncher launcher( - core_context_, machine, provenance, LEGATE_CORE_EXTRACT_SCALAR_TASK_ID, variant); + core_library_, machine, provenance, LEGATE_CORE_EXTRACT_SCALAR_TASK_ID, variant); launcher.add_future_map(result); launcher.add_scalar(Scalar(idx)); return launcher.execute(launch_domain); @@ -686,17 +691,17 @@ Legion::Future Runtime::reduce_future_map(const Legion::FutureMap& future_map, future_map, reduction_op, false /*deterministic*/, - core_context_->get_mapper_id()); + core_library_->get_mapper_id()); } Legion::Future Runtime::reduce_exception_future_map(const Legion::FutureMap& future_map) const { - auto reduction_op = core_context_->get_reduction_op_id(LEGATE_CORE_JOIN_EXCEPTION_OP); + auto reduction_op = core_library_->get_reduction_op_id(LEGATE_CORE_JOIN_EXCEPTION_OP); return legion_runtime_->reduce_future_map(legion_context_, future_map, reduction_op, false /*deterministic*/, - core_context_->get_mapper_id(), + core_library_->get_mapper_id(), LEGATE_CORE_JOIN_EXCEPTION_TAG); } @@ -710,7 +715,7 @@ void Runtime::issue_execution_fence(bool block /*=false*/) void Runtime::initialize_toplevel_machine() { - auto mapper_id = core_context_->get_mapper_id(); + auto mapper_id = core_library_->get_mapper_id(); auto num_nodes = get_tunable(mapper_id, LEGATE_CORE_TUNABLE_NUM_NODES); auto num_gpus = get_tunable(mapper_id, LEGATE_CORE_TUNABLE_TOTAL_GPUS); @@ -722,9 +727,9 @@ void Runtime::initialize_toplevel_machine() return mapping::ProcessorRange(0, num_procs, per_node_count); }; - mapping::MachineDesc machine({{mapping::TaskTarget::GPU, create_range(num_gpus)}, - {mapping::TaskTarget::OMP, create_range(num_omps)}, - {mapping::TaskTarget::CPU, create_range(num_cpus)}}); + mapping::detail::Machine machine({{mapping::TaskTarget::GPU, create_range(num_gpus)}, + {mapping::TaskTarget::OMP, create_range(num_omps)}, + {mapping::TaskTarget::CPU, create_range(num_cpus)}}); #ifdef DEBUG_LEGATE assert(machine_manager_ != nullptr); #endif @@ -732,7 +737,7 @@ void Runtime::initialize_toplevel_machine() machine_manager_->push_machine(std::move(machine)); } -const mapping::MachineDesc& Runtime::get_machine() const +const mapping::detail::Machine& Runtime::get_machine() const { #ifdef DEBUG_LEGATE assert(machine_manager_ != nullptr); @@ -758,7 +763,7 @@ Legion::ProjectionID Runtime::get_projection(int32_t src_ndim, const proj::Symbo auto finder = registered_projections_.find(key); if (registered_projections_.end() != finder) return finder->second; - auto proj_id = core_context_->get_projection_id(next_projection_id_++); + auto proj_id = core_library_->get_projection_id(next_projection_id_++); auto ndim = point.size(); std::vector dims; @@ -783,10 +788,10 @@ Legion::ProjectionID Runtime::get_projection(int32_t src_ndim, const proj::Symbo Legion::ProjectionID Runtime::get_delinearizing_projection() { - return core_context_->get_projection_id(LEGATE_CORE_DELINEARIZE_PROJ_ID); + return core_library_->get_projection_id(LEGATE_CORE_DELINEARIZE_PROJ_ID); } -Legion::ShardingID Runtime::get_sharding(const mapping::MachineDesc& machine, +Legion::ShardingID Runtime::get_sharding(const mapping::detail::Machine& machine, Legion::ProjectionID proj_id) { // If we're running on a single node, we don't need to generate sharding functors @@ -811,7 +816,7 @@ Legion::ShardingID Runtime::get_sharding(const mapping::MachineDesc& machine, return finder->second; } - auto sharding_id = core_context_->get_sharding_id(next_sharding_id_++); + auto sharding_id = core_library_->get_sharding_id(next_sharding_id_++); registered_shardings_.insert({key, sharding_id}); #ifdef DEBUG_LEGATE @@ -918,7 +923,7 @@ void Runtime::destroy() pending_exceptions_.clear(); // We finally deallocate managers - for (auto& [_, context] : libraries_) delete context; + for (auto& [_, library] : libraries_) delete library; libraries_.clear(); for (auto& [_, region_manager] : region_managers_) delete region_manager; region_managers_.clear(); @@ -943,7 +948,7 @@ static void extract_scalar_task( Core::show_progress(task, legion_context, runtime); - TaskContext context(task, *regions, legion_context, runtime); + detail::TaskContext context(task, *regions); auto idx = context.scalars()[0].value(); auto value_and_size = ReturnValues::extract(task->futures[0], idx); @@ -953,7 +958,7 @@ static void extract_scalar_task( void register_legate_core_tasks(Legion::Machine machine, Legion::Runtime* runtime, - LibraryContext* context) + Library* core_lib) { auto task_info = std::make_unique("core::extract_scalar"); auto register_extract_scalar_variant = [&](auto variant_id) { @@ -968,9 +973,9 @@ void register_legate_core_tasks(Legion::Machine machine, #ifdef LEGATE_USE_OPENMP register_extract_scalar_variant(LEGATE_OMP_VARIANT); #endif - context->register_task(LEGATE_CORE_EXTRACT_SCALAR_TASK_ID, std::move(task_info)); + core_lib->register_task(LEGATE_CORE_EXTRACT_SCALAR_TASK_ID, std::move(task_info)); - comm::register_tasks(machine, runtime, context); + comm::register_tasks(runtime, core_lib); } #define BUILTIN_REDOP_ID(OP, TYPE_CODE) \ @@ -1021,8 +1026,7 @@ void register_builtin_reduction_ops() RECORD_INT(XOR_LT) } -extern void register_exception_reduction_op(Legion::Runtime* runtime, - const LibraryContext* context); +extern void register_exception_reduction_op(Legion::Runtime* runtime, const Library* context); void core_library_registration(Legion::Machine machine, Legion::Runtime* legion_runtime, @@ -1037,7 +1041,7 @@ void core_library_registration(Legion::Machine machine, auto runtime = Runtime::get_runtime(); auto core_lib = Runtime::get_runtime()->create_library( - core_library_name, config, mapping::create_core_mapper()); + core_library_name, config, mapping::detail::create_core_mapper()); register_legate_core_tasks(machine, legion_runtime, core_lib); diff --git a/src/core/runtime/detail/runtime.h b/src/core/runtime/detail/runtime.h index f4371a3a5c..37fce3437a 100644 --- a/src/core/runtime/detail/runtime.h +++ b/src/core/runtime/detail/runtime.h @@ -19,14 +19,16 @@ #include #include +#include "core/data/detail/scalar.h" +#include "core/data/detail/store.h" #include "core/data/logical_store.h" #include "core/data/shape.h" -#include "core/data/store.h" #include "core/mapping/machine.h" #include "core/runtime/detail/communicator_manager.h" #include "core/runtime/detail/field_manager.h" #include "core/runtime/detail/machine_manager.h" #include "core/runtime/detail/partition_manager.h" +#include "core/runtime/detail/projection.h" #include "core/runtime/detail/provenance_manager.h" #include "core/runtime/detail/region_manager.h" #include "core/runtime/resource.h" @@ -34,14 +36,11 @@ #include "core/type/type_info.h" #include "core/utilities/multi_set.h" -namespace legate { -class LibraryContext; -} // namespace legate - namespace legate::detail { class AutoTask; class Copy; +class Library; class LogicalRegionField; class LogicalStore; class ManualTask; @@ -53,14 +52,14 @@ class Runtime { ~Runtime(); public: - LibraryContext* create_library(const std::string& library_name, - const ResourceConfig& config, - std::unique_ptr mapper); - LibraryContext* find_library(const std::string& library_name, bool can_fail) const; - LibraryContext* find_or_create_library(const std::string& library_name, - const ResourceConfig& config, - std::unique_ptr mapper, - bool* created); + Library* create_library(const std::string& library_name, + const ResourceConfig& config, + std::unique_ptr mapper); + Library* find_library(const std::string& library_name, bool can_fail) const; + Library* find_or_create_library(const std::string& library_name, + const ResourceConfig& config, + std::unique_ptr mapper, + bool* created); public: uint32_t get_type_uid(); @@ -71,9 +70,9 @@ class Runtime { void initialize(Legion::Context legion_context); public: - mapping::MachineDesc slice_machine_for_task(LibraryContext* library, int64_t task_id); - std::unique_ptr create_task(LibraryContext* library, int64_t task_id); - std::unique_ptr create_task(LibraryContext* library, + mapping::detail::Machine slice_machine_for_task(Library* library, int64_t task_id); + std::unique_ptr create_task(Library* library, int64_t task_id); + std::unique_ptr create_task(Library* library, int64_t task_id, const Shape& launch_shape); void issue_copy(std::shared_ptr target, std::shared_ptr source); @@ -92,9 +91,9 @@ class Runtime { void submit(std::unique_ptr op); public: - std::shared_ptr create_store(std::unique_ptr type, int32_t dim = 1); + std::shared_ptr create_store(std::shared_ptr type, int32_t dim = 1); std::shared_ptr create_store(const Shape& extents, - std::unique_ptr type, + std::shared_ptr type, bool optimize_scalar = false); std::shared_ptr create_store(const Scalar& scalar); @@ -192,12 +191,12 @@ class Runtime { public: void initialize_toplevel_machine(); - const mapping::MachineDesc& get_machine() const; + const mapping::detail::Machine& get_machine() const; public: Legion::ProjectionID get_projection(int32_t src_ndim, const proj::SymbolicPoint& point); Legion::ProjectionID get_delinearizing_projection(); - Legion::ShardingID get_sharding(const mapping::MachineDesc& machine, + Legion::ShardingID get_sharding(const mapping::detail::Machine& machine, Legion::ProjectionID proj_id); private: @@ -209,13 +208,13 @@ class Runtime { bool initialized() const { return initialized_; } void destroy(); int32_t finish(); - const LibraryContext* core_context() const { return core_context_; } + const Library* core_library() const { return core_library_; } private: bool initialized_{false}; Legion::Runtime* legion_runtime_{nullptr}; Legion::Context legion_context_{nullptr}; - LibraryContext* core_context_{nullptr}; + Library* core_library_{nullptr}; private: using FieldManagerKey = std::pair; @@ -252,7 +251,7 @@ class Runtime { uint64_t next_storage_id_{1}; private: - std::map libraries_{}; + std::map libraries_{}; private: uint32_t next_type_uid_; diff --git a/src/core/runtime/shard.cc b/src/core/runtime/detail/shard.cc similarity index 79% rename from src/core/runtime/shard.cc rename to src/core/runtime/detail/shard.cc index 48426f9913..0f1094c7e8 100644 --- a/src/core/runtime/shard.cc +++ b/src/core/runtime/detail/shard.cc @@ -14,16 +14,16 @@ * */ +#include "core/runtime/detail/shard.h" + #include #include -#include "legate.h" - -#include "core/runtime/projection.h" -#include "core/runtime/shard.h" +#include "core/runtime/detail/library.h" +#include "core/runtime/detail/projection.h" #include "core/utilities/linearize.h" -namespace legate { +namespace legate::detail { static std::unordered_map functor_id_table; static std::mutex functor_table_lock; @@ -82,19 +82,21 @@ class LinearizingShardingFunctor : public Legion::ShardingFunctor { } }; -void register_legate_core_sharding_functors(Legion::Runtime* runtime, const LibraryContext* context) +void register_legate_core_sharding_functors(Legion::Runtime* runtime, + const detail::Library* core_library) { - runtime->register_sharding_functor(context->get_sharding_id(LEGATE_CORE_TOPLEVEL_TASK_SHARD_ID), - new ToplevelTaskShardingFunctor(), - true /*silence warnings*/); + runtime->register_sharding_functor( + core_library->get_sharding_id(LEGATE_CORE_TOPLEVEL_TASK_SHARD_ID), + new ToplevelTaskShardingFunctor(), + true /*silence warnings*/); - auto sharding_id = context->get_sharding_id(LEGATE_CORE_LINEARIZE_SHARD_ID); + auto sharding_id = core_library->get_sharding_id(LEGATE_CORE_LINEARIZE_SHARD_ID); runtime->register_sharding_functor( sharding_id, new LinearizingShardingFunctor(), true /*silence warnings*/); // Use linearizing functor for identity projections functor_id_table[0] = sharding_id; // and for the delinearizing projection - functor_id_table[context->get_projection_id(LEGATE_CORE_DELINEARIZE_PROJ_ID)] = sharding_id; + functor_id_table[core_library->get_projection_id(LEGATE_CORE_DELINEARIZE_PROJ_ID)] = sharding_id; } class LegateShardingFunctor : public Legion::ShardingFunctor { @@ -138,7 +140,7 @@ class LegateShardingFunctor : public Legion::ShardingFunctor { Legion::ShardingID find_sharding_functor_by_projection_functor(Legion::ProjectionID proj_id) { - const std::lock_guard lock(legate::functor_table_lock); + const std::lock_guard lock(functor_table_lock); assert(functor_id_table.find(proj_id) != functor_id_table.end()); return functor_id_table[proj_id]; } @@ -156,17 +158,16 @@ static void sharding_functor_registration_callback(const Legion::RegistrationCal { auto p_args = static_cast(args.buffer.get_ptr()); - auto runtime = Legion::Runtime::get_runtime(); - auto sharding_functor = - new legate::LegateShardingFunctor(legate::find_legate_projection_functor(p_args->proj_id), - p_args->start_node, - p_args->end_node, - p_args->offset, - p_args->per_node_count); + auto runtime = Legion::Runtime::get_runtime(); + auto sharding_functor = new LegateShardingFunctor(find_legate_projection_functor(p_args->proj_id), + p_args->start_node, + p_args->end_node, + p_args->offset, + p_args->per_node_count); runtime->register_sharding_functor(p_args->shard_id, sharding_functor, true /*silence warnings*/); } -} // namespace legate +} // namespace legate::detail extern "C" { @@ -178,14 +179,17 @@ void legate_create_sharding_functor_using_projection(Legion::ShardID shard_id, uint32_t per_node_count) { auto runtime = Legion::Runtime::get_runtime(); - legate::ShardingCallbackArgs args{ + legate::detail::ShardingCallbackArgs args{ shard_id, proj_id, start_node, end_node, offset, per_node_count}; { - const std::lock_guard lock(legate::functor_table_lock); - legate::functor_id_table[proj_id] = shard_id; + const std::lock_guard lock(legate::detail::functor_table_lock); + legate::detail::functor_id_table[proj_id] = shard_id; } Legion::UntypedBuffer buffer(&args, sizeof(args)); Legion::Runtime::perform_registration_callback( - legate::sharding_functor_registration_callback, buffer, false /*global*/, false /*dedup*/); + legate::detail::sharding_functor_registration_callback, + buffer, + false /*global*/, + false /*dedup*/); } } diff --git a/src/core/runtime/shard.h b/src/core/runtime/detail/shard.h similarity index 83% rename from src/core/runtime/shard.h rename to src/core/runtime/detail/shard.h index ca110f5793..2db94047d9 100644 --- a/src/core/runtime/shard.h +++ b/src/core/runtime/detail/shard.h @@ -18,13 +18,13 @@ #include "legion.h" -#include "core/runtime/context.h" +namespace legate::detail { -namespace legate { +class Library; void register_legate_core_sharding_functors(Legion::Runtime* runtime, - const LibraryContext* context); + const detail::Library* core_library); Legion::ShardingID find_sharding_functor_by_projection_functor(Legion::ProjectionID proj_id); -} // namespace legate +} // namespace legate::detail diff --git a/src/core/runtime/library.cc b/src/core/runtime/library.cc new file mode 100644 index 0000000000..6f28e8088d --- /dev/null +++ b/src/core/runtime/library.cc @@ -0,0 +1,114 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/runtime/library.h" + +#include "core/mapping/mapping.h" +#include "core/runtime/detail/library.h" + +namespace legate { + +const std::string& Library::get_library_name() const { return impl_->get_library_name(); } + +Legion::TaskID Library::get_task_id(int64_t local_task_id) const +{ + return impl_->get_task_id(local_task_id); +} + +Legion::MapperID Library::get_mapper_id() const { return impl_->get_mapper_id(); } + +Legion::ReductionOpID Library::get_reduction_op_id(int64_t local_redop_id) const +{ + return impl_->get_reduction_op_id(local_redop_id); +} + +Legion::ProjectionID Library::get_projection_id(int64_t local_proj_id) const +{ + return impl_->get_projection_id(local_proj_id); +} + +Legion::ShardingID Library::get_sharding_id(int64_t local_shard_id) const +{ + return impl_->get_sharding_id(local_shard_id); +} + +int64_t Library::get_local_task_id(Legion::TaskID task_id) const +{ + return impl_->get_local_task_id(task_id); +} + +int64_t Library::get_local_reduction_op_id(Legion::ReductionOpID redop_id) const +{ + return impl_->get_local_reduction_op_id(redop_id); +} + +int64_t Library::get_local_projection_id(Legion::ProjectionID proj_id) const +{ + return impl_->get_local_projection_id(proj_id); +} + +int64_t Library::get_local_sharding_id(Legion::ShardingID shard_id) const +{ + return impl_->get_local_sharding_id(shard_id); +} + +bool Library::valid_task_id(Legion::TaskID task_id) const { return impl_->valid_task_id(task_id); } + +bool Library::valid_reduction_op_id(Legion::ReductionOpID redop_id) const +{ + return impl_->valid_reduction_op_id(redop_id); +} + +bool Library::valid_projection_id(Legion::ProjectionID proj_id) const +{ + return impl_->valid_projection_id(proj_id); +} + +bool Library::valid_sharding_id(Legion::ShardingID shard_id) const +{ + return impl_->valid_sharding_id(shard_id); +} + +int64_t Library::get_new_task_id() { return impl_->get_new_task_id(); } + +const std::string& Library::get_task_name(int64_t local_task_id) const +{ + return impl_->get_task_name(local_task_id); +} + +void Library::register_mapper(std::unique_ptr mapper) +{ + impl_->register_mapper(std::move(mapper)); +} + +void Library::register_task(int64_t local_task_id, std::unique_ptr task_info) +{ + impl_->register_task(local_task_id, std::move(task_info)); +} + +Library::Library(detail::Library* impl) : impl_(impl) {} + +bool Library::operator==(const Library& other) const { return impl_ == other.impl_; } + +bool Library::operator!=(const Library& other) const { return impl_ != other.impl_; } + +void Library::perform_callback(Legion::RegistrationWithArgsCallbackFnptr callback, + Legion::UntypedBuffer buffer) +{ + Legion::Runtime::perform_registration_callback(callback, buffer, true /*global*/); +} + +} // namespace legate diff --git a/src/core/runtime/context.h b/src/core/runtime/library.h similarity index 51% rename from src/core/runtime/context.h rename to src/core/runtime/library.h index 7880c530c5..eb78e83206 100644 --- a/src/core/runtime/context.h +++ b/src/core/runtime/library.h @@ -19,52 +19,32 @@ #include #include -#include "legion.h" -// Must be included after legion.h -#include "legate_defines.h" - -#include "core/comm/communicator.h" -#include "core/mapping/machine.h" #include "core/runtime/resource.h" -#include "core/task/return.h" #include "core/task/task_info.h" #include "core/utilities/typedefs.h" /** * @file - * @brief Class definitions for legate::LibraryContext and legate::TaskContext + * @brief Class definition for legate::Library */ +namespace legate::detail { +class Library; +} // namespace legate::detail + namespace legate::mapping { class Mapper; } // namespace legate::mapping -namespace legate::detail { -class Runtime; -} // namespace legate::detail - namespace legate { -class Store; -class Scalar; - -extern Legion::Logger log_legate; +class Runtime; /** * @ingroup runtime - * @brief A library context that provides APIs for registering components + * @brief A library class that provides APIs for registering components */ -class LibraryContext { - private: - friend class detail::Runtime; - LibraryContext(const std::string& library_name, - const ResourceConfig& config, - std::unique_ptr mapper); - - public: - LibraryContext(const LibraryContext&) = delete; - LibraryContext(LibraryContext&&) = default; - +class Library { public: /** * @brief Returns the name of the library @@ -75,7 +55,7 @@ class LibraryContext { public: Legion::TaskID get_task_id(int64_t local_task_id) const; - Legion::MapperID get_mapper_id() const { return mapper_id_; } + Legion::MapperID get_mapper_id() const; Legion::ReductionOpID get_reduction_op_id(int64_t local_redop_id) const; Legion::ProjectionID get_projection_id(int64_t local_proj_id) const; Legion::ShardingID get_sharding_id(int64_t local_shard_id) const; @@ -93,7 +73,7 @@ class LibraryContext { bool valid_sharding_id(Legion::ShardingID shard_id) const; public: - int64_t get_new_task_id() { return task_scope_.generate_id(); } + int64_t get_new_task_id(); public: /** @@ -169,133 +149,30 @@ class LibraryContext { public: void register_task(int64_t local_task_id, std::unique_ptr task_info); - const TaskInfo* find_task(int64_t local_task_id) const; - - private: - void perform_callback(Legion::RegistrationWithArgsCallbackFnptr callback, - Legion::UntypedBuffer buffer); - - private: - Legion::Runtime* runtime_; - const std::string library_name_; - ResourceIdScope task_scope_; - ResourceIdScope redop_scope_; - ResourceIdScope proj_scope_; - ResourceIdScope shard_scope_; private: - Legion::MapperID mapper_id_; - std::unique_ptr mapper_; - std::unordered_map> tasks_; -}; + friend class Runtime; + Library(detail::Library* impl); -/** - * @ingroup task - * @brief A task context that contains task arguments and communicators - */ -class TaskContext { public: - TaskContext(const Legion::Task* task, - const std::vector& regions, - Legion::Context context, - Legion::Runtime* runtime); + Library(const Library&) = default; + Library(Library&&) = default; public: - /** - * @brief Returns input stores of the task - * - * @return Vector of input stores - */ - std::vector& inputs() { return inputs_; } - /** - * @brief Returns output stores of the task - * - * @return Vector of output stores - */ - std::vector& outputs() { return outputs_; } - /** - * @brief Returns reduction stores of the task - * - * @return Vector of reduction stores - */ - std::vector& reductions() { return reductions_; } - /** - * @brief Returns by-value arguments of the task - * - * @return Vector of scalar objects - */ - std::vector& scalars() { return scalars_; } - /** - * @brief Returns communicators of the task - * - * If a task launch ends up emitting only a single point task, that task will not get passed a - * communicator, even if one was requested at task launching time. Therefore, most tasks using - * communicators should be prepared to handle the case where the returned vector is empty. - * - * @return Vector of communicator objects - */ - std::vector& communicators() { return comms_; } + bool operator==(const Library& other) const; + bool operator!=(const Library& other) const; public: - /** - * @brief Indicates whether the task is parallelized - * - * @return true The task is a single task - * @return false The task is one in a set of multiple parallel tasks - */ - bool is_single_task() const; - /** - * @brief Indicates whether the task is allowed to raise an exception - * - * @return true The task can raise an exception - * @return false The task must not raise an exception - */ - bool can_raise_exception() const { return can_raise_exception_; } - /** - * @brief Returns the point of the task. A 0D point will be returned for a single task. - * - * @return The point of the task - */ - DomainPoint get_task_index() const; - /** - * @brief Returns the task group's launch domain. A single task returns an empty domain - * - * @return The task group's launch domain - */ - Domain get_launch_domain() const; - - public: - const mapping::MachineDesc& machine_desc() const { return machine_desc_; } - - public: - /** - * @brief Makes all of unbound output stores of this task empty - */ - void make_all_unbound_stores_empty(); - ReturnValues pack_return_values() const; - ReturnValues pack_return_values_with_exception(int32_t index, - const std::string& error_message) const; - - public: - const std::string& get_provenance() const; + detail::Library* impl() const { return impl_; } private: - std::vector get_return_values() const; - - private: - const Legion::Task* task_; - const std::vector& regions_; - Legion::Context context_; - Legion::Runtime* runtime_; + void perform_callback(Legion::RegistrationWithArgsCallbackFnptr callback, + Legion::UntypedBuffer buffer); private: - std::vector inputs_, outputs_, reductions_; - std::vector scalars_; - std::vector comms_; - bool can_raise_exception_; - mapping::MachineDesc machine_desc_; + detail::Library* impl_; }; } // namespace legate -#include "core/runtime/context.inl" +#include "core/runtime/library.inl" diff --git a/src/core/runtime/context.inl b/src/core/runtime/library.inl similarity index 95% rename from src/core/runtime/context.inl rename to src/core/runtime/library.inl index f8b2ea1f76..91b8214d7d 100644 --- a/src/core/runtime/context.inl +++ b/src/core/runtime/library.inl @@ -17,7 +17,7 @@ #pragma once // Useful for IDEs -#include "core/runtime/context.h" +#include "core/runtime/library.h" namespace legate::detail { @@ -69,10 +69,11 @@ void register_reduction_callback(const Legion::RegistrationCallbackArgs& args) namespace legate { template -int32_t LibraryContext::register_reduction_operator(int32_t redop_id) +int32_t Library::register_reduction_operator(int32_t redop_id) { int32_t legion_redop_id = get_reduction_op_id(redop_id); #if defined(LEGATE_USE_CUDA) && !defined(REALM_COMPILER_IS_NVCC) + extern Logger log_legate; log_legate.error("Reduction operators must be registered in a .cu file when CUDA is enabled"); LEGATE_ABORT; #endif diff --git a/src/core/runtime/resource.h b/src/core/runtime/resource.h index c9fb560b20..b546d45c31 100644 --- a/src/core/runtime/resource.h +++ b/src/core/runtime/resource.h @@ -16,6 +16,8 @@ #pragma once +#include + namespace legate { /** @@ -35,39 +37,4 @@ struct ResourceConfig { int64_t max_shardings{0}; }; -class ResourceIdScope { - public: - ResourceIdScope() = default; - ResourceIdScope(int64_t base, int64_t size) : base_(base), size_(size) {} - - public: - ResourceIdScope(const ResourceIdScope&) = default; - - public: - int64_t translate(int64_t local_resource_id) const { return base_ + local_resource_id; } - int64_t invert(int64_t resource_id) const - { - assert(in_scope(resource_id)); - return resource_id - base_; - } - int64_t generate_id() - { - if (next_ == size_) throw std::overflow_error("The scope ran out of IDs"); - return next_++; - } - - public: - bool valid() const { return base_ != -1; } - bool in_scope(int64_t resource_id) const - { - return base_ <= resource_id && resource_id < base_ + size_; - } - int64_t size() const { return size_; } - - private: - int64_t base_{-1}; - int64_t size_{-1}; - int64_t next_{0}; -}; - } // namespace legate diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 9e9097939d..9a87c62e85 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -19,17 +19,13 @@ #include "core/operation/detail/copy.h" #include "core/operation/detail/operation.h" #include "core/operation/detail/task.h" -#include "core/runtime/context.h" +#include "core/partitioning/detail/constraint.h" #include "core/runtime/detail/runtime.h" namespace legate { extern Logger log_legate; -// This is the unique string name for our library which can be used -// from both C++ and Python to generate IDs -extern const char* const core_library_name; - /*static*/ bool Core::show_progress_requested = false; /*static*/ bool Core::use_empty_task = false; @@ -120,50 +116,45 @@ extern const char* const core_library_name; LEGATE_ABORT; } -/*static*/ void Core::retrieve_tunable(Legion::Context legion_context, - Legion::Runtime* legion_runtime, - LibraryContext* context) +/*static*/ void Core::perform_callback(Legion::RegistrationCallbackFnptr callback) { - auto fut = legion_runtime->select_tunable_value( - legion_context, LEGATE_CORE_TUNABLE_HAS_SOCKET_MEM, context->get_mapper_id()); - Core::has_socket_mem = fut.get_result(); + Legion::Runtime::perform_registration_callback(callback, true /*global*/); } -/*static*/ void Core::perform_callback(Legion::RegistrationCallbackFnptr callback) +Library Runtime::find_library(const std::string& library_name) const { - Legion::Runtime::perform_registration_callback(callback, true /*global*/); + return Library(impl_->find_library(library_name, false)); } -LibraryContext* Runtime::find_library(const std::string& library_name, - bool can_fail /*=false*/) const +std::optional Runtime::maybe_find_library(const std::string& library_name) const { - return impl_->find_library(library_name, can_fail); + auto result = impl_->find_library(library_name, true); + return result != nullptr ? std::optional(Library(result)) : std::nullopt; } -LibraryContext* Runtime::create_library(const std::string& library_name, - const ResourceConfig& config, - std::unique_ptr mapper) +Library Runtime::create_library(const std::string& library_name, + const ResourceConfig& config, + std::unique_ptr mapper) { - return impl_->create_library(library_name, config, std::move(mapper)); + return Library(impl_->create_library(library_name, config, std::move(mapper))); } -LibraryContext* Runtime::find_or_create_library(const std::string& library_name, - const ResourceConfig& config, - std::unique_ptr mapper, - bool* created) +Library Runtime::find_or_create_library(const std::string& library_name, + const ResourceConfig& config, + std::unique_ptr mapper, + bool* created) { - return impl_->find_or_create_library(library_name, config, std::move(mapper), created); + return Library(impl_->find_or_create_library(library_name, config, std::move(mapper), created)); } -// This function should be moved to the library context -AutoTask Runtime::create_task(LibraryContext* library, int64_t task_id) +AutoTask Runtime::create_task(Library library, int64_t task_id) { - return AutoTask(impl_->create_task(library, task_id)); + return AutoTask(impl_->create_task(library.impl(), task_id)); } -ManualTask Runtime::create_task(LibraryContext* library, int64_t task_id, const Shape& launch_shape) +ManualTask Runtime::create_task(Library library, int64_t task_id, const Shape& launch_shape) { - return ManualTask(impl_->create_task(library, task_id, launch_shape)); + return ManualTask(impl_->create_task(library.impl(), task_id, launch_shape)); } void Runtime::issue_copy(LogicalStore target, LogicalStore source) @@ -204,33 +195,21 @@ void Runtime::submit(AutoTask&& task) { impl_->submit(std::move(task.impl_)); } void Runtime::submit(ManualTask&& task) { impl_->submit(std::move(task.impl_)); } -LogicalStore Runtime::create_store(std::unique_ptr type, int32_t dim) -{ - return LogicalStore(impl_->create_store(std::move(type), dim)); -} - LogicalStore Runtime::create_store(const Type& type, int32_t dim) { - return create_store(type.clone(), dim); -} - -LogicalStore Runtime::create_store(const Shape& extents, - std::unique_ptr type, - bool optimize_scalar /*=false*/) -{ - return LogicalStore(impl_->create_store(extents, std::move(type), optimize_scalar)); + return LogicalStore(impl_->create_store(type.impl(), dim)); } LogicalStore Runtime::create_store(const Shape& extents, const Type& type, bool optimize_scalar /*=false*/) { - return create_store(extents, type.clone(), optimize_scalar); + return LogicalStore(impl_->create_store(extents, type.impl(), optimize_scalar)); } LogicalStore Runtime::create_store(const Scalar& scalar) { - return LogicalStore(impl_->create_store(scalar)); + return LogicalStore(impl_->create_store(*scalar.impl_)); } uint32_t Runtime::max_pending_exceptions() const { return impl_->max_pending_exceptions(); } @@ -249,7 +228,7 @@ std::optional Runtime::check_pending_task_exception() void Runtime::issue_execution_fence(bool block /*=false*/) { impl_->issue_execution_fence(block); } -const mapping::MachineDesc& Runtime::get_machine() const { return impl_->get_machine(); } +mapping::Machine Runtime::get_machine() const { return mapping::Machine(impl_->get_machine()); } /*static*/ Runtime* Runtime::get_runtime() { @@ -273,7 +252,7 @@ int32_t start(int32_t argc, char** argv) { return detail::Runtime::start(argc, a int32_t finish() { return detail::Runtime::get_runtime()->finish(); } -const mapping::MachineDesc& get_machine() { return Runtime::get_runtime()->get_machine(); } +mapping::Machine get_machine() { return Runtime::get_runtime()->get_machine(); } } // namespace legate diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 0bea67d5dd..47b10631d3 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -22,7 +22,9 @@ #include "core/data/logical_store.h" #include "core/data/shape.h" #include "core/data/store.h" +#include "core/mapping/machine.h" #include "core/operation/task.h" +#include "core/runtime/library.h" #include "core/runtime/resource.h" #include "core/task/exception.h" #include "core/utilities/typedefs.h" @@ -36,7 +38,6 @@ class Mapper; namespace legate { -class LibraryContext; class Scalar; class Type; @@ -56,9 +57,6 @@ struct Core { Legion::Context ctx, Legion::Runtime* runtime); static void report_unexpected_exception(const Legion::Task* task, const TaskException& e); - static void retrieve_tunable(Legion::Context legion_context, - Legion::Runtime* legion_runtime, - LibraryContext* context); public: /** @@ -115,25 +113,33 @@ class Runtime { * @param config Optional configuration object * @param mapper Optional mapper object * - * @return Context object for the library + * @return Library object * * @throw std::invalid_argument If a library already exists for a given name */ - LibraryContext* create_library(const std::string& library_name, - const ResourceConfig& config = ResourceConfig{}, - std::unique_ptr mapper = nullptr); + Library create_library(const std::string& library_name, + const ResourceConfig& config = ResourceConfig{}, + std::unique_ptr mapper = nullptr); /** * @brief Finds a library * * @param library_name Library name - * @param can_fail Optional flag indicating that the query can fail. When it's true and no - * library is found for a given name, `nullptr` is returned. * - * @return Context object for the library + * @return Library object + * + * @throw std::out_of_range If no library is found for a given name + */ + Library find_library(const std::string& library_name) const; + /** + * @brief Attempts to find a library. + * + * If no library exists for a given name, a null value will be returned * - * @throw std::out_of_range If no library is found for a given name and `can_fail` is `false` + * @param library_name Library name + * + * @return Library object if a library exists for a given name, a null object otherwise */ - LibraryContext* find_library(const std::string& library_name, bool can_fail = false) const; + std::optional maybe_find_library(const std::string& library_name) const; /** * @brief Finds or creates a library. * @@ -148,10 +154,10 @@ class Runtime { * * @return Context object for the library */ - LibraryContext* find_or_create_library(const std::string& library_name, - const ResourceConfig& config = ResourceConfig{}, - std::unique_ptr mapper = nullptr, - bool* created = nullptr); + Library find_or_create_library(const std::string& library_name, + const ResourceConfig& config = ResourceConfig{}, + std::unique_ptr mapper = nullptr, + bool* created = nullptr); public: /** @@ -162,7 +168,7 @@ class Runtime { * * @return Task object */ - AutoTask create_task(LibraryContext* library, int64_t task_id); + AutoTask create_task(Library library, int64_t task_id); /** * @brief Creates a ManualTask * @@ -172,7 +178,7 @@ class Runtime { * * @return Task object */ - ManualTask create_task(LibraryContext* library, int64_t task_id, const Shape& launch_shape); + ManualTask create_task(Library library, int64_t task_id, const Shape& launch_shape); /** * @brief Issues a copy between stores. * @@ -251,15 +257,6 @@ class Runtime { void submit(ManualTask&& task); public: - /** - * @brief Creates an unbound store - * - * @param type Element type - * @param dim Number of dimensions of the store - * - * @return Logical store - */ - LogicalStore create_store(std::unique_ptr type, int32_t dim = 1); /** * @brief Creates an unbound store * @@ -269,19 +266,6 @@ class Runtime { * @return Logical store */ LogicalStore create_store(const Type& type, int32_t dim = 1); - /** - * @brief Creates a normal store - * - * @param extents Shape of the store - * @param type Element type - * @param optimize_scalar When true, the runtime internally uses futures optimized for storing - * scalars - * - * @return Logical store - */ - LogicalStore create_store(const Shape& extents, - std::unique_ptr type, - bool optimize_scalar = false); /** * @brief Creates a normal store * @@ -345,7 +329,7 @@ class Runtime { * * @return Machine object */ - const mapping::MachineDesc& get_machine() const; + mapping::Machine get_machine() const; public: /** @@ -388,7 +372,7 @@ int32_t finish(); * * @return Machine object */ -const mapping::MachineDesc& get_machine(); +mapping::Machine get_machine(); } // namespace legate diff --git a/src/core/runtime/tracker.cc b/src/core/runtime/tracker.cc index e96652f9b9..3a08cd3eed 100644 --- a/src/core/runtime/tracker.cc +++ b/src/core/runtime/tracker.cc @@ -16,6 +16,7 @@ #include "core/runtime/tracker.h" +#include "core/mapping/detail/machine.h" #include "core/runtime/detail/machine_manager.h" #include "core/runtime/detail/provenance_manager.h" #include "core/runtime/detail/runtime.h" @@ -47,13 +48,13 @@ const std::string& ProvenanceTracker::get_current_provenance() const // legate::MachineTracker //////////////////////////////////////////// -MachineTracker::MachineTracker(const mapping::MachineDesc& machine) +MachineTracker::MachineTracker(const mapping::Machine& machine) { auto* runtime = detail::Runtime::get_runtime(); - auto result = machine & runtime->get_machine(); + auto result = machine & mapping::Machine(runtime->get_machine()); if (result.count() == 0) throw std::runtime_error("Empty machines cannot be used for resource scoping"); - runtime->machine_manager()->push_machine(std::move(result)); + runtime->machine_manager()->push_machine(std::move(*result.impl())); } MachineTracker::~MachineTracker() @@ -62,9 +63,9 @@ MachineTracker::~MachineTracker() runtime->machine_manager()->pop_machine(); } -const mapping::MachineDesc& MachineTracker::get_current_machine() const +mapping::Machine MachineTracker::get_current_machine() const { - return detail::Runtime::get_runtime()->get_machine(); + return mapping::Machine(detail::Runtime::get_runtime()->get_machine()); } } // namespace legate diff --git a/src/core/runtime/tracker.h b/src/core/runtime/tracker.h index 29b616e1a8..a4b30338bb 100644 --- a/src/core/runtime/tracker.h +++ b/src/core/runtime/tracker.h @@ -62,7 +62,7 @@ struct ProvenanceTracker { * * By default, Legate operations target the entire machine available for the program. When a client * program wants to assign a subset of the machine to its operations, it can subdivide the machine - * using the machine API (see `MachineDesc` for details) and set a sub-machine for the scope using + * using the machine API (see `Machine` for details) and set a sub-machine for the scope using * `MachineTracker`. All operations within the scope where the `MachineTracker` object is alive will * only target that sub-machine, instead of the entire machine. * @@ -73,7 +73,7 @@ struct MachineTracker { * * @param machine Machine to use for the scope */ - MachineTracker(const mapping::MachineDesc& machine); + MachineTracker(const mapping::Machine& machine); /** * @brief Pops out the machine set by this tracker @@ -85,7 +85,7 @@ struct MachineTracker { * * @return Machine */ - const mapping::MachineDesc& get_current_machine() const; + mapping::Machine get_current_machine() const; }; } // namespace legate diff --git a/src/core/task/return.cc b/src/core/task/detail/return.cc similarity index 96% rename from src/core/task/return.cc rename to src/core/task/detail/return.cc index 4d5ffc7aa8..2d48589ce5 100644 --- a/src/core/task/return.cc +++ b/src/core/task/detail/return.cc @@ -1,4 +1,4 @@ -/* Copyright 2021-2022 NVIDIA Corporation +/* Copyright 2023 NVIDIA Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,16 +14,9 @@ * */ -#include -#include -#include +#include "core/task/detail/return.h" -#include "legion.h" - -#include "core/data/buffer.h" -#include "core/legate_c.h" -#include "core/runtime/context.h" -#include "core/task/return.h" +#include "core/runtime/detail/library.h" #include "core/utilities/machine.h" #include "core/utilities/typedefs.h" #ifdef LEGATE_USE_CUDA @@ -31,7 +24,7 @@ #include "core/cuda/stream_pool.h" #endif -namespace legate { +namespace legate::detail { ReturnValue::ReturnValue(Legion::UntypedDeferredValue value, size_t size) : value_(value), size_(size) @@ -272,10 +265,6 @@ void ReturnValues::finalize(Legion::Context legion_context) const return_buffer.finalize(legion_context); } -} // namespace legate - -namespace legate::detail { - struct JoinReturnedException { using LHS = ReturnedException; using RHS = LHS; @@ -335,9 +324,9 @@ static void returned_exception_fold(const Legion::ReductionOp* reduction_op, pack_returned_exception(lhs, lhs_ptr, lhs_size); } -void register_exception_reduction_op(Legion::Runtime* runtime, const LibraryContext* context) +void register_exception_reduction_op(Legion::Runtime* runtime, const Library* library) { - auto redop_id = context->get_reduction_op_id(LEGATE_CORE_JOIN_EXCEPTION_OP); + auto redop_id = library->get_reduction_op_id(LEGATE_CORE_JOIN_EXCEPTION_OP); auto* redop = Realm::ReductionOpUntyped::create_reduction_op(); Legion::Runtime::register_reduction_op( redop_id, redop, returned_exception_init, returned_exception_fold); diff --git a/src/core/task/return.h b/src/core/task/detail/return.h similarity index 96% rename from src/core/task/return.h rename to src/core/task/detail/return.h index ae8bb1c7a8..11ce7958e0 100644 --- a/src/core/task/return.h +++ b/src/core/task/detail/return.h @@ -1,4 +1,4 @@ -/* Copyright 2021-2022 NVIDIA Corporation +/* Copyright 2023 NVIDIA Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ #include "core/task/exception.h" #include "core/utilities/typedefs.h" -namespace legate { +namespace legate::detail { struct ReturnValue { public: @@ -108,4 +108,4 @@ struct ReturnValues { std::vector return_values_{}; }; -} // namespace legate +} // namespace legate::detail diff --git a/src/core/task/detail/task_context.cc b/src/core/task/detail/task_context.cc new file mode 100644 index 0000000000..d66a9e2eb4 --- /dev/null +++ b/src/core/task/detail/task_context.cc @@ -0,0 +1,145 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/task/detail/task_context.h" + +#include "core/data/detail/store.h" +#include "core/utilities/deserializer.h" + +#ifdef LEGATE_USE_CUDA +#include "core/cuda/cuda_help.h" +#endif + +namespace legate::detail { + +TaskContext::TaskContext(const Legion::Task* task, + const std::vector& regions) + : task_(task), regions_(regions) +{ + { + mapping::MapperDataDeserializer dez(task); + machine_ = dez.unpack(); + } + + TaskDeserializer dez(task, regions); + inputs_ = dez.unpack>(); + outputs_ = dez.unpack>(); + reductions_ = dez.unpack>(); + scalars_ = dez.unpack>(); + + // Make copies of stores that we need to postprocess, as clients might move the stores away + for (auto& output : outputs_) { + if (output.is_unbound_store()) { + unbound_stores_.push_back(output); + } else if (output.is_future()) { + scalar_stores_.push_back(output); + } + } + for (auto& reduction : reductions_) { + if (reduction.is_future()) { scalar_stores_.push_back(reduction); } + } + + can_raise_exception_ = dez.unpack(); + + bool insert_barrier = false; + Legion::PhaseBarrier arrival, wait; + if (task->is_index_space) { + insert_barrier = dez.unpack(); + if (insert_barrier) { + arrival = dez.unpack(); + wait = dez.unpack(); + } + comms_ = dez.unpack>(); + } + + // For reduction tree cases, some input stores may be mapped to NO_REGION + // when the number of subregions isn't a multiple of the chosen radix. + // To simplify the programming mode, we filter out those "invalid" stores out. + if (task_->tag == LEGATE_CORE_TREE_REDUCE_TAG) { + std::vector inputs; + for (auto& input : inputs_) + if (input.valid()) inputs.push_back(std::move(input)); + inputs_.swap(inputs); + } + + // CUDA drivers < 520 have a bug that causes deadlock under certain circumstances + // if the application has multiple threads that launch blocking kernels, such as + // NCCL all-reduce kernels. This barrier prevents such deadlock by making sure + // all CUDA driver calls from Realm are done before any of the GPU tasks starts + // making progress. + if (insert_barrier) { + arrival.arrive(); + wait.wait(); + } +#ifdef LEGATE_USE_CUDA + // If the task is running on a GPU and there is at least one scalar store for reduction, + // we need to wait for all the host-to-device copies for initialization to finish + if (Processor::get_executing_processor().kind() == Processor::Kind::TOC_PROC) + for (auto& reduction : reductions_) + if (reduction.is_future()) { + CHECK_CUDA(cudaDeviceSynchronize()); + break; + } +#endif +} + +void TaskContext::make_all_unbound_stores_empty() +{ + for (auto& output : outputs_) + if (output.is_unbound_store()) output.bind_empty_data(); +} + +ReturnValues TaskContext::pack_return_values() const +{ + auto return_values = get_return_values(); + if (can_raise_exception_) { + ReturnedException exn{}; + return_values.push_back(exn.pack()); + } + return ReturnValues(std::move(return_values)); +} + +ReturnValues TaskContext::pack_return_values_with_exception(int32_t index, + const std::string& error_message) const +{ + auto return_values = get_return_values(); + if (can_raise_exception_) { + ReturnedException exn(index, error_message); + return_values.push_back(exn.pack()); + } + return ReturnValues(std::move(return_values)); +} + +std::vector TaskContext::get_return_values() const +{ + std::vector return_values; + + for (auto& store : unbound_stores_) { return_values.push_back(store.impl()->pack_weight()); } + for (auto& store : scalar_stores_) { return_values.push_back(store.impl()->pack()); } + + // If this is a reduction task, we do sanity checks on the invariants + // the Python code relies on. + if (task_->tag == LEGATE_CORE_TREE_REDUCE_TAG) { + if (return_values.size() != 1 || unbound_stores_.size() != 1) { + legate::log_legate.error("Reduction tasks must have only one unbound output and no others"); + LEGATE_ABORT; + } + } + + return return_values; +} + +} // namespace legate::detail diff --git a/src/core/task/detail/task_context.h b/src/core/task/detail/task_context.h new file mode 100644 index 0000000000..f3116de124 --- /dev/null +++ b/src/core/task/detail/task_context.h @@ -0,0 +1,74 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include "core/comm/communicator.h" +#include "core/data/scalar.h" +#include "core/data/store.h" +#include "core/mapping/detail/machine.h" +#include "core/task/detail/return.h" + +namespace legate::detail { + +class TaskContext { + public: + TaskContext(const Legion::Task* task, const std::vector& regions); + + public: + std::vector& inputs() { return inputs_; } + std::vector& outputs() { return outputs_; } + std::vector& reductions() { return reductions_; } + std::vector& scalars() { return scalars_; } + std::vector& communicators() { return comms_; } + + public: + bool is_single_task() const { return !task_->is_index_space; } + bool can_raise_exception() const { return can_raise_exception_; } + DomainPoint get_task_index() const { return task_->index_point; } + Domain get_launch_domain() const { return task_->index_domain; } + + public: + const mapping::detail::Machine& machine() const { return machine_; } + const std::string& get_provenance() const { return task_->get_provenance_string(); } + + public: + /** + * @brief Makes all of unbound output stores of this task empty + */ + void make_all_unbound_stores_empty(); + ReturnValues pack_return_values() const; + ReturnValues pack_return_values_with_exception(int32_t index, + const std::string& error_message) const; + + private: + std::vector get_return_values() const; + + private: + const Legion::Task* task_; + const std::vector& regions_; + + private: + std::vector inputs_, outputs_, reductions_; + std::vector unbound_stores_; + std::vector scalar_stores_; + std::vector scalars_; + std::vector comms_; + bool can_raise_exception_; + mapping::detail::Machine machine_; +}; + +} // namespace legate::detail diff --git a/src/core/task/exception.h b/src/core/task/exception.h index 9f34f8b765..7d756cd3d3 100644 --- a/src/core/task/exception.h +++ b/src/core/task/exception.h @@ -16,7 +16,9 @@ #pragma once +#include #include +#include /** * @file diff --git a/src/core/task/registrar.cc b/src/core/task/registrar.cc index 15efadad44..193381fc26 100644 --- a/src/core/task/registrar.cc +++ b/src/core/task/registrar.cc @@ -16,22 +16,31 @@ #include "core/task/registrar.h" -#include "core/runtime/context.h" +#include "core/runtime/detail/library.h" #include "core/task/task_info.h" #include "core/utilities/typedefs.h" namespace legate { +struct TaskRegistrar::Impl { + std::vector>> pending_task_infos; +}; + void TaskRegistrar::record_task(int64_t local_task_id, std::unique_ptr task_info) { - pending_task_infos_.push_back(std::make_pair(local_task_id, std::move(task_info))); + impl_->pending_task_infos.push_back(std::make_pair(local_task_id, std::move(task_info))); } -void TaskRegistrar::register_all_tasks(LibraryContext* context) +void TaskRegistrar::register_all_tasks(Library library) { - for (auto& [local_task_id, task_info] : pending_task_infos_) - context->register_task(local_task_id, std::move(task_info)); - pending_task_infos_.clear(); + auto* lib_impl = library.impl(); + for (auto& [local_task_id, task_info] : impl_->pending_task_infos) + lib_impl->register_task(local_task_id, std::move(task_info)); + impl_->pending_task_infos.clear(); } +TaskRegistrar::TaskRegistrar() : impl_(new TaskRegistrar::Impl()) {} + +TaskRegistrar::~TaskRegistrar() { delete impl_; } + } // namespace legate diff --git a/src/core/task/registrar.h b/src/core/task/registrar.h index 482d96b4c5..7cb04ab875 100644 --- a/src/core/task/registrar.h +++ b/src/core/task/registrar.h @@ -20,6 +20,7 @@ #include "legion.h" +#include "core/runtime/library.h" #include "core/task/variant_options.h" #include "core/utilities/typedefs.h" @@ -30,7 +31,6 @@ namespace legate { -class LibraryContext; class TaskInfo; /** @@ -75,20 +75,31 @@ class TaskInfo; * @endcode */ class TaskRegistrar { - public: - void record_task(int64_t local_task_id, std::unique_ptr task_info); - public: /** * @brief Registers all tasks recorded in this registrar. Typically invoked in a registration * callback of a library. * - * @param context Context of the library that owns this registrar + * @param library Library that owns this registrar */ - void register_all_tasks(LibraryContext* context); + void register_all_tasks(Library library); + + public: + void record_task(int64_t local_task_id, std::unique_ptr task_info); + + public: + TaskRegistrar(); + ~TaskRegistrar(); + + private: + TaskRegistrar(const TaskRegistrar&) = delete; + TaskRegistrar& operator=(const TaskRegistrar&) = delete; + TaskRegistrar(TaskRegistrar&&) = delete; + TaskRegistrar& operator=(TaskRegistrar&&) = delete; private: - std::vector>> pending_task_infos_; + class Impl; + Impl* impl_; }; } // namespace legate diff --git a/src/core/task/task.cc b/src/core/task/task.cc index 34a51b53ff..cfbd61046a 100644 --- a/src/core/task/task.cc +++ b/src/core/task/task.cc @@ -1,4 +1,4 @@ -/* Copyright 2021-2022 NVIDIA Corporation +/* Copyright 2023 NVIDIA Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,11 +20,12 @@ #include "realm/faults.h" -#include "core/runtime/context.h" #include "core/runtime/runtime.h" +#include "core/task/detail/return.h" +#include "core/task/detail/task_context.h" #include "core/task/exception.h" #include "core/task/registrar.h" -#include "core/task/return.h" +#include "core/task/task_context.h" #include "core/utilities/deserializer.h" #include "core/utilities/nvtx_help.h" #include "core/utilities/typedefs.h" @@ -67,11 +68,12 @@ void task_wrapper(VariantImpl variant_impl, Core::show_progress(task, legion_context, runtime); - TaskContext context(task, *regions, legion_context, runtime); + detail::TaskContext context(task, *regions); ReturnValues return_values{}; try { - if (!Core::use_empty_task) (*variant_impl)(context); + legate::TaskContext ctx(&context); + if (!Core::use_empty_task) (*variant_impl)(ctx); return_values = context.pack_return_values(); } catch (legate::TaskException& e) { if (context.can_raise_exception()) { diff --git a/src/core/task/task.h b/src/core/task/task.h index 7660ffbffd..39561e3ee6 100644 --- a/src/core/task/task.h +++ b/src/core/task/task.h @@ -16,7 +16,7 @@ #pragma once -#include "core/runtime/context.h" +#include "core/runtime/library.h" #include "core/task/task_info.h" #include "core/task/variant_helper.h" #include "core/task/variant_options.h" @@ -72,29 +72,28 @@ struct LegateTask { /** * @brief Registers all variants of this task immediately. * - * Unlike the other method, this one takes a library context so the registration can be done - * immediately. The value of T::TASK_ID is used as the task id. + * Unlike the other method, this one takes a library so the registration can be done immediately. + * The value of T::TASK_ID is used as the task id. * - * @param context Library to which the task should be registered + * @param library Library to which the task should be registered * @param all_options Options for task variants. Variants with no entires in `all_options` will * use the default set of options */ static void register_variants( - LibraryContext* context, const std::map& all_options = {}); + Library library, const std::map& all_options = {}); /** * @brief Registers all variants of this task immediately. * - * Unlike the other method, this one takes a library context so the registration can be done - * immediately. + * Unlike the other method, this one takes a library so the registration can be done immediately. * - * @param context Library to which the task should be registered + * @param library Library to which the task should be registered * @param task_id Task id * @param all_options Options for task variants. Variants with no entires in `all_options` will * use the default set of options */ static void register_variants( - LibraryContext* context, + Library library, int64_t task_id, const std::map& all_options = {}); diff --git a/src/core/task/task.inl b/src/core/task/task.inl index 0e19aeb700..02295a7a7a 100644 --- a/src/core/task/task.inl +++ b/src/core/task/task.inl @@ -48,19 +48,17 @@ template template /*static*/ void LegateTask::register_variants( - LibraryContext* context, const std::map& all_options) + Library library, const std::map& all_options) { - register_variants(context, T::TASK_ID, all_options); + register_variants(library, T::TASK_ID, all_options); } template /*static*/ void LegateTask::register_variants( - LibraryContext* context, - int64_t task_id, - const std::map& all_options) + Library library, int64_t task_id, const std::map& all_options) { auto task_info = create_task_info(all_options); - context->register_task(task_id, std::move(task_info)); + library.register_task(task_id, std::move(task_info)); } template diff --git a/src/core/task/task_context.cc b/src/core/task/task_context.cc new file mode 100644 index 0000000000..2fb5405a9e --- /dev/null +++ b/src/core/task/task_context.cc @@ -0,0 +1,53 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/task/task_context.h" +#include "core/task/detail/task_context.h" + +namespace legate { + +std::vector& TaskContext::inputs() { return impl_->inputs(); } + +std::vector& TaskContext::outputs() { return impl_->outputs(); } + +std::vector& TaskContext::reductions() { return impl_->reductions(); } + +std::vector& TaskContext::scalars() { return impl_->scalars(); } + +std::vector& TaskContext::communicators() { return impl_->communicators(); } + +bool TaskContext::is_single_task() const { return impl_->is_single_task(); } + +bool TaskContext::can_raise_exception() const { return impl_->can_raise_exception(); } + +DomainPoint TaskContext::get_task_index() const { return impl_->get_task_index(); } + +Domain TaskContext::get_launch_domain() const { return impl_->get_launch_domain(); } + +mapping::Machine TaskContext::machine() const { return mapping::Machine(impl_->machine()); } + +const std::string& TaskContext::get_provenance() const { return impl_->get_provenance(); } + +TaskContext::TaskContext(detail::TaskContext* impl) : impl_(impl) {} + +// No need to delete impl_, as it's owned by the caller +TaskContext::~TaskContext() {} + +TaskContext::TaskContext(const TaskContext&) = default; + +TaskContext& TaskContext::operator=(const TaskContext&) = default; + +} // namespace legate diff --git a/src/core/task/task_context.h b/src/core/task/task_context.h new file mode 100644 index 0000000000..b18c4557f9 --- /dev/null +++ b/src/core/task/task_context.h @@ -0,0 +1,132 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include "core/comm/communicator.h" +#include "core/data/scalar.h" +#include "core/data/store.h" +#include "core/mapping/machine.h" + +/** + * @file + * @brief Class definition for legate::TaskContext + */ + +namespace legate::detail { +class TaskContext; +} // namespace legate::detail + +namespace legate { + +class Store; +class Scalar; + +/** + * @ingroup task + * @brief A task context that contains task arguments and communicators + */ +class TaskContext { + public: + /** + * @brief Returns input stores of the task + * + * @return Vector of input stores + */ + std::vector& inputs(); + /** + * @brief Returns output stores of the task + * + * @return Vector of output stores + */ + std::vector& outputs(); + /** + * @brief Returns reduction stores of the task + * + * @return Vector of reduction stores + */ + std::vector& reductions(); + /** + * @brief Returns by-value arguments of the task + * + * @return Vector of scalar objects + */ + std::vector& scalars(); + /** + * @brief Returns communicators of the task + * + * If a task launch ends up emitting only a single point task, that task will not get passed a + * communicator, even if one was requested at task launching time. Therefore, most tasks using + * communicators should be prepared to handle the case where the returned vector is empty. + * + * @return Vector of communicator objects + */ + std::vector& communicators(); + + public: + /** + * @brief Indicates whether the task is parallelized + * + * @return true The task is a single task + * @return false The task is one in a set of multiple parallel tasks + */ + bool is_single_task() const; + /** + * @brief Indicates whether the task is allowed to raise an exception + * + * @return true The task can raise an exception + * @return false The task must not raise an exception + */ + bool can_raise_exception() const; + /** + * @brief Returns the point of the task. A 0D point will be returned for a single task. + * + * @return The point of the task + */ + DomainPoint get_task_index() const; + /** + * @brief Returns the task group's launch domain. A single task returns an empty domain + * + * @return The task group's launch domain + */ + Domain get_launch_domain() const; + + public: + mapping::Machine machine() const; + + public: + const std::string& get_provenance() const; + + public: + TaskContext(detail::TaskContext* impl); + ~TaskContext(); + + public: + TaskContext(const TaskContext&); + TaskContext& operator=(const TaskContext&); + + private: + TaskContext(TaskContext&&) = delete; + TaskContext& operator=(TaskContext&&) = delete; + + public: + detail::TaskContext* impl() const { return impl_; } + + private: + detail::TaskContext* impl_{nullptr}; +}; + +} // namespace legate diff --git a/src/core/task/task_info.cc b/src/core/task/task_info.cc index c00af17d1c..5f9eb9fa5f 100644 --- a/src/core/task/task_info.cc +++ b/src/core/task/task_info.cc @@ -31,31 +31,58 @@ const Processor::Kind VARIANT_PROC_KINDS[] = { } // namespace -TaskInfo::TaskInfo(const std::string& task_name) : task_name_(task_name) {} +class TaskInfo::Impl { + public: + Impl(const std::string& task_name); -void TaskInfo::add_variant(LegateVariantCode vid, - VariantImpl body, - const Legion::CodeDescriptor& code_desc, - const VariantOptions& options) + public: + const std::string& name() const { return task_name_; } + + public: + void add_variant(LegateVariantCode vid, + VariantImpl body, + const Legion::CodeDescriptor& code_desc, + const VariantOptions& options); + const VariantInfo& find_variant(LegateVariantCode vid) const; + bool has_variant(LegateVariantCode vid) const; + + public: + void register_task(Legion::TaskID task_id); + + public: + const std::map& variants() const { return variants_; } + + private: + std::string task_name_; + std::map variants_{}; +}; + +TaskInfo::Impl::Impl(const std::string& task_name) : task_name_(task_name) {} + +const std::string& TaskInfo::name() const { return impl_->name(); } + +void TaskInfo::Impl::add_variant(LegateVariantCode vid, + VariantImpl body, + const Legion::CodeDescriptor& code_desc, + const VariantOptions& options) { -#ifdef DEBUG_LEGATE - assert(variants_.find(vid) == variants_.end()); -#endif + if (variants_.find(vid) != variants_.end()) + throw std::invalid_argument("Task " + task_name_ + " already has variant " + + std::to_string(vid)); variants_.emplace(std::make_pair(vid, VariantInfo{body, code_desc, options})); } -const VariantInfo* TaskInfo::find_variant(LegateVariantCode vid) const +const VariantInfo& TaskInfo::Impl::find_variant(LegateVariantCode vid) const { - auto finder = variants_.find(vid); - return finder != variants_.end() ? &finder->second : nullptr; + return variants_.at(vid); } -bool TaskInfo::has_variant(LegateVariantCode vid) const +bool TaskInfo::Impl::has_variant(LegateVariantCode vid) const { return variants_.find(vid) != variants_.end(); } -void TaskInfo::register_task(Legion::TaskID task_id) +void TaskInfo::Impl::register_task(Legion::TaskID task_id) { auto runtime = Legion::Runtime::get_runtime(); runtime->attach_name(task_id, task_name_.c_str(), false /*mutable*/, true /*local_only*/); @@ -68,6 +95,25 @@ void TaskInfo::register_task(Legion::TaskID task_id) } } +TaskInfo::TaskInfo(const std::string& task_name) : impl_(new Impl(task_name)) {} + +void TaskInfo::add_variant(LegateVariantCode vid, + VariantImpl body, + const Legion::CodeDescriptor& code_desc, + const VariantOptions& options) +{ + impl_->add_variant(vid, body, code_desc, options); +} + +const VariantInfo& TaskInfo::find_variant(LegateVariantCode vid) const +{ + return impl_->find_variant(vid); +} + +bool TaskInfo::has_variant(LegateVariantCode vid) const { return impl_->has_variant(vid); } + +void TaskInfo::register_task(Legion::TaskID task_id) { return impl_->register_task(task_id); } + std::ostream& operator<<(std::ostream& os, const VariantInfo& info) { std::stringstream ss; @@ -80,7 +126,8 @@ std::ostream& operator<<(std::ostream& os, const TaskInfo& info) { std::stringstream ss; ss << info.name() << " {"; - for (auto [vid, vinfo] : info.variants_) ss << VARIANT_NAMES[vid] << ":[" << vinfo << "],"; + for (auto [vid, vinfo] : info.impl_->variants()) + ss << VARIANT_NAMES[vid] << ":[" << vinfo << "],"; ss << "}"; os << std::move(ss).str(); return os; diff --git a/src/core/task/task_info.h b/src/core/task/task_info.h index d8ce309248..96170e4ce5 100644 --- a/src/core/task/task_info.h +++ b/src/core/task/task_info.h @@ -32,14 +32,14 @@ class TaskInfo { TaskInfo(const std::string& task_name); public: - const std::string& name() const { return task_name_; } + const std::string& name() const; public: void add_variant(LegateVariantCode vid, VariantImpl body, const Legion::CodeDescriptor& code_desc, const VariantOptions& options); - const VariantInfo* find_variant(LegateVariantCode vid) const; + const VariantInfo& find_variant(LegateVariantCode vid) const; bool has_variant(LegateVariantCode vid) const; public: @@ -47,8 +47,16 @@ class TaskInfo { private: friend std::ostream& operator<<(std::ostream& os, const TaskInfo& info); - std::string task_name_; - std::map variants_; + + private: + TaskInfo(const TaskInfo&) = delete; + TaskInfo& operator=(const TaskInfo&) = delete; + TaskInfo(TaskInfo&&) = delete; + TaskInfo& operator=(TaskInfo&&) = delete; + + private: + class Impl; + Impl* impl_; }; } // namespace legate diff --git a/src/core/type/detail/type_info.cc b/src/core/type/detail/type_info.cc new file mode 100644 index 0000000000..3b90d9d1b9 --- /dev/null +++ b/src/core/type/detail/type_info.cc @@ -0,0 +1,289 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include + +#include "core/runtime/detail/runtime.h" +#include "core/type/detail/type_info.h" +#include "core/utilities/detail/buffer_builder.h" + +namespace legate::detail { + +namespace { + +const std::unordered_map SIZEOF = { + {Type::Code::BOOL, sizeof(legate_type_of)}, + {Type::Code::INT8, sizeof(legate_type_of)}, + {Type::Code::INT16, sizeof(legate_type_of)}, + {Type::Code::INT32, sizeof(legate_type_of)}, + {Type::Code::INT64, sizeof(legate_type_of)}, + {Type::Code::UINT8, sizeof(legate_type_of)}, + {Type::Code::UINT16, sizeof(legate_type_of)}, + {Type::Code::UINT32, sizeof(legate_type_of)}, + {Type::Code::UINT64, sizeof(legate_type_of)}, + {Type::Code::FLOAT16, sizeof(legate_type_of)}, + {Type::Code::FLOAT32, sizeof(legate_type_of)}, + {Type::Code::FLOAT64, sizeof(legate_type_of)}, + {Type::Code::COMPLEX64, sizeof(legate_type_of)}, + {Type::Code::COMPLEX128, sizeof(legate_type_of)}, +}; + +const std::unordered_map TYPE_NAMES = { + {Type::Code::BOOL, "bool"}, + {Type::Code::INT8, "int8"}, + {Type::Code::INT16, "int16"}, + {Type::Code::INT32, "int32"}, + {Type::Code::INT64, "int64"}, + {Type::Code::UINT8, "uint8"}, + {Type::Code::UINT16, "uint16"}, + {Type::Code::UINT32, "uint32"}, + {Type::Code::UINT64, "uint64"}, + {Type::Code::FLOAT16, "float16"}, + {Type::Code::FLOAT32, "float32"}, + {Type::Code::FLOAT64, "float64"}, + {Type::Code::COMPLEX64, "complex64"}, + {Type::Code::COMPLEX128, "complex128"}, + {Type::Code::STRING, "string"}, +}; + +const char* _VARIABLE_SIZE_ERROR_MESSAGE = "Variable-size element type cannot be used"; + +} // namespace + +Type::Type(Code c) : code(c) {} + +const FixedArrayType& Type::as_fixed_array_type() const +{ + throw std::invalid_argument("Type is not a fixed array type"); + return *static_cast(nullptr); +} + +const StructType& Type::as_struct_type() const +{ + throw std::invalid_argument("Type is not a struct type"); + return *static_cast(nullptr); +} + +void Type::record_reduction_operator(int32_t op_kind, int32_t global_op_id) const +{ + detail::Runtime::get_runtime()->record_reduction_operator(uid(), op_kind, global_op_id); +} + +int32_t Type::find_reduction_operator(int32_t op_kind) const +{ + return detail::Runtime::get_runtime()->find_reduction_operator(uid(), op_kind); +} + +int32_t Type::find_reduction_operator(ReductionOpKind op_kind) const +{ + return find_reduction_operator(static_cast(op_kind)); +} + +bool Type::operator==(const Type& other) const { return equal(other); } + +PrimitiveType::PrimitiveType(Code code) : Type(code), size_(SIZEOF.at(code)) {} + +int32_t PrimitiveType::uid() const { return static_cast(code); } + +std::string PrimitiveType::to_string() const { return TYPE_NAMES.at(code); } + +void PrimitiveType::pack(BufferBuilder& buffer) const +{ + buffer.pack(static_cast(code)); +} + +bool PrimitiveType::equal(const Type& other) const { return code == other.code; } + +ExtensionType::ExtensionType(int32_t uid, Type::Code code) : Type(code), uid_(uid) {} + +FixedArrayType::FixedArrayType(int32_t uid, + std::shared_ptr element_type, + uint32_t N) noexcept(false) + : ExtensionType(uid, Type::Code::FIXED_ARRAY), + element_type_(std::move(element_type)), + N_(N), + size_(element_type_->size() * N) +{ + if (element_type_->variable_size()) throw std::invalid_argument(_VARIABLE_SIZE_ERROR_MESSAGE); +} + +std::string FixedArrayType::to_string() const +{ + std::stringstream ss; + ss << element_type_->to_string() << "[" << N_ << "]"; + return std::move(ss).str(); +} + +void FixedArrayType::pack(BufferBuilder& buffer) const +{ + buffer.pack(static_cast(code)); + buffer.pack(uid_); + buffer.pack(N_); + element_type_->pack(buffer); +} + +const FixedArrayType& FixedArrayType::as_fixed_array_type() const { return *this; } + +bool FixedArrayType::equal(const Type& other) const +{ + if (code != other.code) return false; + auto& casted = static_cast(other); + +#ifdef DEBUG_LEGATE + // Do a structural check in debug mode + return uid_ == casted.uid_ && N_ == casted.N_ && element_type_ == casted.element_type_; +#else + // Each type is uniquely identified by the uid, so it's sufficient to compare between uids + return uid_ == casted.uid_; +#endif +} + +StructType::StructType(int32_t uid, + std::vector>&& field_types, + bool align) noexcept(false) + : ExtensionType(uid, Type::Code::STRUCT), + aligned_(align), + alignment_(1), + size_(0), + field_types_(std::move(field_types)) +{ + offsets_.reserve(field_types_.size()); + if (aligned_) { + static constexpr auto align_offset = [](uint32_t offset, uint32_t align) { + return (offset + (align - 1)) & -align; + }; + + for (auto& field_type : field_types_) { + if (field_type->variable_size()) throw std::invalid_argument(_VARIABLE_SIZE_ERROR_MESSAGE); + uint32_t _my_align = field_type->alignment(); + alignment_ = std::max(_my_align, alignment_); + + uint32_t offset = align_offset(size_, _my_align); + offsets_.push_back(offset); + size_ = offset + field_type->size(); + } + size_ = align_offset(size_, alignment_); + } else { + for (auto& field_type : field_types_) { + if (field_type->variable_size()) throw std::invalid_argument(_VARIABLE_SIZE_ERROR_MESSAGE); + offsets_.push_back(size_); + size_ += field_type->size(); + } + } +} + +std::string StructType::to_string() const +{ + std::stringstream ss; + ss << "{"; + for (uint32_t idx = 0; idx < field_types_.size(); ++idx) { + if (idx > 0) ss << ","; + ss << field_types_.at(idx)->to_string() << ":" << offsets_.at(idx); + } + ss << "}"; + return std::move(ss).str(); +} + +void StructType::pack(BufferBuilder& buffer) const +{ + buffer.pack(static_cast(code)); + buffer.pack(uid_); + buffer.pack(field_types_.size()); + for (auto& field_type : field_types_) field_type->pack(buffer); + buffer.pack(aligned_); +} + +const StructType& StructType::as_struct_type() const { return *this; } + +bool StructType::equal(const Type& other) const +{ + if (code != other.code) return false; + auto& casted = static_cast(other); + +#ifdef DEBUG_LEGATE + // Do a structural check in debug mode + if (uid_ != casted.uid_) return false; + uint32_t nf = num_fields(); + if (nf != casted.num_fields()) return false; + for (uint32_t idx = 0; idx < nf; ++idx) + if (field_type(idx) != casted.field_type(idx)) return false; + return true; +#else + // Each type is uniquely identified by the uid, so it's sufficient to compare between uids + return uid_ == casted.uid_; +#endif +} + +std::shared_ptr StructType::field_type(uint32_t field_idx) const +{ + return field_types_.at(field_idx); +} + +StringType::StringType() : Type(Type::Code::STRING) {} + +int32_t StringType::uid() const { return static_cast(code); } + +std::string StringType::to_string() const { return "string"; } + +void StringType::pack(BufferBuilder& buffer) const +{ + buffer.pack(static_cast(code)); +} + +bool StringType::equal(const Type& other) const { return code == other.code; } + +std::shared_ptr primitive_type(Type::Code code) +{ + return std::make_shared(code); +} + +std::shared_ptr string_type() +{ + static auto type = std::make_shared(); + return type; +} + +std::shared_ptr fixed_array_type(std::shared_ptr element_type, + uint32_t N) noexcept(false) +{ + // We use UIDs of the following format for "common" fixed array types + // 1B 1B + // +--------+-------------------+ + // | length | element type code | + // +--------+-------------------+ + int32_t uid = [&N](const Type& elem_type) { + if (!elem_type.is_primitive() || N > 0xFFU) return Runtime::get_runtime()->get_type_uid(); + return static_cast(elem_type.code) | N << 8; + }(*element_type); + return std::make_shared(uid, std::move(element_type), N); +} + +std::shared_ptr struct_type(const std::vector>& field_types, + bool align) noexcept(false) +{ + return std::make_shared( + Runtime::get_runtime()->get_type_uid(), std::vector>(field_types), align); +} + +std::shared_ptr struct_type(std::vector>&& field_types, + bool align) noexcept(false) +{ + return std::make_shared( + Runtime::get_runtime()->get_type_uid(), std::move(field_types), align); +} + +} // namespace legate::detail diff --git a/src/core/type/detail/type_info.h b/src/core/type/detail/type_info.h new file mode 100644 index 0000000000..ad92349849 --- /dev/null +++ b/src/core/type/detail/type_info.h @@ -0,0 +1,162 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include "core/type/type_info.h" + +namespace legate::detail { + +class BufferBuilder; +class FixedArrayType; +class StructType; + +class Type { + public: + using Code = legate::Type::Code; + + public: + Type(Code code); + + public: + virtual ~Type() {} + virtual uint32_t size() const = 0; + virtual uint32_t alignment() const = 0; + virtual int32_t uid() const = 0; + virtual bool variable_size() const = 0; + virtual std::string to_string() const = 0; + virtual bool is_primitive() const = 0; + virtual void pack(BufferBuilder& buffer) const = 0; + virtual const FixedArrayType& as_fixed_array_type() const; + virtual const StructType& as_struct_type() const; + virtual bool equal(const Type& other) const = 0; + + public: + void record_reduction_operator(int32_t op_kind, int32_t global_op_id) const; + int32_t find_reduction_operator(int32_t op_kind) const; + int32_t find_reduction_operator(ReductionOpKind op_kind) const; + bool operator==(const Type& other) const; + bool operator!=(const Type& other) const { return !operator==(other); } + + const Code code; +}; + +class PrimitiveType : public Type { + public: + PrimitiveType(Code code); + uint32_t size() const override { return size_; } + uint32_t alignment() const override { return size_; } + int32_t uid() const override; + bool variable_size() const override { return false; } + std::string to_string() const override; + bool is_primitive() const override { return true; } + void pack(BufferBuilder& buffer) const override; + + private: + bool equal(const Type& other) const override; + + private: + const uint32_t size_; +}; + +class StringType : public Type { + public: + StringType(); + bool variable_size() const override { return true; } + uint32_t size() const override { return 0; } + uint32_t alignment() const override { return 0; } + int32_t uid() const override; + std::string to_string() const override; + bool is_primitive() const override { return false; } + void pack(BufferBuilder& buffer) const override; + + private: + bool equal(const Type& other) const override; +}; + +class ExtensionType : public Type { + public: + ExtensionType(int32_t uid, Type::Code code); + int32_t uid() const override { return uid_; } + bool is_primitive() const override { return false; } + + protected: + const uint32_t uid_; +}; + +class FixedArrayType : public ExtensionType { + public: + FixedArrayType(int32_t uid, std::shared_ptr element_type, uint32_t N) noexcept(false); + uint32_t size() const override { return size_; } + uint32_t alignment() const override { return element_type_->alignment(); } + bool variable_size() const override { return false; } + std::string to_string() const override; + void pack(BufferBuilder& buffer) const override; + const FixedArrayType& as_fixed_array_type() const override; + + uint32_t num_elements() const { return N_; } + std::shared_ptr element_type() const { return element_type_; } + + private: + bool equal(const Type& other) const override; + + private: + const std::shared_ptr element_type_; + const uint32_t N_; + const uint32_t size_; +}; + +class StructType : public ExtensionType { + public: + StructType(int32_t uid, + std::vector>&& field_types, + bool align = false) noexcept(false); + uint32_t size() const override { return size_; } + uint32_t alignment() const override { return alignment_; } + bool variable_size() const override { return false; } + std::string to_string() const override; + void pack(BufferBuilder& buffer) const override; + const StructType& as_struct_type() const override; + + uint32_t num_fields() const { return field_types_.size(); } + std::shared_ptr field_type(uint32_t field_idx) const; + bool aligned() const { return aligned_; } + + private: + bool equal(const Type& other) const override; + + private: + bool aligned_; + uint32_t size_; + uint32_t alignment_; + std::vector> field_types_{}; + std::vector offsets_{}; +}; + +std::shared_ptr primitive_type(Type::Code code); + +std::shared_ptr string_type(); + +std::shared_ptr fixed_array_type(std::shared_ptr element_type, + uint32_t N) noexcept(false); + +std::shared_ptr struct_type(const std::vector>& field_types, + bool align) noexcept(false); + +std::shared_ptr struct_type(std::vector>&& field_types, + bool align) noexcept(false); + +} // namespace legate::detail diff --git a/src/core/type/type_info.cc b/src/core/type/type_info.cc index e815324ec2..667679ed7d 100644 --- a/src/core/type/type_info.cc +++ b/src/core/type/type_info.cc @@ -14,324 +14,205 @@ * */ -#include -#include - -#include "core/runtime/detail/runtime.h" #include "core/type/type_info.h" -#include "core/type/type_traits.h" -#include "core/utilities/buffer_builder.h" +#include "core/runtime/detail/runtime.h" +#include "core/type/detail/type_info.h" namespace legate { -namespace { +Type::Code Type::code() const { return impl_->code; } -const std::unordered_map SIZEOF = { - {Type::Code::BOOL, sizeof(legate_type_of)}, - {Type::Code::INT8, sizeof(legate_type_of)}, - {Type::Code::INT16, sizeof(legate_type_of)}, - {Type::Code::INT32, sizeof(legate_type_of)}, - {Type::Code::INT64, sizeof(legate_type_of)}, - {Type::Code::UINT8, sizeof(legate_type_of)}, - {Type::Code::UINT16, sizeof(legate_type_of)}, - {Type::Code::UINT32, sizeof(legate_type_of)}, - {Type::Code::UINT64, sizeof(legate_type_of)}, - {Type::Code::FLOAT16, sizeof(legate_type_of)}, - {Type::Code::FLOAT32, sizeof(legate_type_of)}, - {Type::Code::FLOAT64, sizeof(legate_type_of)}, - {Type::Code::COMPLEX64, sizeof(legate_type_of)}, - {Type::Code::COMPLEX128, sizeof(legate_type_of)}, -}; - -const std::unordered_map TYPE_NAMES = { - {Type::Code::BOOL, "bool"}, - {Type::Code::INT8, "int8"}, - {Type::Code::INT16, "int16"}, - {Type::Code::INT32, "int32"}, - {Type::Code::INT64, "int64"}, - {Type::Code::UINT8, "uint8"}, - {Type::Code::UINT16, "uint16"}, - {Type::Code::UINT32, "uint32"}, - {Type::Code::UINT64, "uint64"}, - {Type::Code::FLOAT16, "float16"}, - {Type::Code::FLOAT32, "float32"}, - {Type::Code::FLOAT64, "float64"}, - {Type::Code::COMPLEX64, "complex64"}, - {Type::Code::COMPLEX128, "complex128"}, - {Type::Code::STRING, "string"}, -}; - -const char* _VARIABLE_SIZE_ERROR_MESSAGE = "Variable-size element type cannot be used"; +uint32_t Type::size() const { return impl_->size(); } -} // namespace +uint32_t Type::alignment() const { return impl_->alignment(); } -Type::Type(Code c) : code(c) {} +int32_t Type::uid() const { return impl_->uid(); } -const FixedArrayType& Type::as_fixed_array_type() const +bool Type::variable_size() const { return impl_->variable_size(); } + +std::string Type::to_string() const { return impl_->to_string(); } + +bool Type::is_primitive() const { return impl_->is_primitive(); } + +FixedArrayType Type::as_fixed_array_type() const { - throw std::invalid_argument("Type is not a fixed array type"); - return *static_cast(nullptr); + if (impl_->code != Code::FIXED_ARRAY) { + throw std::invalid_argument("Type is not a fixed array type"); + } + return FixedArrayType(impl_); } -const StructType& Type::as_struct_type() const +StructType Type::as_struct_type() const { - throw std::invalid_argument("Type is not a struct type"); - return *static_cast(nullptr); + if (impl_->code != Code::STRUCT) { throw std::invalid_argument("Type is not a struct type"); } + return StructType(impl_); } void Type::record_reduction_operator(int32_t op_kind, int32_t global_op_id) const { - detail::Runtime::get_runtime()->record_reduction_operator(uid(), op_kind, global_op_id); + impl_->record_reduction_operator(op_kind, global_op_id); } int32_t Type::find_reduction_operator(int32_t op_kind) const { - return detail::Runtime::get_runtime()->find_reduction_operator(uid(), op_kind); + return impl_->find_reduction_operator(op_kind); } int32_t Type::find_reduction_operator(ReductionOpKind op_kind) const { - return find_reduction_operator(static_cast(op_kind)); + return impl_->find_reduction_operator(op_kind); } -bool Type::operator==(const Type& other) const { return equal(other); } +bool Type::operator==(const Type& other) const { return impl_->equal(*other.impl_); } -PrimitiveType::PrimitiveType(Code code) : Type(code), size_(SIZEOF.at(code)) {} +bool Type::operator!=(const Type& other) const { return !operator==(other); } -int32_t PrimitiveType::uid() const { return static_cast(code); } +Type::Type() {} -std::unique_ptr PrimitiveType::clone() const { return std::make_unique(code); } +Type::Type(std::shared_ptr impl) : impl_(std::move(impl)) {} -std::string PrimitiveType::to_string() const { return TYPE_NAMES.at(code); } +Type::~Type() {} -void PrimitiveType::pack(BufferBuilder& buffer) const +uint32_t FixedArrayType::num_elements() const { - buffer.pack(static_cast(code)); + return static_cast(impl_.get())->num_elements(); } -bool PrimitiveType::equal(const Type& other) const { return code == other.code; } - -ExtensionType::ExtensionType(int32_t uid, Type::Code code) : Type(code), uid_(uid) {} - -FixedArrayType::FixedArrayType(int32_t uid, - std::unique_ptr element_type, - uint32_t N) noexcept(false) - : ExtensionType(uid, Type::Code::FIXED_ARRAY), - element_type_(std::move(element_type)), - N_(N), - size_(element_type_->size() * N) +Type FixedArrayType::element_type() const { - if (element_type_->variable_size()) throw std::invalid_argument(_VARIABLE_SIZE_ERROR_MESSAGE); + return Type(static_cast(impl_.get())->element_type()); } -std::unique_ptr FixedArrayType::clone() const +FixedArrayType::FixedArrayType(std::shared_ptr type) : Type(std::move(type)) {} + +uint32_t StructType::num_fields() const { - return std::make_unique(uid_, element_type_->clone(), N_); + return static_cast(impl_.get())->num_fields(); } -std::string FixedArrayType::to_string() const +Type StructType::field_type(uint32_t field_idx) const { - std::stringstream ss; - ss << element_type_->to_string() << "[" << N_ << "]"; - return std::move(ss).str(); + return Type(static_cast(impl_.get())->field_type(field_idx)); } -void FixedArrayType::pack(BufferBuilder& buffer) const +bool StructType::aligned() const { - buffer.pack(static_cast(code)); - buffer.pack(uid_); - buffer.pack(N_); - element_type_->pack(buffer); + return static_cast(impl_.get())->aligned(); } -const FixedArrayType& FixedArrayType::as_fixed_array_type() const { return *this; } +StructType::StructType(std::shared_ptr type) : Type(std::move(type)) {} -bool FixedArrayType::equal(const Type& other) const -{ - if (code != other.code) return false; - auto& casted = static_cast(other); +Type primitive_type(Type::Code code) { return Type(detail::primitive_type(code)); } -#ifdef DEBUG_LEGATE - // Do a structural check in debug mode - return uid_ == casted.uid_ && N_ == casted.N_ && element_type_ == casted.element_type_; -#else - // Each type is uniquely identified by the uid, so it's sufficient to compare between uids - return uid_ == casted.uid_; -#endif -} +Type string_type() { return Type(detail::string_type()); } -StructType::StructType(int32_t uid, - std::vector>&& field_types, - bool align) noexcept(false) - : ExtensionType(uid, Type::Code::STRUCT), - aligned_(align), - alignment_(1), - size_(0), - field_types_(std::move(field_types)) +Type fixed_array_type(const Type& element_type, uint32_t N) noexcept(false) { - offsets_.reserve(field_types_.size()); - if (aligned_) { - static constexpr auto align_offset = [](uint32_t offset, uint32_t align) { - return (offset + (align - 1)) & -align; - }; - - for (auto& field_type : field_types_) { - if (field_type->variable_size()) throw std::invalid_argument(_VARIABLE_SIZE_ERROR_MESSAGE); - uint32_t _my_align = field_type->alignment(); - alignment_ = std::max(_my_align, alignment_); - - uint32_t offset = align_offset(size_, _my_align); - offsets_.push_back(offset); - size_ = offset + field_type->size(); - } - size_ = align_offset(size_, alignment_); - } else { - for (auto& field_type : field_types_) { - if (field_type->variable_size()) throw std::invalid_argument(_VARIABLE_SIZE_ERROR_MESSAGE); - offsets_.push_back(size_); - size_ += field_type->size(); - } - } + return Type(detail::fixed_array_type(element_type.impl(), N)); } -std::unique_ptr StructType::clone() const +Type struct_type(const std::vector& field_types, bool align) noexcept(false) { - std::vector> field_types; - for (auto& field_type : field_types_) field_types.push_back(field_type->clone()); - return std::make_unique(uid_, std::move(field_types), aligned_); + std::vector> detail_field_types; + for (const auto& field_type : field_types) { detail_field_types.push_back(field_type.impl()); } + return Type(detail::struct_type(std::move(detail_field_types), align)); } -std::string StructType::to_string() const +std::ostream& operator<<(std::ostream& ostream, const Type::Code& code) { - std::stringstream ss; - ss << "{"; - for (uint32_t idx = 0; idx < field_types_.size(); ++idx) { - if (idx > 0) ss << ","; - ss << field_types_.at(idx)->to_string() << ":" << offsets_.at(idx); - } - ss << "}"; - return std::move(ss).str(); + ostream << static_cast(code); + return ostream; } -void StructType::pack(BufferBuilder& buffer) const +std::ostream& operator<<(std::ostream& ostream, const Type& type) { - buffer.pack(static_cast(code)); - buffer.pack(uid_); - buffer.pack(field_types_.size()); - for (auto& field_type : field_types_) field_type->pack(buffer); - buffer.pack(aligned_); + ostream << type.to_string(); + return ostream; } -const StructType& StructType::as_struct_type() const { return *this; } - -bool StructType::equal(const Type& other) const +Type bool_() { - if (code != other.code) return false; - auto& casted = static_cast(other); - -#ifdef DEBUG_LEGATE - // Do a structural check in debug mode - if (uid_ != casted.uid_) return false; - uint32_t nf = num_fields(); - if (nf != casted.num_fields()) return false; - for (uint32_t idx = 0; idx < nf; ++idx) - if (field_type(idx) != casted.field_type(idx)) return false; - return true; -#else - // Each type is uniquely identified by the uid, so it's sufficient to compare between uids - return uid_ == casted.uid_; -#endif + static auto result = primitive_type(Type::Code::BOOL); + return result; } -const Type& StructType::field_type(uint32_t field_idx) const { return *field_types_.at(field_idx); } - -StringType::StringType() : Type(Type::Code::STRING) {} - -int32_t StringType::uid() const { return static_cast(code); } - -std::unique_ptr StringType::clone() const { return string_type(); } - -std::string StringType::to_string() const { return "string"; } - -void StringType::pack(BufferBuilder& buffer) const +Type int8() { - buffer.pack(static_cast(code)); + static auto result = primitive_type(Type::Code::INT8); + return result; } -bool StringType::equal(const Type& other) const { return code == other.code; } - -std::unique_ptr primitive_type(Type::Code code) +Type int16() { - return std::make_unique(code); + static auto result = primitive_type(Type::Code::INT16); + return result; } -std::unique_ptr string_type() { return std::make_unique(); } - -std::unique_ptr fixed_array_type(std::unique_ptr element_type, - uint32_t N) noexcept(false) +Type int32() { - // We use UIDs of the following format for "common" fixed array types - // 1B 1B - // +--------+-------------------+ - // | length | element type code | - // +--------+-------------------+ - auto generate_uid = [](const Type& elem_type, uint32_t N) { - if (!elem_type.is_primitive() || N > 0xFFU) - return detail::Runtime::get_runtime()->get_type_uid(); - return static_cast(elem_type.code) | N << 8; - }; - int32_t uid = generate_uid(*element_type, N); - return std::make_unique(uid, std::move(element_type), N); + static auto result = primitive_type(Type::Code::INT32); + return result; } -std::unique_ptr struct_type(std::vector>&& field_types, - bool align) noexcept(false) +Type int64() { - return std::make_unique( - detail::Runtime::get_runtime()->get_type_uid(), std::move(field_types), align); + static auto result = primitive_type(Type::Code::INT64); + return result; } -std::ostream& operator<<(std::ostream& ostream, const Type::Code& code) +Type uint8() { - ostream << static_cast(code); - return ostream; + static auto result = primitive_type(Type::Code::UINT8); + return result; } -std::ostream& operator<<(std::ostream& ostream, const Type& type) +Type uint16() { - ostream << type.to_string(); - return ostream; + static auto result = primitive_type(Type::Code::UINT16); + return result; } -std::unique_ptr bool_() { return primitive_type(Type::Code::BOOL); } - -std::unique_ptr int8() { return primitive_type(Type::Code::INT8); } - -std::unique_ptr int16() { return primitive_type(Type::Code::INT16); } - -std::unique_ptr int32() { return primitive_type(Type::Code::INT32); } - -std::unique_ptr int64() { return primitive_type(Type::Code::INT64); } - -std::unique_ptr uint8() { return primitive_type(Type::Code::UINT8); } - -std::unique_ptr uint16() { return primitive_type(Type::Code::UINT16); } - -std::unique_ptr uint32() { return primitive_type(Type::Code::UINT32); } - -std::unique_ptr uint64() { return primitive_type(Type::Code::UINT64); } +Type uint32() +{ + static auto result = primitive_type(Type::Code::UINT32); + return result; +} -std::unique_ptr float16() { return primitive_type(Type::Code::FLOAT16); } +Type uint64() +{ + static auto result = primitive_type(Type::Code::UINT64); + return result; +} -std::unique_ptr float32() { return primitive_type(Type::Code::FLOAT32); } +Type float16() +{ + static auto result = primitive_type(Type::Code::FLOAT16); + return result; +} -std::unique_ptr float64() { return primitive_type(Type::Code::FLOAT64); } +Type float32() +{ + static auto result = primitive_type(Type::Code::FLOAT32); + return result; +} -std::unique_ptr complex64() { return primitive_type(Type::Code::COMPLEX64); } +Type float64() +{ + static auto result = primitive_type(Type::Code::FLOAT64); + return result; +} -std::unique_ptr complex128() { return primitive_type(Type::Code::COMPLEX128); } +Type complex64() +{ + static auto result = primitive_type(Type::Code::COMPLEX64); + return result; +} -std::unique_ptr string() { return string_type(); } +Type complex128() +{ + static auto result = primitive_type(Type::Code::COMPLEX128); + return result; +} namespace { @@ -340,27 +221,35 @@ constexpr int32_t RECT_UID_BASE = POINT_UID_BASE + LEGATE_MAX_DIM + 1; } // namespace -std::unique_ptr point_type(int32_t ndim) +Type point_type(int32_t ndim) { - return std::make_unique(POINT_UID_BASE + ndim, int64(), ndim); + static Type cache[LEGATE_MAX_DIM + 1]; + + if (ndim <= 0 || ndim > LEGATE_MAX_DIM) + throw std::out_of_range(std::to_string(ndim) + " is not a supported number of dimensions"); + if (cache[ndim].impl() == nullptr) { + cache[ndim] = + Type(std::make_shared(POINT_UID_BASE + ndim, int64().impl(), ndim)); + } + return cache[ndim]; } -std::unique_ptr rect_type(int32_t ndim) +Type rect_type(int32_t ndim) { - std::vector> field_types; - field_types.push_back(point_type(ndim)); - field_types.push_back(point_type(ndim)); - return std::make_unique(RECT_UID_BASE + ndim, std::move(field_types), true /*align*/); + std::vector> field_types{point_type(ndim).impl(), + point_type(ndim).impl()}; + return Type(std::make_shared( + RECT_UID_BASE + ndim, std::move(field_types), true /*align*/)); } bool is_point_type(const Type& type, int32_t ndim) { - switch (type.code) { + switch (type.code()) { case Type::Code::INT64: { return 1 == ndim; } case Type::Code::FIXED_ARRAY: { - return static_cast(type).num_elements() == ndim; + return type.as_fixed_array_type().num_elements() == ndim; } default: { return false; @@ -371,8 +260,8 @@ bool is_point_type(const Type& type, int32_t ndim) bool is_rect_type(const Type& type, int32_t ndim) { - if (type.code != Type::Code::STRUCT) return false; - auto& st_type = static_cast(type); + if (type.code() != Type::Code::STRUCT) return false; + auto st_type = type.as_struct_type(); return st_type.num_fields() == 2 && is_point_type(st_type.field_type(0), ndim) && is_point_type(st_type.field_type(1), ndim); } diff --git a/src/core/type/type_info.h b/src/core/type/type_info.h index abb6baea46..4dcfa7b3d8 100644 --- a/src/core/type/type_info.h +++ b/src/core/type/type_info.h @@ -24,9 +24,12 @@ /** @defgroup types Type system */ +namespace legate::detail { +class Type; +} // namespace legate::detail + namespace legate { -class BufferBuilder; class FixedArrayType; class StructType; @@ -77,71 +80,51 @@ class Type { INVALID = INVALID_LT, /*!< Invalid type */ }; - protected: - Type(Code code); - - virtual bool equal(const Type& other) const = 0; - public: - virtual ~Type() {} - + /** + * @brief Code of the type + * + * @return Type code + */ + Code code() const; /** * @brief Size of the data type in bytes * * @return Data type size in bytes */ - virtual uint32_t size() const = 0; - + uint32_t size() const; /** * @brief Alignment of the type * * @return Alignment in bytes */ - virtual uint32_t alignment() const = 0; - + uint32_t alignment() const; /** * @brief Unique ID of the data type * * @return Unique ID */ - virtual int32_t uid() const = 0; - + int32_t uid() const; /** * @brief Inidicates whether the data type is of varible size elements * * @return true Elements can be variable size * @return false Elements have fixed size */ - virtual bool variable_size() const = 0; - - /** - * @brief Returns a copy of the data type - * - * @return A copy of the data type - */ - virtual std::unique_ptr clone() const = 0; - + bool variable_size() const; /** * @brief Converts the data type into a string * * @return A string of the data type */ - virtual std::string to_string() const = 0; - + std::string to_string() const; /** * @brief Indicates whether the type is a primitive type * * @return true If the type is a primitive type * @return false Otherwise */ - virtual bool is_primitive() const = 0; - /** - * @brief Serializes the type into a buffer - * - * @param buffer A BufferBuilder object to serialize the type into - */ - virtual void pack(BufferBuilder& buffer) const = 0; - + bool is_primitive() const; /** * @brief Dynamically casts the type into a fixed size array type. * @@ -149,8 +132,7 @@ class Type { * * @return Type object */ - virtual const FixedArrayType& as_fixed_array_type() const; - + FixedArrayType as_fixed_array_type() const; /** * @brief Dynamically casts the type into a struct type. * @@ -158,8 +140,7 @@ class Type { * * @return Type object */ - virtual const StructType& as_struct_type() const; - + StructType as_struct_type() const; /** * @brief Records a reduction operator. * @@ -170,7 +151,6 @@ class Type { * @param global_op_id Global reduction operator ID */ void record_reduction_operator(int32_t op_kind, int32_t global_op_id) const; - /** * @brief Finds the global operator ID for a given reduction operator kind. * @@ -181,7 +161,6 @@ class Type { * @return Global reduction operator ID */ int32_t find_reduction_operator(int32_t op_kind) const; - /** * @brief Finds the global operator ID for a given reduction operator kind. * @@ -192,7 +171,6 @@ class Type { * @return Global reduction operator ID */ int32_t find_reduction_operator(ReductionOpKind op_kind) const; - /** * @brief Equality check between types * @@ -205,148 +183,62 @@ class Type { * @return false Types are different */ bool operator==(const Type& other) const; - bool operator!=(const Type& other) const { return !operator==(other); } - - const Code code; -}; + bool operator!=(const Type& other) const; -/** - * @ingroup types - * @brief A class for primitive data types - */ -class PrimitiveType : public Type { public: - /** - * @brief Constructs a primitive type metadata object - * - * @param code Type code. Must be one of the primitive types. - */ - PrimitiveType(Code code); - uint32_t size() const override { return size_; } - uint32_t alignment() const override { return size_; } - int32_t uid() const override; - bool variable_size() const override { return false; } - std::unique_ptr clone() const override; - std::string to_string() const override; - bool is_primitive() const override { return true; } - void pack(BufferBuilder& buffer) const override; - - private: - bool equal(const Type& other) const override; + Type(); + Type(std::shared_ptr impl); + Type(const Type&) = default; + Type(Type&&) = default; + Type& operator=(const Type&) = default; + Type& operator=(Type&&) = default; - private: - const uint32_t size_; -}; - -/** - * @ingroup types - * @brief String data type - */ -class StringType : public Type { public: - StringType(); - bool variable_size() const override { return true; } - uint32_t size() const override { return 0; } - uint32_t alignment() const override { return 0; } - int32_t uid() const override; - std::unique_ptr clone() const override; - std::string to_string() const override; - bool is_primitive() const override { return false; } - void pack(BufferBuilder& buffer) const override; + virtual ~Type(); - private: - bool equal(const Type& other) const override; -}; - -/** - * @ingroup types - * @brief A class for all extension types. Each extension type expects a unique ID. - */ -class ExtensionType : public Type { public: - ExtensionType(int32_t uid, Type::Code code); - int32_t uid() const override { return uid_; } - bool is_primitive() const override { return false; } + std::shared_ptr impl() const { return impl_; } protected: - const uint32_t uid_; + std::shared_ptr impl_{nullptr}; }; /** * @ingroup types * @brief A class for fixed-size array data types */ -class FixedArrayType : public ExtensionType { +class FixedArrayType : public Type { public: - /** - * @brief Constructs a metadata object for a fixed-size array type - * - * @param uid Unique ID - * @param element_type Type of the array elements - * @param N Size of the array - */ - FixedArrayType(int32_t uid, std::unique_ptr element_type, uint32_t N) noexcept(false); - uint32_t size() const override { return size_; } - uint32_t alignment() const override { return element_type_->alignment(); } - bool variable_size() const override { return false; } - std::unique_ptr clone() const override; - std::string to_string() const override; - void pack(BufferBuilder& buffer) const override; - const FixedArrayType& as_fixed_array_type() const override; - /** * @brief Returns the number of elements * * @return Number of elements */ - uint32_t num_elements() const { return N_; } + uint32_t num_elements() const; /** * @brief Returns the element type * * @return Element type */ - const Type& element_type() const { return *element_type_; } - - private: - bool equal(const Type& other) const override; + Type element_type() const; private: - const std::unique_ptr element_type_; - const uint32_t N_; - const uint32_t size_; + friend class Type; + FixedArrayType(std::shared_ptr type); }; /** * @ingroup types * @brief A class for struct data types */ -class StructType : public ExtensionType { +class StructType : public Type { public: - /** - * @brief Constructs a metadata object for a struct type - * - * @param uid Unique ID - * @param field_types A vector of field types - * @param align Optional boolean flag indicating whether the struct fields should be aligned. - * false by default. - */ - StructType(int32_t uid, - std::vector>&& field_types, - bool align = false) noexcept(false); - uint32_t size() const override { return size_; } - uint32_t alignment() const override { return alignment_; } - bool variable_size() const override { return false; } - std::unique_ptr clone() const override; - std::string to_string() const override; - void pack(BufferBuilder& buffer) const override; - const StructType& as_struct_type() const override; - /** * @brief Returns the number of fields * * @return Number of fields */ - uint32_t num_fields() const { return field_types_.size(); } + uint32_t num_fields() const; /** * @brief Returns the element type * @@ -354,24 +246,18 @@ class StructType : public ExtensionType { * * @return Element type */ - const Type& field_type(uint32_t field_idx) const; + Type field_type(uint32_t field_idx) const; /** * @brief Indiciates whether the fields are aligned * * @return true Fields are aligned * @return false Fields are compact */ - bool aligned() const { return aligned_; } + bool aligned() const; private: - bool equal(const Type& other) const override; - - private: - bool aligned_; - uint32_t size_; - uint32_t alignment_; - std::vector> field_types_{}; - std::vector offsets_{}; + friend class Type; + StructType(std::shared_ptr type); }; /** @@ -382,7 +268,7 @@ class StructType : public ExtensionType { * * @return Type object */ -std::unique_ptr primitive_type(Type::Code code); +Type primitive_type(Type::Code code); /** * @ingroup types @@ -390,7 +276,7 @@ std::unique_ptr primitive_type(Type::Code code); * * @return Type object */ -std::unique_ptr string_type(); +Type string_type(); /** * @ingroup types @@ -401,8 +287,7 @@ std::unique_ptr string_type(); * * @return Type object */ -std::unique_ptr fixed_array_type(std::unique_ptr element_type, - uint32_t N) noexcept(false); +Type fixed_array_type(const Type& element_type, uint32_t N) noexcept(false); /** * @ingroup types @@ -413,8 +298,7 @@ std::unique_ptr fixed_array_type(std::unique_ptr element_type, * * @return Type object */ -std::unique_ptr struct_type(std::vector>&& field_types, - bool align = false) noexcept(false); +Type struct_type(const std::vector& field_types, bool align = false) noexcept(false); /** * @ingroup types @@ -426,15 +310,15 @@ std::unique_ptr struct_type(std::vector>&& field_typ * @return Type object */ template -std::enable_if_t...>, std::unique_ptr> -struct_type(bool align, std::unique_ptr... field_types) noexcept(false) +std::enable_if_t...>, Type> struct_type( + bool align, Types... field_types) noexcept(false) { - std::vector> vec_field_types; - auto move_field_type = [&vec_field_types](auto&& field_type) { - vec_field_types.push_back(std::move(field_type)); + std::vector vec_field_types; + auto copy_field_type = [&vec_field_types](const auto& field_type) { + vec_field_types.push_back(field_type); }; - (move_field_type(std::move(field_types)), ...); - return struct_type(std::move(vec_field_types), align); + (copy_field_type(field_types), ...); + return struct_type(vec_field_types, align); } std::ostream& operator<<(std::ostream&, const Type::Code&); @@ -447,7 +331,7 @@ std::ostream& operator<<(std::ostream&, const Type&); * * @return Type object */ -std::unique_ptr bool_(); +Type bool_(); /** * @ingroup types @@ -455,7 +339,7 @@ std::unique_ptr bool_(); * * @return Type object */ -std::unique_ptr int8(); +Type int8(); /** * @ingroup types @@ -463,7 +347,7 @@ std::unique_ptr int8(); * * @return Type object */ -std::unique_ptr int16(); +Type int16(); /** * @ingroup types @@ -471,7 +355,7 @@ std::unique_ptr int16(); * * @return Type object */ -std::unique_ptr int32(); +Type int32(); /** * @ingroup types @@ -479,7 +363,7 @@ std::unique_ptr int32(); * * @return Type object */ -std::unique_ptr int64(); +Type int64(); /** * @ingroup types @@ -487,7 +371,7 @@ std::unique_ptr int64(); * * @return Type object */ -std::unique_ptr uint8(); +Type uint8(); /** * @ingroup types @@ -495,7 +379,7 @@ std::unique_ptr uint8(); * * @return Type object */ -std::unique_ptr uint16(); +Type uint16(); /** * @ingroup types @@ -503,7 +387,7 @@ std::unique_ptr uint16(); * * @return Type object */ -std::unique_ptr uint32(); +Type uint32(); /** * @ingroup types @@ -511,7 +395,7 @@ std::unique_ptr uint32(); * * @return Type object */ -std::unique_ptr uint64(); +Type uint64(); /** * @ingroup types @@ -519,7 +403,7 @@ std::unique_ptr uint64(); * * @return Type object */ -std::unique_ptr float16(); +Type float16(); /** * @ingroup types @@ -527,7 +411,7 @@ std::unique_ptr float16(); * * @return Type object */ -std::unique_ptr float32(); +Type float32(); /** * @ingroup types @@ -535,7 +419,7 @@ std::unique_ptr float32(); * * @return Type object */ -std::unique_ptr float64(); +Type float64(); /** * @ingroup types @@ -543,7 +427,7 @@ std::unique_ptr float64(); * * @return Type object */ -std::unique_ptr complex64(); +Type complex64(); /** * @ingroup types @@ -551,15 +435,7 @@ std::unique_ptr complex64(); * * @return Type object */ -std::unique_ptr complex128(); - -/** - * @ingroup types - * @brief Creates a string type - * - * @return Type object - */ -std::unique_ptr string(); +Type complex128(); /** * @ingroup types @@ -569,7 +445,7 @@ std::unique_ptr string(); * * @return Type object */ -std::unique_ptr point_type(int32_t ndim); +Type point_type(int32_t ndim); /** * @ingroup types @@ -579,7 +455,7 @@ std::unique_ptr point_type(int32_t ndim); * * @return Type object */ -std::unique_ptr rect_type(int32_t ndim); +Type rect_type(int32_t ndim); /** * @ingroup types diff --git a/src/core/utilities/deserializer.cc b/src/core/utilities/deserializer.cc index d9a53e8b37..52fbcd5310 100644 --- a/src/core/utilities/deserializer.cc +++ b/src/core/utilities/deserializer.cc @@ -15,6 +15,7 @@ */ #include "core/utilities/deserializer.h" +#include "core/data/detail/store.h" #include "core/data/scalar.h" #include "core/data/store.h" #include "core/utilities/machine.h" @@ -39,7 +40,7 @@ TaskDeserializer::TaskDeserializer(const Legion::Task* task, first_task_ = !task->is_index_space || (task->index_point == task->index_domain.lo()); } -void TaskDeserializer::_unpack(Store& value) +std::shared_ptr TaskDeserializer::_unpack_store() { auto is_future = unpack(); auto is_output_region = unpack(); @@ -50,22 +51,25 @@ void TaskDeserializer::_unpack(Store& value) if (is_future) { auto redop_id = unpack(); - auto fut = unpack(); + auto fut = unpack(); if (redop_id != -1 && !first_task_) fut.initialize_with_identity(redop_id); - value = Store(dim, std::move(type), redop_id, fut, std::move(transform)); + return std::make_shared( + dim, std::move(type), redop_id, fut, std::move(transform)); } else if (!is_output_region) { auto redop_id = unpack(); - auto rf = unpack(); - value = Store(dim, std::move(type), redop_id, std::move(rf), std::move(transform)); + auto rf = unpack(); + return std::make_shared( + dim, std::move(type), redop_id, std::move(rf), std::move(transform)); } else { auto redop_id = unpack(); assert(redop_id == -1); - auto out = unpack(); - value = Store(dim, std::move(type), std::move(out), std::move(transform)); + auto out = unpack(); + return std::make_shared( + dim, std::move(type), std::move(out), std::move(transform)); } } -void TaskDeserializer::_unpack(FutureWrapper& value) +void TaskDeserializer::_unpack(detail::FutureWrapper& value) { auto read_only = unpack(); auto has_storage = unpack(); @@ -85,25 +89,25 @@ void TaskDeserializer::_unpack(FutureWrapper& value) futures_ = futures_.subspan(1); } - value = FutureWrapper(read_only, field_size, domain, future, has_storage && first_task_); + value = detail::FutureWrapper(read_only, field_size, domain, future, has_storage && first_task_); } -void TaskDeserializer::_unpack(RegionField& value) +void TaskDeserializer::_unpack(detail::RegionField& value) { auto dim = unpack(); auto idx = unpack(); auto fid = unpack(); - value = RegionField(dim, regions_[idx], fid); + value = detail::RegionField(dim, regions_[idx], fid); } -void TaskDeserializer::_unpack(UnboundRegionField& value) +void TaskDeserializer::_unpack(detail::UnboundRegionField& value) { auto dim = unpack(); auto idx = unpack(); auto fid = unpack(); - value = UnboundRegionField(outputs_[idx], fid); + value = detail::UnboundRegionField(outputs_[idx], fid); } void TaskDeserializer::_unpack(comm::Communicator& value) @@ -142,7 +146,7 @@ TaskDeserializer::TaskDeserializer(const Legion::Task* task, first_task_ = false; } -void TaskDeserializer::_unpack(Store& value) +void TaskDeserializer::_unpack(detail::Store& store) { auto is_future = unpack(); auto is_output_region = unpack(); @@ -154,24 +158,24 @@ void TaskDeserializer::_unpack(Store& value) if (is_future) { // We still need to parse the reduction op unpack(); - auto fut = unpack(); - value = Store(dim, std::move(type), fut, std::move(transform)); + auto fut = unpack(); + store = detail::Store(dim, std::move(type), fut, std::move(transform)); } else { auto redop_id = unpack(); - RegionField rf; + detail::RegionField rf; _unpack(rf, is_output_region); - value = Store(runtime_, - context_, - dim, - std::move(type), - redop_id, - rf, - is_output_region, - std::move(transform)); + store = detail::Store(runtime_, + context_, + dim, + std::move(type), + redop_id, + rf, + is_output_region, + std::move(transform)); } } -void TaskDeserializer::_unpack(FutureWrapper& value) +void TaskDeserializer::_unpack(detail::FutureWrapper& value) { // We still need to deserialize these fields to get to the domain unpack(); @@ -186,17 +190,17 @@ void TaskDeserializer::_unpack(FutureWrapper& value) domain.rect_data[idx + domain.dim] = point[idx] - 1; } - value = FutureWrapper(future_index_++, domain); + value = detail::FutureWrapper(future_index_++, domain); } -void TaskDeserializer::_unpack(RegionField& value, bool is_output_region) +void TaskDeserializer::_unpack(detail::RegionField& value, bool is_output_region) { auto dim = unpack(); auto idx = unpack(); auto fid = unpack(); auto req = is_output_region ? &task_->output_regions[idx] : &task_->regions[idx]; - value = RegionField(req, dim, idx, fid); + value = detail::RegionField(req, dim, idx, fid); } CopyDeserializer::CopyDeserializer(const Legion::Copy* copy, @@ -221,7 +225,7 @@ void CopyDeserializer::next_requirement_list() ++curr_reqs_; } -void CopyDeserializer::_unpack(Store& value) +void CopyDeserializer::_unpack(detail::Store& store) { auto is_future = unpack(); auto is_output_region = unpack(); @@ -234,20 +238,20 @@ void CopyDeserializer::_unpack(Store& value) assert(!is_future && !is_output_region); #endif auto redop_id = unpack(); - RegionField rf; + detail::RegionField rf; _unpack(rf); - value = Store( + store = detail::Store( runtime_, context_, dim, std::move(type), redop_id, rf, is_output_region, std::move(transform)); } -void CopyDeserializer::_unpack(RegionField& value) +void CopyDeserializer::_unpack(detail::RegionField& value) { auto dim = unpack(); auto idx = unpack(); auto fid = unpack(); auto req = &curr_reqs_->get()[idx]; - value = RegionField(req, dim, idx + req_index_offset_, fid); + value = detail::RegionField(req, dim, idx + req_index_offset_, fid); } } // namespace legate::mapping diff --git a/src/core/utilities/deserializer.h b/src/core/utilities/deserializer.h index f00fb9652b..4ba6724a58 100644 --- a/src/core/utilities/deserializer.h +++ b/src/core/utilities/deserializer.h @@ -21,10 +21,14 @@ #include "legion.h" #include "core/comm/communicator.h" +#include "core/data/detail/scalar.h" +#include "core/data/detail/store.h" +#include "core/data/detail/transform.h" #include "core/data/scalar.h" #include "core/data/store.h" -#include "core/mapping/machine.h" -#include "core/mapping/operation.h" +#include "core/mapping/detail/machine.h" +#include "core/mapping/detail/store.h" +#include "core/type/detail/type_info.h" #include "core/type/type_traits.h" #include "core/utilities/span.h" #include "core/utilities/typedefs.h" @@ -79,14 +83,14 @@ class BaseDeserializer { Scalar _unpack_scalar(); void _unpack(mapping::TaskTarget& value); void _unpack(mapping::ProcessorRange& value); - void _unpack(mapping::MachineDesc& value); + void _unpack(mapping::detail::Machine& value); public: Span current_args() const { return args_; } protected: - std::shared_ptr unpack_transform(); - std::unique_ptr unpack_type(); + std::shared_ptr unpack_transform(); + std::shared_ptr unpack_type(); protected: bool first_task_; @@ -101,10 +105,18 @@ class TaskDeserializer : public BaseDeserializer { using BaseDeserializer::_unpack; public: - void _unpack(Store& value); - void _unpack(FutureWrapper& value); - void _unpack(RegionField& value); - void _unpack(UnboundRegionField& value); + void _unpack(std::vector& values) + { + auto size = unpack(); + values.reserve(size); + for (uint32_t idx = 0; idx < size; ++idx) values.emplace_back(_unpack_store()); + } + + public: + std::shared_ptr _unpack_store(); + void _unpack(detail::FutureWrapper& value); + void _unpack(detail::RegionField& value); + void _unpack(detail::UnboundRegionField& value); void _unpack(comm::Communicator& value); void _unpack(Legion::PhaseBarrier& barrier); @@ -136,9 +148,9 @@ class TaskDeserializer : public BaseDeserializer { using BaseDeserializer::_unpack; public: - void _unpack(Store& value); - void _unpack(FutureWrapper& value); - void _unpack(RegionField& value, bool is_output_region); + void _unpack(detail::Store& store); + void _unpack(detail::FutureWrapper& value); + void _unpack(detail::RegionField& value, bool is_output_region); private: const Legion::Task* task_; @@ -165,8 +177,8 @@ class CopyDeserializer : public BaseDeserializer { void next_requirement_list(); public: - void _unpack(Store& value); - void _unpack(RegionField& value); + void _unpack(detail::Store& store); + void _unpack(detail::RegionField& value); private: std::vector all_reqs_; diff --git a/src/core/utilities/deserializer.inl b/src/core/utilities/deserializer.inl index e6e9930382..6a73119496 100644 --- a/src/core/utilities/deserializer.inl +++ b/src/core/utilities/deserializer.inl @@ -30,10 +30,12 @@ BaseDeserializer::BaseDeserializer(const void* args, size_t arglen template Scalar BaseDeserializer::_unpack_scalar() { - auto type = unpack_type(); - Scalar result(std::move(type), args_.ptr()); - args_ = args_.subspan(result.size()); - return result; + // this unpack_type call must be in a separate line from the following one because they both + // read and update the buffer location. + auto type = unpack_type(); + auto result = std::make_unique(type, args_.ptr(), false /*copy*/); + args_ = args_.subspan(result->size()); + return Scalar(std::move(result)); } template @@ -51,7 +53,7 @@ void BaseDeserializer::_unpack(mapping::ProcessorRange& value) } template -void BaseDeserializer::_unpack(mapping::MachineDesc& value) +void BaseDeserializer::_unpack(mapping::detail::Machine& value) { value.preferred_target = unpack(); auto num_ranges = unpack(); @@ -63,46 +65,46 @@ void BaseDeserializer::_unpack(mapping::MachineDesc& value) } template -std::shared_ptr BaseDeserializer::unpack_transform() +std::shared_ptr BaseDeserializer::unpack_transform() { auto code = unpack(); switch (code) { case -1: { - return std::make_shared(); + return std::make_shared(); } case LEGATE_CORE_TRANSFORM_SHIFT: { auto dim = unpack(); auto offset = unpack(); auto parent = unpack_transform(); - return std::make_shared(std::make_unique(dim, offset), - std::move(parent)); + return std::make_shared(std::make_unique(dim, offset), + std::move(parent)); } case LEGATE_CORE_TRANSFORM_PROMOTE: { auto extra_dim = unpack(); auto dim_size = unpack(); auto parent = unpack_transform(); - return std::make_shared(std::make_unique(extra_dim, dim_size), - std::move(parent)); + return std::make_shared( + std::make_unique(extra_dim, dim_size), std::move(parent)); } case LEGATE_CORE_TRANSFORM_PROJECT: { auto dim = unpack(); auto coord = unpack(); auto parent = unpack_transform(); - return std::make_shared(std::make_unique(dim, coord), - std::move(parent)); + return std::make_shared(std::make_unique(dim, coord), + std::move(parent)); } case LEGATE_CORE_TRANSFORM_TRANSPOSE: { auto axes = unpack>(); auto parent = unpack_transform(); - return std::make_shared(std::make_unique(std::move(axes)), - std::move(parent)); + return std::make_shared( + std::make_unique(std::move(axes)), std::move(parent)); } case LEGATE_CORE_TRANSFORM_DELINEARIZE: { auto dim = unpack(); auto sizes = unpack>(); auto parent = unpack_transform(); - return std::make_shared(std::make_unique(dim, std::move(sizes)), - std::move(parent)); + return std::make_shared( + std::make_unique(dim, std::move(sizes)), std::move(parent)); } } assert(false); @@ -110,7 +112,7 @@ std::shared_ptr BaseDeserializer::unpack_transform } template -std::unique_ptr BaseDeserializer::unpack_type() +std::shared_ptr BaseDeserializer::unpack_type() { auto code = static_cast(unpack()); switch (code) { @@ -118,19 +120,19 @@ std::unique_ptr BaseDeserializer::unpack_type() auto uid = unpack(); auto N = unpack(); auto type = unpack_type(); - return std::make_unique(uid, std::move(type), N); + return std::make_shared(uid, std::move(type), N); } case Type::Code::STRUCT: { auto uid = unpack(); auto num_fields = unpack(); - std::vector> field_types; + std::vector> field_types; field_types.reserve(num_fields); for (uint32_t idx = 0; idx < num_fields; ++idx) field_types.emplace_back(unpack_type()); auto align = unpack(); - return std::make_unique(uid, std::move(field_types), align); + return std::make_shared(uid, std::move(field_types), align); } case Type::Code::BOOL: case Type::Code::INT8: @@ -146,10 +148,10 @@ std::unique_ptr BaseDeserializer::unpack_type() case Type::Code::FLOAT64: case Type::Code::COMPLEX64: case Type::Code::COMPLEX128: { - return std::make_unique(code); + return std::make_shared(code); } case Type::Code::STRING: { - return std::make_unique(); + return std::make_shared(); } default: { LEGATE_ABORT; diff --git a/src/core/utilities/buffer_builder.cc b/src/core/utilities/detail/buffer_builder.cc similarity index 90% rename from src/core/utilities/buffer_builder.cc rename to src/core/utilities/detail/buffer_builder.cc index 14bcc78b19..b1ec4c8dc4 100644 --- a/src/core/utilities/buffer_builder.cc +++ b/src/core/utilities/detail/buffer_builder.cc @@ -14,9 +14,9 @@ * */ -#include "core/utilities/buffer_builder.h" +#include "core/utilities/detail/buffer_builder.h" -namespace legate { +namespace legate::detail { BufferBuilder::BufferBuilder() { @@ -37,4 +37,4 @@ Legion::UntypedBuffer BufferBuilder::to_legion_buffer() const return Legion::UntypedBuffer(buffer_.data(), buffer_.size()); } -} // namespace legate +} // namespace legate::detail diff --git a/src/core/utilities/buffer_builder.h b/src/core/utilities/detail/buffer_builder.h similarity index 94% rename from src/core/utilities/buffer_builder.h rename to src/core/utilities/detail/buffer_builder.h index bba3133761..a16b63bfd6 100644 --- a/src/core/utilities/buffer_builder.h +++ b/src/core/utilities/detail/buffer_builder.h @@ -20,7 +20,7 @@ #include "core/utilities/tuple.h" -namespace legate { +namespace legate::detail { /** * @ingroup util @@ -78,6 +78,6 @@ class BufferBuilder { std::vector buffer_; }; -} // namespace legate +} // namespace legate::detail -#include "core/utilities/buffer_builder.inl" +#include "core/utilities/detail/buffer_builder.inl" diff --git a/src/core/utilities/buffer_builder.inl b/src/core/utilities/detail/buffer_builder.inl similarity index 87% rename from src/core/utilities/buffer_builder.inl rename to src/core/utilities/detail/buffer_builder.inl index 3cdb7c850b..fc779d5591 100644 --- a/src/core/utilities/buffer_builder.inl +++ b/src/core/utilities/detail/buffer_builder.inl @@ -1,4 +1,4 @@ -/* Copyright 2021 NVIDIA Corporation +/* Copyright 2023 NVIDIA Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,9 +17,9 @@ #pragma once // Useful for IDEs -#include "core/utilities/buffer_builder.h" +#include "core/utilities/detail/buffer_builder.h" -namespace legate { +namespace legate::detail { template void BufferBuilder::pack(const T& value) @@ -41,4 +41,4 @@ void BufferBuilder::pack(const tuple& values) pack(values.data()); } -} // namespace legate +} // namespace legate::detail diff --git a/src/core/utilities/nvtx_help.h b/src/core/utilities/nvtx_help.h index 113c294202..5d69b7fd3e 100644 --- a/src/core/utilities/nvtx_help.h +++ b/src/core/utilities/nvtx_help.h @@ -16,7 +16,7 @@ #pragma once -#include "legate.h" +#include "legate_defines.h" #ifdef LEGATE_USE_CUDA diff --git a/src/core/utilities/typedefs.h b/src/core/utilities/typedefs.h index 63ccc70fb4..8b249b1be7 100644 --- a/src/core/utilities/typedefs.h +++ b/src/core/utilities/typedefs.h @@ -250,7 +250,7 @@ using Memory = Legion::Memory; * types (`__half`, `float`, and `double`) are supported by all but bitwise operators. Arithmetic * operators also cover complex types `complex<__half>` and `complex`. * - * For details about reduction operators, See LibraryContext::register_reduction_operator. + * For details about reduction operators, See Library::register_reduction_operator. * * @{ */ diff --git a/src/env_defaults.h b/src/env_defaults.h index 7ae3f840d2..9d238ea088 100644 --- a/src/env_defaults.h +++ b/src/env_defaults.h @@ -1,4 +1,4 @@ -/* Copyright 2021-2022 NVIDIA Corporation +/* Copyright 2023 NVIDIA Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -55,4 +55,4 @@ #define MAX_LRU_LENGTH_TEST 1 #define DISABLE_MPI_DEFAULT 0 -#define DISABLE_MPI_TEST 0 \ No newline at end of file +#define DISABLE_MPI_TEST 0 diff --git a/src/legate.h b/src/legate.h index 2aa129372f..fdaee963a0 100644 --- a/src/legate.h +++ b/src/legate.h @@ -30,16 +30,18 @@ #include "core/data/store.h" #include "core/legate_c.h" #include "core/mapping/mapping.h" +#include "core/mapping/operation.h" #include "core/operation/task.h" #include "core/partitioning/constraint.h" #include "core/partitioning/partition.h" +#include "core/runtime/library.h" #include "core/runtime/runtime.h" #include "core/runtime/tracker.h" #include "core/task/exception.h" #include "core/task/registrar.h" #include "core/task/task.h" +#include "core/task/task_context.h" #include "core/type/type_traits.h" -#include "core/utilities/deserializer.h" #include "core/utilities/dispatch.h" #include "core/utilities/typedefs.h" #include "legate_defines.h" diff --git a/tests/cpp/integration/copy_gather.cc b/tests/cpp/integration/copy_gather.cc index c10e2c109d..83602e0fe7 100644 --- a/tests/cpp/integration/copy_gather.cc +++ b/tests/cpp/integration/copy_gather.cc @@ -56,7 +56,7 @@ struct CheckGatherTask : public legate::LegateTask::register_variants(context); } -void check_gather_output(legate::LibraryContext* context, +void check_gather_output(legate::Library library, const legate::LogicalStore& src, const legate::LogicalStore& tgt, const legate::LogicalStore& ind) @@ -99,7 +99,7 @@ void check_gather_output(legate::LibraryContext* context, auto runtime = legate::Runtime::get_runtime(); auto machine = runtime->get_machine(); int32_t task_id = CHECK_GATHER_TASK + ind.dim() * TEST_MAX_DIM + src.dim(); - auto task = runtime->create_task(context, task_id); + auto task = runtime->create_task(library, task_id); auto src_part = task.declare_partition(); auto tgt_part = task.declare_partition(); auto ind_part = task.declare_partition(); @@ -134,19 +134,19 @@ void test_gather(const GatherSpec& spec) logger.print() << "Gather Copy: " << spec.to_string(); auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library(library_name); + auto library = runtime->find_library(library_name); - auto& type = spec.seed.type(); - auto src = runtime->create_store(spec.data_shape, type.clone()); - auto tgt = runtime->create_store(spec.ind_shape, type.clone()); - auto ind = runtime->create_store(spec.ind_shape, legate::point_type(spec.data_shape.size())); + auto type = spec.seed.type(); + auto src = runtime->create_store(spec.data_shape, type); + auto tgt = runtime->create_store(spec.ind_shape, type); + auto ind = runtime->create_store(spec.ind_shape, legate::point_type(spec.data_shape.size())); - fill_input(context, src, spec.seed); - fill_indirect(context, ind, src); + fill_input(library, src, spec.seed); + fill_indirect(library, ind, src); runtime->issue_gather(tgt, src, ind); - check_gather_output(context, src, tgt, ind); + check_gather_output(library, src, tgt, ind); } TEST(Copy, Gather2Dto1D) diff --git a/tests/cpp/integration/copy_gather_scatter.cc b/tests/cpp/integration/copy_gather_scatter.cc index efcd290cac..47b553184e 100644 --- a/tests/cpp/integration/copy_gather_scatter.cc +++ b/tests/cpp/integration/copy_gather_scatter.cc @@ -74,7 +74,7 @@ struct CheckGatherScatterTask IND_DIM * TEST_MAX_DIM + TGT_DIM; static void cpu_variant(legate::TaskContext& context) { - auto type_code = context.inputs().at(0).type().code; + auto type_code = context.inputs().at(0).type().code(); type_dispatch_for_test(type_code, CheckGatherScatterTaskBody{}, context); } }; @@ -100,29 +100,29 @@ struct GatherScatterSpec { void register_tasks() { auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->create_library(library_name); - FillTask<1>::register_variants(context); - FillTask<2>::register_variants(context); - FillTask<3>::register_variants(context); - - FillIndirectTask<1, 1>::register_variants(context); - FillIndirectTask<1, 2>::register_variants(context); - FillIndirectTask<1, 3>::register_variants(context); - FillIndirectTask<2, 1>::register_variants(context); - FillIndirectTask<2, 2>::register_variants(context); - FillIndirectTask<2, 3>::register_variants(context); - FillIndirectTask<3, 1>::register_variants(context); - FillIndirectTask<3, 2>::register_variants(context); - FillIndirectTask<3, 3>::register_variants(context); - - CheckGatherScatterTask<1, 2, 3>::register_variants(context); - CheckGatherScatterTask<2, 3, 1>::register_variants(context); - CheckGatherScatterTask<3, 1, 2>::register_variants(context); - CheckGatherScatterTask<3, 3, 3>::register_variants(context); - CheckGatherScatterTask<2, 2, 3>::register_variants(context); + auto library = runtime->create_library(library_name); + FillTask<1>::register_variants(library); + FillTask<2>::register_variants(library); + FillTask<3>::register_variants(library); + + FillIndirectTask<1, 1>::register_variants(library); + FillIndirectTask<1, 2>::register_variants(library); + FillIndirectTask<1, 3>::register_variants(library); + FillIndirectTask<2, 1>::register_variants(library); + FillIndirectTask<2, 2>::register_variants(library); + FillIndirectTask<2, 3>::register_variants(library); + FillIndirectTask<3, 1>::register_variants(library); + FillIndirectTask<3, 2>::register_variants(library); + FillIndirectTask<3, 3>::register_variants(library); + + CheckGatherScatterTask<1, 2, 3>::register_variants(library); + CheckGatherScatterTask<2, 3, 1>::register_variants(library); + CheckGatherScatterTask<3, 1, 2>::register_variants(library); + CheckGatherScatterTask<3, 3, 3>::register_variants(library); + CheckGatherScatterTask<2, 2, 3>::register_variants(library); } -void check_gather_scatter_output(legate::LibraryContext* context, +void check_gather_scatter_output(legate::Library library, const legate::LogicalStore& src, const legate::LogicalStore& tgt, const legate::LogicalStore& src_ind, @@ -135,7 +135,7 @@ void check_gather_scatter_output(legate::LibraryContext* context, int32_t task_id = CHECK_GATHER_SCATTER_TASK + src.dim() * TEST_MAX_DIM * TEST_MAX_DIM + src_ind.dim() * TEST_MAX_DIM + tgt.dim(); - auto task = runtime->create_task(context, task_id); + auto task = runtime->create_task(library, task_id); auto src_part = task.declare_partition(); auto tgt_part = task.declare_partition(); @@ -161,21 +161,21 @@ void test_gather_scatter(const GatherScatterSpec& spec) logger.print() << "Gather-scatter Copy: " << spec.to_string(); auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library(library_name); + auto library = runtime->find_library(library_name); - auto& type = spec.seed.type(); - auto src = runtime->create_store(spec.src_shape, type.clone()); - auto tgt = runtime->create_store(spec.tgt_shape, type.clone()); + auto type = spec.seed.type(); + auto src = runtime->create_store(spec.src_shape, type); + auto tgt = runtime->create_store(spec.tgt_shape, type); auto src_ind = runtime->create_store(spec.ind_shape, legate::point_type(spec.src_shape.size())); auto tgt_ind = runtime->create_store(spec.ind_shape, legate::point_type(spec.tgt_shape.size())); - fill_input(context, src, spec.seed); - fill_indirect(context, src_ind, src); - fill_indirect(context, tgt_ind, tgt); + fill_input(library, src, spec.seed); + fill_indirect(library, src_ind, src); + fill_indirect(library, tgt_ind, tgt); runtime->issue_fill(tgt, spec.init); runtime->issue_scatter_gather(tgt, tgt_ind, src, src_ind); - check_gather_scatter_output(context, src, tgt, src_ind, tgt_ind, spec.init); + check_gather_scatter_output(library, src, tgt, src_ind, tgt_ind, spec.init); } TEST(Copy, GatherScatter1Dto3Dvia2D) diff --git a/tests/cpp/integration/copy_normal.cc b/tests/cpp/integration/copy_normal.cc index a9c3fcd138..e563e379e3 100644 --- a/tests/cpp/integration/copy_normal.cc +++ b/tests/cpp/integration/copy_normal.cc @@ -52,29 +52,29 @@ struct CheckTask : public legate::LegateTask> { if (shape.empty()) return; - type_dispatch(source.type().code, CheckTaskBody{}, source, target, shape); + type_dispatch(source.type().code(), CheckTaskBody{}, source, target, shape); } }; void register_tasks() { auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->create_library(library_name); - FillTask<1>::register_variants(context); - FillTask<2>::register_variants(context); - FillTask<3>::register_variants(context); - CheckTask<1>::register_variants(context); - CheckTask<2>::register_variants(context); - CheckTask<3>::register_variants(context); + auto library = runtime->create_library(library_name); + FillTask<1>::register_variants(library); + FillTask<2>::register_variants(library); + FillTask<3>::register_variants(library); + CheckTask<1>::register_variants(library); + CheckTask<2>::register_variants(library); + CheckTask<3>::register_variants(library); } -void check_output(legate::LibraryContext* context, +void check_output(legate::Library library, const legate::LogicalStore& src, const legate::LogicalStore& tgt) { auto runtime = legate::Runtime::get_runtime(); auto machine = runtime->get_machine(); - auto task = runtime->create_task(context, CHECK_TASK + tgt.dim()); + auto task = runtime->create_task(library, CHECK_TASK + tgt.dim()); auto src_part = task.declare_partition(); auto tgt_part = task.declare_partition(); @@ -86,31 +86,27 @@ void check_output(legate::LibraryContext* context, runtime->submit(std::move(task)); } -const auto uint32 = legate::uint32(); -const auto int64 = legate::int64(); -const auto float64 = legate::float64(); - struct NormalCopySpec { std::vector shape; - const legate::Type& type; + legate::Type type; legate::Scalar seed; }; void test_normal_copy(const NormalCopySpec& spec) { auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library(library_name); + auto library = runtime->find_library(library_name); auto& [shape, type, seed] = spec; - auto input = runtime->create_store(shape, type.clone()); - auto output = runtime->create_store(shape, type.clone()); + auto input = runtime->create_store(shape, type); + auto output = runtime->create_store(shape, type); - fill_input(context, input, seed); + fill_input(library, input, seed); runtime->issue_copy(output, input); // check the result of copy - check_output(context, input, output); + check_output(library, input, output); } TEST(Copy, Single) @@ -119,8 +115,8 @@ TEST(Copy, Single) // For some reason, clang-format gets tripped over by singleton initialization lists, // so factor out the definition here std::vector shape1d{9}; - test_normal_copy({{4, 7}, *int64, legate::Scalar(int64_t(12))}); - test_normal_copy({{1000, 100}, *uint32, legate::Scalar(uint32_t(3))}); + test_normal_copy({{4, 7}, legate::int64(), legate::Scalar(int64_t(12))}); + test_normal_copy({{1000, 100}, legate::uint32(), legate::Scalar(uint32_t(3))}); } } // namespace copy_normal diff --git a/tests/cpp/integration/copy_scatter.cc b/tests/cpp/integration/copy_scatter.cc index 9e64c68119..adb6612e31 100644 --- a/tests/cpp/integration/copy_scatter.cc +++ b/tests/cpp/integration/copy_scatter.cc @@ -70,7 +70,7 @@ struct CheckScatterTask : public legate::LegateTaskcreate_library(library_name); - FillTask<1>::register_variants(context); - FillTask<2>::register_variants(context); - FillTask<3>::register_variants(context); + auto library = runtime->create_library(library_name); + FillTask<1>::register_variants(library); + FillTask<2>::register_variants(library); + FillTask<3>::register_variants(library); // XXX: Tasks unused by the test cases are commented out - // FillIndirectTask<1, 1>::register_variants(context); - FillIndirectTask<1, 2>::register_variants(context); - // FillIndirectTask<1, 3>::register_variants(context); - // FillIndirectTask<2, 1>::register_variants(context); - FillIndirectTask<2, 2>::register_variants(context); - FillIndirectTask<2, 3>::register_variants(context); - FillIndirectTask<3, 1>::register_variants(context); - FillIndirectTask<3, 2>::register_variants(context); - // FillIndirectTask<3, 3>::register_variants(context); - - // CheckScatterTask<1, 1>::register_variants(context); - CheckScatterTask<1, 2>::register_variants(context); - // CheckScatterTask<1, 3>::register_variants(context); - // CheckScatterTask<2, 1>::register_variants(context); - CheckScatterTask<2, 2>::register_variants(context); - CheckScatterTask<2, 3>::register_variants(context); - CheckScatterTask<3, 1>::register_variants(context); - CheckScatterTask<3, 2>::register_variants(context); - // CheckScatterTask<3, 3>::register_variants(context); + // FillIndirectTask<1, 1>::register_variants(library); + FillIndirectTask<1, 2>::register_variants(library); + // FillIndirectTask<1, 3>::register_variants(library); + // FillIndirectTask<2, 1>::register_variants(library); + FillIndirectTask<2, 2>::register_variants(library); + FillIndirectTask<2, 3>::register_variants(library); + FillIndirectTask<3, 1>::register_variants(library); + FillIndirectTask<3, 2>::register_variants(library); + // FillIndirectTask<3, 3>::register_variants(library); + + // CheckScatterTask<1, 1>::register_variants(library); + CheckScatterTask<1, 2>::register_variants(library); + // CheckScatterTask<1, 3>::register_variants(library); + // CheckScatterTask<2, 1>::register_variants(library); + CheckScatterTask<2, 2>::register_variants(library); + CheckScatterTask<2, 3>::register_variants(library); + CheckScatterTask<3, 1>::register_variants(library); + CheckScatterTask<3, 2>::register_variants(library); + // CheckScatterTask<3, 3>::register_variants(library); } struct ScatterSpec { @@ -126,7 +126,7 @@ struct ScatterSpec { } }; -void check_scatter_output(legate::LibraryContext* context, +void check_scatter_output(legate::Library library, const legate::LogicalStore& src, const legate::LogicalStore& tgt, const legate::LogicalStore& ind, @@ -137,7 +137,7 @@ void check_scatter_output(legate::LibraryContext* context, int32_t task_id = CHECK_SCATTER_TASK + ind.dim() * TEST_MAX_DIM + tgt.dim(); - auto task = runtime->create_task(context, task_id); + auto task = runtime->create_task(library, task_id); auto src_part = task.declare_partition(); auto tgt_part = task.declare_partition(); @@ -160,19 +160,19 @@ void test_scatter(const ScatterSpec& spec) logger.print() << "Scatter Copy: " << spec.to_string(); auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library(library_name); + auto library = runtime->find_library(library_name); - auto& type = spec.seed.type(); - auto src = runtime->create_store(spec.ind_shape, type.clone()); - auto tgt = runtime->create_store(spec.data_shape, type.clone()); - auto ind = runtime->create_store(spec.ind_shape, legate::point_type(spec.data_shape.size())); + auto type = spec.seed.type(); + auto src = runtime->create_store(spec.ind_shape, type); + auto tgt = runtime->create_store(spec.data_shape, type); + auto ind = runtime->create_store(spec.ind_shape, legate::point_type(spec.data_shape.size())); - fill_input(context, src, spec.seed); - fill_indirect(context, ind, tgt); + fill_input(library, src, spec.seed); + fill_indirect(library, ind, tgt); runtime->issue_fill(tgt, spec.init); runtime->issue_scatter(tgt, ind, src); - check_scatter_output(context, src, tgt, ind, spec.init); + check_scatter_output(library, src, tgt, ind, spec.init); } // Note that the volume of indirection field should be smaller than that of the target to avoid diff --git a/tests/cpp/integration/copy_util.inl b/tests/cpp/integration/copy_util.inl index 877ef47565..4786f352bc 100644 --- a/tests/cpp/integration/copy_util.inl +++ b/tests/cpp/integration/copy_util.inl @@ -72,7 +72,7 @@ struct FillTask : public legate::LegateTask> { if (shape.empty()) return; - type_dispatch_for_test(output.type().code, FillTaskBody{}, output, shape, seed); + type_dispatch_for_test(output.type().code(), FillTaskBody{}, output, shape, seed); } }; @@ -113,26 +113,26 @@ struct FillIndirectTask : public legate::LegateTaskget_machine(); - auto task = runtime->create_task(context, FILL_TASK + src.dim()); + auto task = runtime->create_task(library, FILL_TASK + src.dim()); task.add_output(src, task.declare_partition()); task.add_scalar_arg(value); runtime->submit(std::move(task)); } -void fill_indirect(legate::LibraryContext* context, +void fill_indirect(legate::Library library, const legate::LogicalStore& ind, const legate::LogicalStore& data) { auto runtime = legate::Runtime::get_runtime(); auto machine = runtime->get_machine(); int32_t task_id = FILL_INDIRECT_TASK + ind.dim() * TEST_MAX_DIM + data.dim(); - auto task = runtime->create_task(context, task_id); + auto task = runtime->create_task(library, task_id); auto part = task.declare_partition(); task.add_output(ind, part); // Technically indirection fields for gather coipes can have repeated points diff --git a/tests/cpp/integration/image_constraints.cc b/tests/cpp/integration/image_constraints.cc index f1921a1c7d..84e4c89743 100644 --- a/tests/cpp/integration/image_constraints.cc +++ b/tests/cpp/integration/image_constraints.cc @@ -184,7 +184,7 @@ void initialize_function(legate::LogicalStore func, const std::vector ra auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(library_name); - auto is_rect = func.type().code == legate::Type::Code::STRUCT; + auto is_rect = func.type().code() == legate::Type::Code::STRUCT; auto task = runtime->create_task( context, INIT_FUNC + static_cast(is_rect) * TEST_MAX_DIM + func.dim()); auto part = task.declare_partition(); @@ -201,7 +201,7 @@ void check_image(legate::LogicalStore func, legate::LogicalStore range) auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(library_name); - auto is_rect = func.type().code == legate::Type::Code::STRUCT; + auto is_rect = func.type().code() == legate::Type::Code::STRUCT; auto task = runtime->create_task( context, IMAGE_TESTER + static_cast(is_rect) * TEST_MAX_DIM + func.dim()); auto part_domain = task.declare_partition(); diff --git a/tests/cpp/integration/inline_map.cc b/tests/cpp/integration/inline_map.cc index 900d16f8e8..7154a28d8f 100644 --- a/tests/cpp/integration/inline_map.cc +++ b/tests/cpp/integration/inline_map.cc @@ -51,7 +51,7 @@ void test_mapped_regions_leak() { auto l_store = runtime->create_store({5}, legate::int64()); auto p_store = l_store.get_physical_store(); - EXPECT_FALSE(p_store->is_future()); + EXPECT_FALSE(p_store.is_future()); EXPECT_EQ(runtime->impl()->num_inline_mapped(), 1); } EXPECT_EQ(runtime->impl()->num_inline_mapped(), 0); @@ -62,7 +62,7 @@ void test_inline_map_future() auto runtime = legate::Runtime::get_runtime(); auto l_store = runtime->create_store({1}, legate::int64(), true /*optimize_scalar*/); auto p_store = l_store.get_physical_store(); - EXPECT_TRUE(p_store->is_future()); + EXPECT_TRUE(p_store.is_future()); } void test_inline_map_region_and_slice() @@ -70,13 +70,13 @@ void test_inline_map_region_and_slice() auto runtime = legate::Runtime::get_runtime(); auto root_ls = runtime->create_store({5}, legate::int64()); auto root_ps = root_ls.get_physical_store(); - EXPECT_FALSE(root_ps->is_future()); + EXPECT_FALSE(root_ps.is_future()); auto slice_ls = root_ls.slice(0, legate::Slice(1)); auto slice_ps = slice_ls.get_physical_store(); - EXPECT_FALSE(slice_ps->is_future()); - auto root_acc = root_ps->write_accessor(); + EXPECT_FALSE(slice_ps.is_future()); + auto root_acc = root_ps.write_accessor(); root_acc[2] = 42; - auto slice_acc = slice_ps->read_accessor(); + auto slice_acc = slice_ps.read_accessor(); EXPECT_EQ(slice_acc[1], 42); } @@ -87,7 +87,7 @@ void test_inline_map_and_task() auto l_store = runtime->create_store({5}, legate::int64()); { auto p_store = l_store.get_physical_store(); - auto acc = p_store->write_accessor(); + auto acc = p_store.write_accessor(); acc[2] = 42; } auto task = runtime->create_task(context, ADDER, {1}); @@ -95,7 +95,7 @@ void test_inline_map_and_task() task.add_output(l_store); runtime->submit(std::move(task)); auto p_store = l_store.get_physical_store(); - auto acc = p_store->read_accessor(); + auto acc = p_store.read_accessor(); EXPECT_EQ(acc[2], 43); } @@ -104,7 +104,7 @@ void test_inline_map_unmap() auto runtime = legate::Runtime::get_runtime(); auto logical_store = runtime->create_store({1, 3}, legate::int64()); auto store = logical_store.get_physical_store(); - store->unmap(); + store.unmap(); } TEST(InlineMap, MappedRegionsLeak) { test_mapped_regions_leak(); } diff --git a/tests/cpp/integration/machine_scope.cc b/tests/cpp/integration/machine_scope.cc index e1b277c4f3..d4488dfa22 100644 --- a/tests/cpp/integration/machine_scope.cc +++ b/tests/cpp/integration/machine_scope.cc @@ -57,17 +57,17 @@ struct CpuVariantOnlyTask : public legate::LegateTask { void register_tasks() { auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->create_library(library_name); - MultiVariantTask::register_variants(context); - CpuVariantOnlyTask::register_variants(context); + auto library = runtime->create_library(library_name); + MultiVariantTask::register_variants(library); + CpuVariantOnlyTask::register_variants(library); } -void test_scoping(legate::LibraryContext* context) +void test_scoping(legate::Library library) { auto runtime = legate::Runtime::get_runtime(); auto store = runtime->create_store({5, 5}, legate::int64()); auto machine = runtime->get_machine(); - auto task = runtime->create_task(context, MULTI_VARIANT); + auto task = runtime->create_task(library, MULTI_VARIANT); auto part = task.declare_partition(); task.add_output(store, part); task.add_scalar_arg(machine.count()); @@ -75,7 +75,7 @@ void test_scoping(legate::LibraryContext* context) if (machine.count(legate::mapping::TaskTarget::CPU) > 0) { legate::MachineTracker tracker(machine.only(legate::mapping::TaskTarget::CPU)); - auto task_scoped = runtime->create_task(context, MULTI_VARIANT); + auto task_scoped = runtime->create_task(library, MULTI_VARIANT); auto part_scoped = task_scoped.declare_partition(); task_scoped.add_output(store, part_scoped); task_scoped.add_scalar_arg(machine.count(legate::mapping::TaskTarget::CPU)); @@ -84,7 +84,7 @@ void test_scoping(legate::LibraryContext* context) if (machine.count(legate::mapping::TaskTarget::OMP) > 0) { legate::MachineTracker tracker(machine.only(legate::mapping::TaskTarget::OMP)); - auto task_scoped = runtime->create_task(context, MULTI_VARIANT); + auto task_scoped = runtime->create_task(library, MULTI_VARIANT); auto part_scoped = task_scoped.declare_partition(); task_scoped.add_output(store, part_scoped); task_scoped.add_scalar_arg(machine.count(legate::mapping::TaskTarget::OMP)); @@ -93,7 +93,7 @@ void test_scoping(legate::LibraryContext* context) if (machine.count(legate::mapping::TaskTarget::GPU) > 0) { legate::MachineTracker tracker(machine.only(legate::mapping::TaskTarget::GPU)); - auto task_scoped = runtime->create_task(context, MULTI_VARIANT); + auto task_scoped = runtime->create_task(library, MULTI_VARIANT); auto part_scoped = task_scoped.declare_partition(); task_scoped.add_output(store, part_scoped); task_scoped.add_scalar_arg(machine.count(legate::mapping::TaskTarget::GPU)); @@ -101,12 +101,12 @@ void test_scoping(legate::LibraryContext* context) } } -void test_cpu_only(legate::LibraryContext* context) +void test_cpu_only(legate::Library library) { auto runtime = legate::Runtime::get_runtime(); auto store = runtime->create_store({5, 5}, legate::int64()); auto machine = runtime->get_machine(); - auto task = runtime->create_task(context, CPU_VARIANT); + auto task = runtime->create_task(library, CPU_VARIANT); auto part = task.declare_partition(); task.add_output(store, part); task.add_scalar_arg(machine.count(legate::mapping::TaskTarget::CPU)); @@ -114,7 +114,7 @@ void test_cpu_only(legate::LibraryContext* context) if (machine.count(legate::mapping::TaskTarget::CPU) > 0) { legate::MachineTracker tracker(machine.only(legate::mapping::TaskTarget::CPU)); - auto task_scoped = runtime->create_task(context, CPU_VARIANT); + auto task_scoped = runtime->create_task(library, CPU_VARIANT); auto part_scoped = task_scoped.declare_partition(); task_scoped.add_output(store, part_scoped); task_scoped.add_scalar_arg(machine.count(legate::mapping::TaskTarget::CPU)); @@ -131,7 +131,7 @@ void test_cpu_only(legate::LibraryContext* context) // check `slice_machine_for_task` logic if (machine.count(legate::mapping::TaskTarget::GPU) > 0) { legate::MachineTracker tracker(machine.only(legate::mapping::TaskTarget::GPU)); - EXPECT_THROW(runtime->create_task(context, CPU_VARIANT), std::invalid_argument); + EXPECT_THROW(runtime->create_task(library, CPU_VARIANT), std::invalid_argument); } } @@ -140,10 +140,10 @@ TEST(Integration, MachineScope) legate::Core::perform_registration(); auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library(library_name); + auto library = runtime->find_library(library_name); - test_scoping(context); - test_cpu_only(context); + test_scoping(library); + test_cpu_only(library); } } // namespace machine_scope diff --git a/tests/cpp/integration/manual_simple.cc b/tests/cpp/integration/manual_simple.cc index a09de7ace4..eabbab155d 100644 --- a/tests/cpp/integration/manual_simple.cc +++ b/tests/cpp/integration/manual_simple.cc @@ -21,19 +21,19 @@ namespace manualsimple { -void test_auto_task(legate::LibraryContext* context, legate::LogicalStore store) +void test_auto_task(legate::Library library, legate::LogicalStore store) { auto runtime = legate::Runtime::get_runtime(); - auto task = runtime->create_task(context, task::simple::HELLO); + auto task = runtime->create_task(library, task::simple::HELLO); auto part = task.declare_partition(); task.add_output(store, part); runtime->submit(std::move(task)); } -void test_manual_task(legate::LibraryContext* context, legate::LogicalStore store) +void test_manual_task(legate::Library library, legate::LogicalStore store) { auto runtime = legate::Runtime::get_runtime(); - auto task = runtime->create_task(context, task::simple::HELLO, {3, 3}); + auto task = runtime->create_task(library, task::simple::HELLO, {3, 3}); auto part = store.partition_by_tiling({2, 2}); task.add_output(part); runtime->submit(std::move(task)); @@ -43,8 +43,8 @@ void print_store(legate::LogicalStore store) { auto runtime = legate::Runtime::get_runtime(); auto p_store = store.get_physical_store(); - auto acc = p_store->read_accessor(); - auto shape = p_store->shape<2>(); + auto acc = p_store.read_accessor(); + auto shape = p_store.shape<2>(); std::stringstream ss; for (legate::PointInRectIterator<2> it(shape); it.valid(); ++it) ss << acc[*it] << " "; task::simple::logger.print() << ss.str(); @@ -55,12 +55,12 @@ TEST(Integration, ManualSimple) legate::Core::perform_registration(); auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library(task::simple::library_name); + auto library = runtime->find_library(task::simple::library_name); auto store = runtime->create_store({5, 5}, legate::int64()); - test_auto_task(context, store); + test_auto_task(library, store); print_store(store); - test_manual_task(context, store); + test_manual_task(library, store); print_store(store); } diff --git a/tests/cpp/integration/multi_scalar_out.cc b/tests/cpp/integration/multi_scalar_out.cc index 29fddeadf6..01cb9f6fcc 100644 --- a/tests/cpp/integration/multi_scalar_out.cc +++ b/tests/cpp/integration/multi_scalar_out.cc @@ -21,12 +21,12 @@ namespace multiscalarout { -void test_writer_auto(legate::LibraryContext* context, +void test_writer_auto(legate::Library library, legate::LogicalStore scalar1, legate::LogicalStore scalar2) { auto runtime = legate::Runtime::get_runtime(); - auto task = runtime->create_task(context, task::simple::WRITER); + auto task = runtime->create_task(library, task::simple::WRITER); auto part1 = task.declare_partition(); auto part2 = task.declare_partition(); task.add_output(scalar1, part1); @@ -34,13 +34,13 @@ void test_writer_auto(legate::LibraryContext* context, runtime->submit(std::move(task)); } -void test_reducer_auto(legate::LibraryContext* context, +void test_reducer_auto(legate::Library library, legate::LogicalStore scalar1, legate::LogicalStore scalar2, legate::LogicalStore store) { auto runtime = legate::Runtime::get_runtime(); - auto task = runtime->create_task(context, task::simple::REDUCER); + auto task = runtime->create_task(library, task::simple::REDUCER); auto part1 = task.declare_partition(); auto part2 = task.declare_partition(); auto part3 = task.declare_partition(); @@ -50,12 +50,12 @@ void test_reducer_auto(legate::LibraryContext* context, runtime->submit(std::move(task)); } -void test_reducer_manual(legate::LibraryContext* context, +void test_reducer_manual(legate::Library library, legate::LogicalStore scalar1, legate::LogicalStore scalar2) { auto runtime = legate::Runtime::get_runtime(); - auto task = runtime->create_task(context, task::simple::REDUCER, legate::Shape({2})); + auto task = runtime->create_task(library, task::simple::REDUCER, legate::Shape({2})); task.add_reduction(scalar1, legate::ReductionOpKind::ADD); task.add_reduction(scalar2, legate::ReductionOpKind::MUL); runtime->submit(std::move(task)); @@ -66,8 +66,8 @@ void print_stores(legate::LogicalStore scalar1, legate::LogicalStore scalar2) auto runtime = legate::Runtime::get_runtime(); auto p_scalar1 = scalar1.get_physical_store(); auto p_scalar2 = scalar2.get_physical_store(); - auto acc1 = p_scalar1->read_accessor(); - auto acc2 = p_scalar2->read_accessor(); + auto acc1 = p_scalar1.read_accessor(); + auto acc2 = p_scalar2.read_accessor(); std::stringstream ss; ss << static_cast(acc1[{0, 0}]) << " " << acc2[{0, 0, 0}]; task::simple::logger.print() << ss.str(); @@ -78,16 +78,16 @@ TEST(Integration, MultiScalarOut) legate::Core::perform_registration(); auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library(task::simple::library_name); + auto library = runtime->find_library(task::simple::library_name); auto scalar1 = runtime->create_store({1, 1}, legate::int8(), true); auto scalar2 = runtime->create_store({1, 1, 1}, legate::int32(), true); auto store = runtime->create_store({10}, legate::int64()); - test_writer_auto(context, scalar1, scalar2); + test_writer_auto(library, scalar1, scalar2); print_stores(scalar1, scalar2); - test_reducer_auto(context, scalar1, scalar2, store); + test_reducer_auto(library, scalar1, scalar2, store); print_stores(scalar1, scalar2); - test_reducer_manual(context, scalar1, scalar2); + test_reducer_manual(library, scalar1, scalar2); print_stores(scalar1, scalar2); } diff --git a/tests/cpp/integration/nccl.cu b/tests/cpp/integration/nccl.cu index a233515349..dab4b181b4 100644 --- a/tests/cpp/integration/nccl.cu +++ b/tests/cpp/integration/nccl.cu @@ -111,8 +111,8 @@ TEST(Integration, NCCL) { legate::Core::perform_registration(); - auto runtime = legate::Runtime::get_runtime(); - auto& machine = runtime->get_machine(); + auto runtime = legate::Runtime::get_runtime(); + auto machine = runtime->get_machine(); if (machine.count(legate::mapping::TaskTarget::GPU) == 0) return; legate::MachineTracker tracker(machine.only(legate::mapping::TaskTarget::GPU)); diff --git a/tests/cpp/integration/provenance.cc b/tests/cpp/integration/provenance.cc index 859268bd07..a9f5815c29 100644 --- a/tests/cpp/integration/provenance.cc +++ b/tests/cpp/integration/provenance.cc @@ -35,8 +35,8 @@ struct ProvenanceTask : public legate::LegateTask { void register_tasks() { auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->create_library(library_name); - ProvenanceTask::register_variants(context); + auto library = runtime->create_library(library_name); + ProvenanceTask::register_variants(library); } /*static*/ void ProvenanceTask::cpu_variant(legate::TaskContext& context) @@ -46,43 +46,43 @@ void register_tasks() EXPECT_TRUE(provenance.find(scalar) != std::string::npos); } -void test_manual_provenance(legate::LibraryContext* context) +void test_manual_provenance(legate::Library library) { auto runtime = legate::Runtime::get_runtime(); std::string provenance = "test_manual_provenance"; runtime->impl()->provenance_manager()->set_provenance(provenance); // auto task - auto task = runtime->create_task(context, PROVENANCE); + auto task = runtime->create_task(library, PROVENANCE); task.add_scalar_arg(legate::Scalar(provenance)); runtime->submit(std::move(task)); } -void test_push_provenance(legate::LibraryContext* context) +void test_push_provenance(legate::Library library) { auto runtime = legate::Runtime::get_runtime(); std::string provenance = "test_push_provenance"; runtime->impl()->provenance_manager()->push_provenance(provenance); EXPECT_EQ(runtime->impl()->provenance_manager()->get_provenance(), provenance); // auto task - auto task = runtime->create_task(context, PROVENANCE); + auto task = runtime->create_task(library, PROVENANCE); task.add_scalar_arg(legate::Scalar(provenance)); runtime->submit(std::move(task)); } -void test_pop_provenance(legate::LibraryContext* context) +void test_pop_provenance(legate::Library library) { auto runtime = legate::Runtime::get_runtime(); runtime->impl()->provenance_manager()->clear_all(); runtime->impl()->provenance_manager()->push_provenance("some provenance for provenance task"); runtime->impl()->provenance_manager()->pop_provenance(); // auto task - auto task = runtime->create_task(context, PROVENANCE); + auto task = runtime->create_task(library, PROVENANCE); std::string provenance = ""; task.add_scalar_arg(legate::Scalar(provenance)); runtime->submit(std::move(task)); } -void test_underflow(legate::LibraryContext* context) +void test_underflow(legate::Library library) { auto runtime = legate::Runtime::get_runtime(); runtime->impl()->provenance_manager()->clear_all(); @@ -90,48 +90,48 @@ void test_underflow(legate::LibraryContext* context) EXPECT_THROW(runtime->impl()->provenance_manager()->pop_provenance(), std::runtime_error); } -void test_clear_provenance(legate::LibraryContext* context) +void test_clear_provenance(legate::Library library) { auto runtime = legate::Runtime::get_runtime(); runtime->impl()->provenance_manager()->push_provenance("provenance for provenance task"); runtime->impl()->provenance_manager()->push_provenance("another provenance"); runtime->impl()->provenance_manager()->clear_all(); // auto task - auto task = runtime->create_task(context, PROVENANCE); + auto task = runtime->create_task(library, PROVENANCE); std::string provenance = ""; task.add_scalar_arg(legate::Scalar(provenance)); runtime->submit(std::move(task)); } -void test_provenance_tracker(legate::LibraryContext* context) +void test_provenance_tracker(legate::Library library) { legate::ProvenanceTracker track(std::string(__FILE__) + ":" + std::to_string(__LINE__)); auto runtime = legate::Runtime::get_runtime(); // auto task - auto task = runtime->create_task(context, PROVENANCE); + auto task = runtime->create_task(library, PROVENANCE); std::string provenance = "provenance.cc:108"; task.add_scalar_arg(legate::Scalar(provenance)); runtime->submit(std::move(task)); } -void test_nested_provenance_tracker(legate::LibraryContext* context) +void test_nested_provenance_tracker(legate::Library library) { legate::ProvenanceTracker track(std::string(__FILE__) + ":" + std::to_string(__LINE__)); - test_provenance_tracker(context); + test_provenance_tracker(library); // The provenance string used by test_provenance_tracker should be popped out at this point auto runtime = legate::Runtime::get_runtime(); - auto task = runtime->create_task(context, PROVENANCE); + auto task = runtime->create_task(library, PROVENANCE); std::string provenance = "provenance.cc:119"; task.add_scalar_arg(legate::Scalar(provenance)); runtime->submit(std::move(task)); } -void test_manual_tracker(legate::LibraryContext* context) +void test_manual_tracker(legate::Library library) { legate::ProvenanceTracker track("manual provenance through tracker"); auto runtime = legate::Runtime::get_runtime(); // auto task - auto task = runtime->create_task(context, PROVENANCE); + auto task = runtime->create_task(library, PROVENANCE); task.add_scalar_arg(legate::Scalar(track.get_current_provenance())); runtime->submit(std::move(task)); } @@ -141,16 +141,16 @@ TEST(Integration, Provenance) legate::Core::perform_registration(); auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library(library_name); - - test_manual_provenance(context); - test_push_provenance(context); - test_pop_provenance(context); - test_underflow(context); - test_clear_provenance(context); - test_provenance_tracker(context); - test_nested_provenance_tracker(context); - test_manual_tracker(context); + auto library = runtime->find_library(library_name); + + test_manual_provenance(library); + test_push_provenance(library); + test_pop_provenance(library); + test_underflow(library); + test_clear_provenance(library); + test_provenance_tracker(library); + test_nested_provenance_tracker(library); + test_manual_tracker(library); } } // namespace provenance diff --git a/tests/cpp/integration/weighted.cc b/tests/cpp/integration/weighted.cc index 8804a38a84..082155da68 100644 --- a/tests/cpp/integration/weighted.cc +++ b/tests/cpp/integration/weighted.cc @@ -60,16 +60,16 @@ struct Tester : public legate::LegateTask { void prepare() { auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->create_library(library_name); - Initializer::register_variants(context, INIT); - Tester::register_variants(context, CHECK); + auto library = runtime->create_library(library_name); + Initializer::register_variants(library, INIT); + Tester::register_variants(library, CHECK); } void initialize(legate::Runtime* runtime, - legate::LibraryContext* context, + legate::Library library, const std::vector& outputs) { - auto task = runtime->create_task(context, INIT, {NUM_TASKS}); + auto task = runtime->create_task(library, INIT, {NUM_TASKS}); std::vector parts; for (auto& output : outputs) task.add_output(output); @@ -78,10 +78,10 @@ void initialize(legate::Runtime* runtime, } void check(legate::Runtime* runtime, - legate::LibraryContext* context, + legate::Library library, const std::vector& inputs) { - auto task = runtime->create_task(context, CHECK); + auto task = runtime->create_task(library, CHECK); for (auto& input : inputs) { auto part_in = task.declare_partition(); @@ -98,13 +98,13 @@ void check(legate::Runtime* runtime, void test_weighted(uint32_t num_stores) { auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library(library_name); + auto library = runtime->find_library(library_name); std::vector stores; for (uint32_t idx = 0; idx < num_stores; ++idx) stores.push_back(runtime->create_store(legate::int32())); - initialize(runtime, context, stores); - check(runtime, context, stores); + initialize(runtime, library, stores); + check(runtime, library, stores); } } // namespace diff --git a/tests/cpp/unit/library.cc b/tests/cpp/unit/library.cc index 791687a07c..22094c3626 100644 --- a/tests/cpp/unit/library.cc +++ b/tests/cpp/unit/library.cc @@ -21,8 +21,9 @@ TEST(Library, Create) { auto* runtime = legate::Runtime::get_runtime(); - auto* lib = runtime->create_library("libA"); + auto lib = runtime->create_library("libA"); EXPECT_EQ(lib, runtime->find_library("libA")); + EXPECT_EQ(lib, runtime->maybe_find_library("libA").value()); } TEST(Library, FindOrCreate) @@ -33,22 +34,22 @@ TEST(Library, FindOrCreate) config.max_tasks = 1; bool created = false; - auto* p_lib1 = runtime->find_or_create_library("libA", config, nullptr, &created); + auto p_lib1 = runtime->find_or_create_library("libA", config, nullptr, &created); EXPECT_TRUE(created); config.max_tasks = 2; - auto* p_lib2 = runtime->find_or_create_library("libA", config, nullptr, &created); + auto p_lib2 = runtime->find_or_create_library("libA", config, nullptr, &created); EXPECT_FALSE(created); EXPECT_EQ(p_lib1, p_lib2); - EXPECT_TRUE(p_lib2->valid_task_id(p_lib2->get_task_id(0))); - EXPECT_FALSE(p_lib2->valid_task_id(p_lib2->get_task_id(1))); + EXPECT_TRUE(p_lib2.valid_task_id(p_lib2.get_task_id(0))); + EXPECT_FALSE(p_lib2.valid_task_id(p_lib2.get_task_id(1))); } TEST(Library, FindNonExistent) { auto* runtime = legate::Runtime::get_runtime(); - EXPECT_EQ(runtime->find_library("libB", true /*can_fail*/), nullptr); + EXPECT_THROW(runtime->find_library("libB"), std::out_of_range); - EXPECT_THROW(runtime->find_library("libB", false /*can_fail*/), std::out_of_range); + EXPECT_EQ(runtime->maybe_find_library("libB"), std::nullopt); } diff --git a/tests/cpp/unit/machine.cc b/tests/cpp/unit/machine.cc index dd67cdc4ca..4903f68ffc 100644 --- a/tests/cpp/unit/machine.cc +++ b/tests/cpp/unit/machine.cc @@ -20,7 +20,10 @@ #include #include -#include "core/utilities/buffer_builder.h" +#include "core/mapping/detail/machine.h" +#include "core/mapping/machine.h" +#include "core/utilities/deserializer.h" +#include "core/utilities/detail/buffer_builder.h" #include "legate.h" namespace unit { @@ -35,7 +38,7 @@ TEST(Machine, ProcessorRange) EXPECT_EQ(range.low, 1); EXPECT_EQ(range.high, 3); EXPECT_EQ(range.count(), 2); - EXPECT_EQ(range.get_node_range(), std::make_pair(uint32_t(1), uint32_t(3))); + EXPECT_EQ(range.get_node_range(), legate::mapping::NodeRange({1, 3})); } // create empty @@ -46,7 +49,7 @@ TEST(Machine, ProcessorRange) EXPECT_EQ(range.low, 0); EXPECT_EQ(range.high, 0); EXPECT_EQ(range.count(), 0); - EXPECT_THROW(range.get_node_range(), std::runtime_error); + EXPECT_THROW(range.get_node_range(), std::invalid_argument); } // check defaults @@ -62,7 +65,7 @@ TEST(Machine, ProcessorRange) // get_node_range { legate::mapping::ProcessorRange range(0, 7, 2); - EXPECT_EQ(range.get_node_range(), std::make_pair(uint32_t(0), uint32_t(4))); + EXPECT_EQ(range.get_node_range(), legate::mapping::NodeRange({0, 4})); } // intersection nonempty @@ -108,7 +111,7 @@ TEST(Machine, MachineDesc) { // test empty MachineDesc { - legate::mapping::MachineDesc machine; + legate::mapping::detail::Machine machine; EXPECT_EQ(machine.preferred_target, legate::mapping::TaskTarget::CPU); EXPECT_EQ(machine.count(), 0); EXPECT_EQ(machine.count(legate::mapping::TaskTarget::GPU), 0); @@ -116,7 +119,7 @@ TEST(Machine, MachineDesc) EXPECT_EQ(machine.processor_range(legate::mapping::TaskTarget::GPU), legate::mapping::ProcessorRange(0, 0, 1)); EXPECT_EQ(machine.slice(0, 1), - legate::mapping::MachineDesc( + legate::mapping::detail::Machine( {{legate::mapping::TaskTarget::CPU, legate::mapping::ProcessorRange()}})); } @@ -127,43 +130,43 @@ TEST(Machine, MachineDesc) // test equal { - legate::mapping::MachineDesc machine1({{legate::mapping::TaskTarget::CPU, cpu_range}, - {legate::mapping::TaskTarget::OMP, omp_range}}); - legate::mapping::MachineDesc machine2({{legate::mapping::TaskTarget::CPU, cpu_range}, - {legate::mapping::TaskTarget::OMP, omp_range}}); + legate::mapping::detail::Machine machine1({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::OMP, omp_range}}); + legate::mapping::detail::Machine machine2({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::OMP, omp_range}}); EXPECT_EQ(machine1, machine2); - legate::mapping::MachineDesc machine3; + legate::mapping::detail::Machine machine3; EXPECT_NE(machine1, machine3); } // test preferred_target { - legate::mapping::MachineDesc machine1({{legate::mapping::TaskTarget::CPU, cpu_range}}); + legate::mapping::detail::Machine machine1({{legate::mapping::TaskTarget::CPU, cpu_range}}); EXPECT_EQ(machine1.preferred_target, legate::mapping::TaskTarget::CPU); - legate::mapping::MachineDesc machine2({{legate::mapping::TaskTarget::CPU, cpu_range}, - {legate::mapping::TaskTarget::OMP, omp_range}}); + legate::mapping::detail::Machine machine2({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::OMP, omp_range}}); EXPECT_EQ(machine2.preferred_target, legate::mapping::TaskTarget::OMP); - legate::mapping::MachineDesc machine3({{legate::mapping::TaskTarget::CPU, cpu_range}, - {legate::mapping::TaskTarget::OMP, omp_range}, - {legate::mapping::TaskTarget::GPU, gpu_range}}); + legate::mapping::detail::Machine machine3({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::OMP, omp_range}, + {legate::mapping::TaskTarget::GPU, gpu_range}}); EXPECT_EQ(machine3.preferred_target, legate::mapping::TaskTarget::GPU); } // test processor_range { - legate::mapping::MachineDesc machine1({{legate::mapping::TaskTarget::CPU, cpu_range}, - {legate::mapping::TaskTarget::OMP, omp_range}, - {legate::mapping::TaskTarget::GPU, gpu_range}}); + legate::mapping::detail::Machine machine1({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::OMP, omp_range}, + {legate::mapping::TaskTarget::GPU, gpu_range}}); EXPECT_EQ(machine1.processor_range(), gpu_range); EXPECT_EQ(machine1.processor_range(legate::mapping::TaskTarget::CPU), cpu_range); EXPECT_EQ(machine1.processor_range(legate::mapping::TaskTarget::OMP), omp_range); EXPECT_EQ(machine1.processor_range(legate::mapping::TaskTarget::GPU), gpu_range); - legate::mapping::MachineDesc machine2({{legate::mapping::TaskTarget::CPU, cpu_range}, - {legate::mapping::TaskTarget::OMP, omp_range}}); + legate::mapping::detail::Machine machine2({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::OMP, omp_range}}); EXPECT_EQ(machine2.processor_range(), omp_range); EXPECT_EQ(machine2.processor_range(legate::mapping::TaskTarget::CPU), cpu_range); EXPECT_EQ(machine2.processor_range(legate::mapping::TaskTarget::GPU), @@ -172,23 +175,23 @@ TEST(Machine, MachineDesc) // test valid_targets { - legate::mapping::MachineDesc machine1({{legate::mapping::TaskTarget::CPU, cpu_range}, - {legate::mapping::TaskTarget::OMP, omp_range}, - {legate::mapping::TaskTarget::GPU, gpu_range}}); + legate::mapping::detail::Machine machine1({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::OMP, omp_range}, + {legate::mapping::TaskTarget::GPU, gpu_range}}); auto valid_targets1 = machine1.valid_targets(); EXPECT_EQ(valid_targets1.size(), 3); - legate::mapping::MachineDesc machine2({{legate::mapping::TaskTarget::CPU, cpu_range}, - {legate::mapping::TaskTarget::OMP, omp_range}}); + legate::mapping::detail::Machine machine2({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::OMP, omp_range}}); auto valid_targets2 = machine2.valid_targets(); EXPECT_EQ(valid_targets2.size(), 2); } // test valid_targets_except { - legate::mapping::MachineDesc machine({{legate::mapping::TaskTarget::CPU, cpu_range}, - {legate::mapping::TaskTarget::OMP, omp_range}, - {legate::mapping::TaskTarget::GPU, gpu_range}}); + legate::mapping::detail::Machine machine({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::OMP, omp_range}, + {legate::mapping::TaskTarget::GPU, gpu_range}}); std::set exclude_targets; auto valid_targets1 = machine.valid_targets_except(exclude_targets); @@ -209,9 +212,9 @@ TEST(Machine, MachineDesc) // test count { - legate::mapping::MachineDesc machine({{legate::mapping::TaskTarget::CPU, cpu_range}, - {legate::mapping::TaskTarget::OMP, omp_range}, - {legate::mapping::TaskTarget::GPU, gpu_range}}); + legate::mapping::detail::Machine machine({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::OMP, omp_range}, + {legate::mapping::TaskTarget::GPU, gpu_range}}); EXPECT_EQ(machine.count(), 3); EXPECT_EQ(machine.count(legate::mapping::TaskTarget::CPU), 2); EXPECT_EQ(machine.count(legate::mapping::TaskTarget::OMP), 3); @@ -219,21 +222,21 @@ TEST(Machine, MachineDesc) // test_pack { - legate::BufferBuilder buf; - legate::mapping::MachineDesc machine({{legate::mapping::TaskTarget::CPU, cpu_range}, - {legate::mapping::TaskTarget::GPU, gpu_range}}); + legate::detail::BufferBuilder buf; + legate::mapping::detail::Machine machine({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::GPU, gpu_range}}); machine.pack(buf); auto legion_buffer = buf.to_legion_buffer(); legate::BaseDeserializer dez(legion_buffer.get_ptr(), legion_buffer.get_size()); - auto machine_unpack = dez.unpack(); + auto machine_unpack = dez.unpack(); EXPECT_EQ(machine_unpack, machine); } // test only { - legate::mapping::MachineDesc machine({{legate::mapping::TaskTarget::CPU, cpu_range}, - {legate::mapping::TaskTarget::GPU, gpu_range}}); + legate::mapping::detail::Machine machine({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::GPU, gpu_range}}); auto machine1 = machine.only(legate::mapping::TaskTarget::CPU); auto valid_targets1 = machine1.valid_targets(); @@ -242,9 +245,9 @@ TEST(Machine, MachineDesc) EXPECT_EQ(machine1.preferred_target, legate::mapping::TaskTarget::CPU); EXPECT_EQ(machine1.processor_range().per_node_count, 4); - legate::mapping::MachineDesc machine2({{legate::mapping::TaskTarget::CPU, cpu_range}, - {legate::mapping::TaskTarget::OMP, omp_range}, - {legate::mapping::TaskTarget::GPU, gpu_range}}); + legate::mapping::detail::Machine machine2({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::OMP, omp_range}, + {legate::mapping::TaskTarget::GPU, gpu_range}}); auto machine3 = machine2.only({legate::mapping::TaskTarget::GPU, legate::mapping::TaskTarget::CPU}); EXPECT_EQ(machine3, machine); @@ -252,9 +255,9 @@ TEST(Machine, MachineDesc) // test slice { - legate::mapping::MachineDesc machine1({{legate::mapping::TaskTarget::CPU, cpu_range}, - {legate::mapping::TaskTarget::GPU, gpu_range}}); - legate::mapping::MachineDesc expected( + legate::mapping::detail::Machine machine1({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::GPU, gpu_range}}); + legate::mapping::detail::Machine expected( {{legate::mapping::TaskTarget::GPU, gpu_range.slice(0, 1)}}); EXPECT_EQ(machine1.slice(0, 1), expected); @@ -262,7 +265,7 @@ TEST(Machine, MachineDesc) EXPECT_EQ(new_machine1.preferred_target, legate::mapping::TaskTarget::GPU); EXPECT_EQ(new_machine1.processor_range().count(), 2); - legate::mapping::MachineDesc machine2({{legate::mapping::TaskTarget::GPU, gpu_range}}); + legate::mapping::detail::Machine machine2({{legate::mapping::TaskTarget::GPU, gpu_range}}); auto new_machine2 = machine2.slice(0, 2); EXPECT_EQ(new_machine2.preferred_target, legate::mapping::TaskTarget::GPU); EXPECT_EQ(new_machine2.processor_range().count(), 2); @@ -270,8 +273,8 @@ TEST(Machine, MachineDesc) // test operator[] { - legate::mapping::MachineDesc machine({{legate::mapping::TaskTarget::CPU, cpu_range}, - {legate::mapping::TaskTarget::GPU, gpu_range}}); + legate::mapping::detail::Machine machine({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::GPU, gpu_range}}); auto machine1 = machine[legate::mapping::TaskTarget::GPU]; auto valid_targets1 = machine1.valid_targets(); @@ -283,22 +286,22 @@ TEST(Machine, MachineDesc) // test intersection { - legate::mapping::MachineDesc machine1( + legate::mapping::detail::Machine machine1( {{legate::mapping::TaskTarget::CPU, cpu_range.slice(1, 2)}, {legate::mapping::TaskTarget::GPU, gpu_range}}); - legate::mapping::MachineDesc machine2( + legate::mapping::detail::Machine machine2( {{legate::mapping::TaskTarget::CPU, cpu_range}, {legate::mapping::TaskTarget::OMP, omp_range}, {legate::mapping::TaskTarget::GPU, gpu_range.slice(0, 1)}}); - legate::mapping::MachineDesc machine3( + legate::mapping::detail::Machine machine3( {{legate::mapping::TaskTarget::CPU, cpu_range.slice(1, 2)}, {legate::mapping::TaskTarget::GPU, gpu_range.slice(0, 1)}}); EXPECT_EQ(machine3, (machine1 & machine2)); EXPECT_EQ(machine2.only(legate::mapping::TaskTarget::OMP) & machine1, - legate::mapping::MachineDesc()); + legate::mapping::detail::Machine()); } } diff --git a/tests/cpp/unit/registration.cc b/tests/cpp/unit/registration.cc index 18a651380b..3946b8c293 100644 --- a/tests/cpp/unit/registration.cc +++ b/tests/cpp/unit/registration.cc @@ -37,7 +37,7 @@ struct GPUVariantTask : public legate::LegateTask> { void test_duplicates() { auto* runtime = legate::Runtime::get_runtime(); - auto* library = runtime->create_library("libA"); + auto library = runtime->create_library("libA"); test_registration::CPUVariantTask<0>::register_variants(library); EXPECT_THROW(test_registration::CPUVariantTask<0>::register_variants(library), std::invalid_argument); @@ -48,7 +48,7 @@ void test_out_of_bounds_task_id() legate::ResourceConfig config; config.max_tasks = 1; auto* runtime = legate::Runtime::get_runtime(); - auto* library = runtime->create_library("libA", config); + auto library = runtime->create_library("libA", config); EXPECT_THROW(test_registration::CPUVariantTask<1>::register_variants(library), std::out_of_range); } diff --git a/tests/cpp/unit/scalar.cc b/tests/cpp/unit/scalar.cc index af49cdde30..8386c753f6 100644 --- a/tests/cpp/unit/scalar.cc +++ b/tests/cpp/unit/scalar.cc @@ -16,8 +16,9 @@ #include -#include "core/utilities/buffer_builder.h" -#include "legate.h" +#include "core/data/detail/scalar.h" +#include "core/utilities/deserializer.h" +#include "core/utilities/detail/buffer_builder.h" namespace scalar_test { @@ -68,9 +69,9 @@ TEST(ScalarUnit, CreateWithObject) // constructor with Scalar object legate::Scalar scalar1(INT32_VALUE); legate::Scalar scalar2(scalar1); - EXPECT_EQ(scalar2.type().code, scalar1.type().code); + EXPECT_EQ(scalar2.type().code(), scalar1.type().code()); EXPECT_EQ(scalar2.size(), scalar1.size()); - EXPECT_EQ(scalar2.size(), legate::uint32()->size()); + EXPECT_EQ(scalar2.size(), legate::uint32().size()); EXPECT_NE(scalar2.ptr(), nullptr); EXPECT_EQ(scalar2.value(), scalar1.value()); EXPECT_EQ(scalar2.values().size(), scalar1.values().size()); @@ -86,8 +87,8 @@ TEST(ScalarUnit, CreateSharedScalar) auto data_vec = std::vector(DATA_SIZE, UINT64_VALUE); const auto* data = data_vec.data(); legate::Scalar scalar(legate::uint64(), data); - EXPECT_EQ(scalar.type().code, legate::Type::Code::UINT64); - EXPECT_EQ(scalar.size(), legate::uint64()->size()); + EXPECT_EQ(scalar.type().code(), legate::Type::Code::UINT64); + EXPECT_EQ(scalar.size(), legate::uint64().size()); EXPECT_EQ(scalar.ptr(), data); EXPECT_EQ(scalar.value(), UINT64_VALUE); @@ -105,7 +106,7 @@ template void checkType(T value) { legate::Scalar scalar(value); - EXPECT_EQ(scalar.type().code, legate::legate_type_code_of); + EXPECT_EQ(scalar.type().code(), legate::legate_type_code_of); EXPECT_EQ(scalar.size(), sizeof(T)); EXPECT_EQ(scalar.value(), value); EXPECT_EQ(scalar.values().size(), 1); @@ -135,9 +136,9 @@ TEST(ScalarUnit, CreateWithVector) // constructor with arrays std::vector scalar_data{INT32_VALUE, INT32_VALUE}; legate::Scalar scalar(scalar_data); - EXPECT_EQ(scalar.type().code, legate::Type::Code::FIXED_ARRAY); + EXPECT_EQ(scalar.type().code(), legate::Type::Code::FIXED_ARRAY); auto fixed_type = legate::fixed_array_type(legate::int32(), scalar_data.size()); - EXPECT_EQ(scalar.size(), fixed_type->size()); + EXPECT_EQ(scalar.size(), fixed_type.size()); // check values here. Note: no value allowed for a fixed arrays scalar std::vector data_vec = {INT32_VALUE, INT32_VALUE}; @@ -158,7 +159,7 @@ TEST(ScalarUnit, CreateWithString) // constructor with string auto inputString = STRING_VALUE; legate::Scalar scalar(inputString); - EXPECT_EQ(scalar.type().code, legate::Type::Code::STRING); + EXPECT_EQ(scalar.type().code(), legate::Type::Code::STRING); auto expectedSize = sizeof(uint32_t) + sizeof(char) * inputString.size(); EXPECT_EQ(scalar.size(), expectedSize); EXPECT_NE(scalar.ptr(), inputString.data()); @@ -182,7 +183,7 @@ TEST(ScalarUnit, CreateWithStructType) PaddingStructData structData = {BOOL_VALUE, INT32_VALUE, UINT64_VALUE}; legate::Scalar scalar( structData, legate::struct_type(true, legate::bool_(), legate::int32(), legate::uint64())); - EXPECT_EQ(scalar.type().code, legate::Type::Code::STRUCT); + EXPECT_EQ(scalar.type().code(), legate::Type::Code::STRUCT); EXPECT_EQ(scalar.size(), sizeof(PaddingStructData)); EXPECT_NE(scalar.ptr(), nullptr); @@ -207,7 +208,7 @@ TEST(ScalarUnit, CreateWithStructType) NoPaddingStructData structData = {BOOL_VALUE, INT32_VALUE, UINT64_VALUE}; legate::Scalar scalar( structData, legate::struct_type(false, legate::bool_(), legate::int32(), legate::uint64())); - EXPECT_EQ(scalar.type().code, legate::Type::Code::STRUCT); + EXPECT_EQ(scalar.type().code(), legate::Type::Code::STRUCT); EXPECT_EQ(scalar.size(), sizeof(NoPaddingStructData)); EXPECT_NE(scalar.ptr(), nullptr); @@ -233,7 +234,7 @@ TEST(ScalarUnit, OperatorEqual) legate::Scalar scalar1(INT32_VALUE); legate::Scalar scalar2(UINT64_VALUE); scalar2 = scalar1; - EXPECT_EQ(scalar2.type().code, scalar1.type().code); + EXPECT_EQ(scalar2.type().code(), scalar1.type().code()); EXPECT_EQ(scalar2.size(), scalar2.size()); EXPECT_EQ(scalar2.value(), scalar1.value()); EXPECT_EQ(scalar2.values().size(), scalar1.values().size()); @@ -241,14 +242,14 @@ TEST(ScalarUnit, OperatorEqual) void checkPack(const legate::Scalar& scalar) { - legate::BufferBuilder buf; - scalar.pack(buf); + legate::detail::BufferBuilder buf; + scalar.impl()->pack(buf); auto legion_buffer = buf.to_legion_buffer(); EXPECT_NE(legion_buffer.get_ptr(), nullptr); legate::BaseDeserializer deserializer(legion_buffer.get_ptr(), legion_buffer.get_size()); auto scalar_unpack = deserializer._unpack_scalar(); - EXPECT_EQ(scalar_unpack.type().code, scalar.type().code); + EXPECT_EQ(scalar_unpack.type().code(), scalar.type().code()); EXPECT_EQ(scalar_unpack.size(), scalar.size()); } diff --git a/tests/cpp/unit/store.cc b/tests/cpp/unit/store.cc index b57cfa09a0..095b6e3180 100644 --- a/tests/cpp/unit/store.cc +++ b/tests/cpp/unit/store.cc @@ -32,7 +32,7 @@ TEST(Store, Creation) EXPECT_FALSE(store.unbound()); EXPECT_EQ(store.dim(), 2); EXPECT_EQ(store.extents(), (std::vector{4, 4})); - EXPECT_EQ(store.type(), *legate::int64()); + EXPECT_EQ(store.type(), legate::int64()); EXPECT_FALSE(store.transformed()); } @@ -42,7 +42,7 @@ TEST(Store, Creation) auto store = runtime->create_store(legate::int64()); EXPECT_TRUE(store.unbound()); EXPECT_EQ(store.dim(), 1); - EXPECT_EQ(store.type(), *legate::int64()); + EXPECT_EQ(store.type(), legate::int64()); EXPECT_FALSE(store.transformed()); EXPECT_THROW(store.extents(), std::invalid_argument); } diff --git a/tests/integration/scoping/examples/test_scoping.py b/tests/integration/scoping/examples/test_scoping.py index c6a73dc0c4..466eb7dcaf 100644 --- a/tests/integration/scoping/examples/test_scoping.py +++ b/tests/integration/scoping/examples/test_scoping.py @@ -59,7 +59,7 @@ def test_scoping(): task.add_scalar_arg(m.count(ProcessorKind.CPU), ty.int32) task.execute() - store.get_inline_allocation(user_context) + store.get_inline_allocation() def test_cpu_only(): @@ -90,7 +90,7 @@ def test_cpu_only(): user_lib.shared_object.CPU_VARIANT_ONLY ) - store.get_inline_allocation(user_context) + store.get_inline_allocation() if __name__ == "__main__": diff --git a/tests/unit/legate/core/test_invalid_scalar_arg.py b/tests/unit/legate/core/test_invalid_scalar_arg.py index 9c0793bf0b..dc1a6b79e4 100644 --- a/tests/unit/legate/core/test_invalid_scalar_arg.py +++ b/tests/unit/legate/core/test_invalid_scalar_arg.py @@ -22,7 +22,7 @@ class Test_scalar_arg: def test_unimplemented_types(self) -> None: context = get_legate_runtime().core_context # Create a task object only to test validation logic - task = context.create_auto_task(0) + task = context.create_auto_task(1) with pytest.raises(NotImplementedError): task.add_scalar_arg(None, ty.struct_type([ty.int8])) with pytest.raises(NotImplementedError): @@ -35,14 +35,14 @@ def test_unimplemented_types(self) -> None: def test_scalar_arg_with_array_type(self) -> None: context = get_legate_runtime().core_context # Create a task object only to test validation logic - task = context.create_auto_task(0) + task = context.create_auto_task(1) with pytest.raises(ValueError): task.add_scalar_arg(1, ty.array_type(ty.int8, 1)) def test_array_size_mismatch(self) -> None: context = get_legate_runtime().core_context # Create a task object only to test validation logic - task = context.create_auto_task(0) + task = context.create_auto_task(1) with pytest.raises(ValueError): task.add_scalar_arg((1, 2, 3), ty.array_type(ty.int8, 1)) From 86adf29ea30a1dee1274736add02f07d732ba6a8 Mon Sep 17 00:00:00 2001 From: Vladislav Zhurba Date: Tue, 25 Jul 2023 11:55:08 -0700 Subject: [PATCH 0169/1425] Add custom test runnner with configurable mpi rank * Configure mpirun output options * Revert "Make sure registration callbacks have completed before proceeding" This reverts commit ca0ef785eda83706ff55c788423e4bcf43de268f. * Port multi-process fixes from !61 * Make sure registration callbacks have completed before proceeding * Add support for legion cmd flags to C++ tests * Add custom test runnner with configurable mpi rank Co-authored-by: Manolis Papadakis See merge request legate/legate.core.internal!83 --- tests/cpp/CMakeLists.txt | 1 + .../cpp/integration/alignment_constraints.cc | 4 - .../cpp/integration/broadcast_constraints.cc | 4 - tests/cpp/integration/cpu_communicator.cc | 4 - tests/cpp/integration/exception.cc | 4 - tests/cpp/integration/nccl.cu | 4 - tests/cpp/integration/weighted.cc | 4 - tests/cpp/run.py | 165 ++++++++++++++++++ tests/cpp/run.sh | 13 +- 9 files changed, 177 insertions(+), 26 deletions(-) create mode 100755 tests/cpp/run.py diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt index 4bc45b35e3..e1139d3d16 100644 --- a/tests/cpp/CMakeLists.txt +++ b/tests/cpp/CMakeLists.txt @@ -66,6 +66,7 @@ target_link_libraries(cpp_tests PRIVATE NCCL::NCCL) endif() include(GoogleTest) +set_property(TARGET cpp_tests PROPERTY ENABLE_EXPORTS ON) gtest_discover_tests(cpp_tests DISCOVERY_MODE PRE_TEST WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) include(CPack) diff --git a/tests/cpp/integration/alignment_constraints.cc b/tests/cpp/integration/alignment_constraints.cc index c35ddaeea6..e0d2ab8d6c 100644 --- a/tests/cpp/integration/alignment_constraints.cc +++ b/tests/cpp/integration/alignment_constraints.cc @@ -20,8 +20,6 @@ namespace alignment_constraints { -namespace { - static const char* library_name = "test_alignment_constraints"; enum TaskIDs { @@ -218,8 +216,6 @@ void test_invalid_alignment() EXPECT_THROW(runtime->submit(std::move(task)), std::invalid_argument); } -} // namespace - TEST(Alignment, Basic) { legate::Core::perform_registration(); diff --git a/tests/cpp/integration/broadcast_constraints.cc b/tests/cpp/integration/broadcast_constraints.cc index 73bbe43527..ac217cfafd 100644 --- a/tests/cpp/integration/broadcast_constraints.cc +++ b/tests/cpp/integration/broadcast_constraints.cc @@ -20,8 +20,6 @@ namespace broadcast_constraints { -namespace { - static const char* library_name = "test_broadcast_constraints"; constexpr size_t EXT_SMALL = 10; @@ -130,8 +128,6 @@ void test_invalid_broadcast() EXPECT_THROW(runtime->submit(std::move(task)), std::invalid_argument); } -} // namespace - TEST(Broadcast, Basic) { legate::Core::perform_registration(); diff --git a/tests/cpp/integration/cpu_communicator.cc b/tests/cpp/integration/cpu_communicator.cc index fe621def19..4f395b6919 100644 --- a/tests/cpp/integration/cpu_communicator.cc +++ b/tests/cpp/integration/cpu_communicator.cc @@ -21,8 +21,6 @@ namespace cpu_communicator { -namespace { - const char* library_name = "test_cpu_communicator"; enum TaskIDs { @@ -90,8 +88,6 @@ void test_cpu_communicator_manual(int32_t ndim) runtime->submit(std::move(task)); } -} // namespace - // Test case with single unbound store TEST(Integration, CPUCommunicator) { diff --git a/tests/cpp/integration/exception.cc b/tests/cpp/integration/exception.cc index 9ee1da566f..0c35acd689 100644 --- a/tests/cpp/integration/exception.cc +++ b/tests/cpp/integration/exception.cc @@ -20,8 +20,6 @@ namespace exception { -namespace { - const char* library_name = "test_exception"; static legate::Logger logger(library_name); @@ -180,8 +178,6 @@ void test_pending() // Finish the test with a pending exception to check if the runtime cleans things up correctly } -} // namespace - TEST(Exception, Single) { legate::Core::perform_registration(); diff --git a/tests/cpp/integration/nccl.cu b/tests/cpp/integration/nccl.cu index dab4b181b4..e9a3dd7241 100644 --- a/tests/cpp/integration/nccl.cu +++ b/tests/cpp/integration/nccl.cu @@ -24,8 +24,6 @@ namespace nccl { -namespace { - const char* library_name = "test_nccl"; enum TaskIDs { @@ -104,8 +102,6 @@ void test_nccl_manual(int32_t ndim) runtime->submit(std::move(task)); } -} // namespace - // Test case with single unbound store TEST(Integration, NCCL) { diff --git a/tests/cpp/integration/weighted.cc b/tests/cpp/integration/weighted.cc index 082155da68..eff377d50c 100644 --- a/tests/cpp/integration/weighted.cc +++ b/tests/cpp/integration/weighted.cc @@ -20,8 +20,6 @@ namespace weighted { -namespace { - const char* library_name = "test_weighted"; enum TaskIDs { @@ -107,8 +105,6 @@ void test_weighted(uint32_t num_stores) check(runtime, library, stores); } -} // namespace - // Test case with single unbound store TEST(Integration, WeightedSingle) { diff --git a/tests/cpp/run.py b/tests/cpp/run.py new file mode 100755 index 0000000000..d3736cdf88 --- /dev/null +++ b/tests/cpp/run.py @@ -0,0 +1,165 @@ +# Copyright 2023 NVIDIA Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import argparse +import os +import subprocess +import sys + +LAUNCHER_VAR_PREFIXES = ( + "CONDA_", + "LEGATE_", + "LEGION_", + "LG_", + "REALM_", + "GASNET_", + "PYTHON", + "UCX_", + "NCCL_", + "CUNUMERIC_", + "NVIDIA_", +) + +test_args_dict = { + # Example of usage + # "Alignment.Basic" : ["-logfile", "build/example_file.log"] +} + + +def fetch_test_names(binary_path): + list_command = [binary_path] + ["--gtest_list_tests"] + + result = subprocess.check_output(list_command, stderr=subprocess.STDOUT) + result = result.decode(sys.stdout.encoding).split("\n") + + test_group = "" + test_names = [] + for line in result: + # Skip empty entry + if not line.strip(): + continue + + # Check if this is a test group + if line[0] != " ": + test_group = line.strip() + continue + + # Assign test to test group + test_names += [test_group + line.strip()] + + return test_names + + +def run_test(config, test_name, log, extra_args): + test_command = [] + if config.ranks != 0: + test_command += ["mpirun", "-n", str(config.ranks)] + test_command += ["--output-filename", "build/mpi_result"] + test_command += ["--merge-stderr-to-stdout"] + + def is_launcher_var(name: str) -> bool: + # Whether an environment variable name is relevant for the laucher + return name.endswith("PATH") or any( + name.startswith(prefix) for prefix in LAUNCHER_VAR_PREFIXES + ) + + for var in dict(os.environ): + if is_launcher_var(var): + test_command += ["-x", var] + + test_command += [config.binary_path] + test_command += [f"--gtest_filter={test_name}"] + test_command += ["-ll:cpu", str(config.cpus)] + test_command += extra_args + + if test_name in test_args_dict: + test_command += test_args_dict[test_name] + + task = subprocess.Popen(test_command, stdout=log, stderr=subprocess.STDOUT) + task.communicate() + + return task.returncode + + +def main(): + parser = argparse.ArgumentParser(description="Run Legate cpp tests.") + parser.add_argument( + "--binary-path", + dest="binary_path", + required=False, + default="build/cpp_tests", + help="Path to binary under test.", + ) + parser.add_argument( + "--log-path", + dest="log_path", + required=False, + default="build/results.log", + help="Path to output log file.", + ) + parser.add_argument( + "--ranks", + dest="ranks", + required=False, + type=int, + default=0, + help="Runs mpirun with rank if non-zero.", + ) + parser.add_argument( + "--cpus", + dest="cpus", + required=False, + type=int, + default=4, + help="Legion cmd argument for CPU processors to create per process.", + ) + config, extra_args = parser.parse_known_args() + + # Get names + test_names = fetch_test_names(config.binary_path) + + # Run each test with popen + total_count = len(test_names) + failed_count = 0 + failed_tests = [] + with open(config.log_path, "w") as log: + for count, test_name in enumerate(test_names): + return_code = run_test(config, test_name, log, extra_args) + + # Record test result + if return_code: + failed_tests += [test_name] + failed_count += 1 + print( + f"{count+1:3d}/{total_count}: {test_name} ".ljust(50, "."), + "Failed" if return_code else "Passed", + ) + + # Summarize results + print( + f"\n{int((total_count - failed_count) / total_count * 100)}% " + f"tests passed, {failed_count} tests failed out of {total_count}" + ) + if failed_tests: + print("\nThe following tests FAILED:") + for test in failed_tests: + print(f" - {test} (Failed)") + print(f"\nLog file generated: {config.log_path}") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/cpp/run.sh b/tests/cpp/run.sh index 962ef8b319..6e012b101e 100755 --- a/tests/cpp/run.sh +++ b/tests/cpp/run.sh @@ -1,4 +1,13 @@ #!/bin/bash -cd build -REALM_BACKTRACE=1 LEGATE_TEST=1 LEGION_DEFAULT_ARGS="-ll:cpu 4" ctest --output-on-failure "$@" +if [ $# -eq 0 ] + then + REALM_BACKTRACE=1 LEGATE_TEST=1 python run.py +elif [ $# -eq 1 ] && [ "$1" = "ctest" ] + then + echo "Using ctest" + cd build + REALM_BACKTRACE=1 LEGATE_TEST=1 LEGION_DEFAULT_ARGS="-ll:cpu 4" ctest --output-on-failure "$@" +else + echo "Invalid arguments" +fi From c3e5928366bbb69192b91ddef10a325642f59e76 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Tue, 25 Jul 2023 15:25:10 -0700 Subject: [PATCH 0170/1425] Fix for network-enabled build * Fix for network-enabled build See merge request legate/legate.core.internal!92 --- src/core/comm/coll.h | 1 + src/core/comm/mpi_comm.cc | 11 +++-------- src/core/comm/pthread_barrier.h | 16 ++++++++++++++++ 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/core/comm/coll.h b/src/core/comm/coll.h index 6b3ae1bfb7..130906ec82 100644 --- a/src/core/comm/coll.h +++ b/src/core/comm/coll.h @@ -29,6 +29,7 @@ // does not implement pthread barriers, so we need to include an // implementation in case they are not defined. We also need to // include unistd.h since that defines _POSIX_BARRIERS. +#include #include #if !defined(_POSIX_BARRIERS) || (_POSIX_BARRIERS < 0) #include "core/comm/pthread_barrier.h" diff --git a/src/core/comm/mpi_comm.cc b/src/core/comm/mpi_comm.cc index e24ace6565..6754f7cc0c 100644 --- a/src/core/comm/mpi_comm.cc +++ b/src/core/comm/mpi_comm.cc @@ -1,4 +1,4 @@ -/* Copyright 2022 NVIDIA Corporation +/* Copyright 2023 NVIDIA Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,14 +14,9 @@ * */ -#include -#include -#include -#include - #include "coll.h" -#include "legate_defines.h" -#include "legion.h" + +#include "core/utilities/typedefs.h" namespace legate::comm::coll { diff --git a/src/core/comm/pthread_barrier.h b/src/core/comm/pthread_barrier.h index aef8a59ff3..4c62ac83b7 100644 --- a/src/core/comm/pthread_barrier.h +++ b/src/core/comm/pthread_barrier.h @@ -1,3 +1,19 @@ +/* Copyright 2022 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + #if !defined(_POSIX_BARRIERS) || (_POSIX_BARRIERS < 0) #ifndef PTHREAD_BARRIER_H_ From acf4f5b1fe8b2e0a67185e5553fd006f9cb7dc98 Mon Sep 17 00:00:00 2001 From: Mark Vaz Date: Tue, 25 Jul 2023 16:23:51 -0700 Subject: [PATCH 0171/1425] Update Licensing to NV Proprietary * remove contributing.md reference in README * Remove CONTRIBUTING file * update new file with license header * update contribution statement with NV license * update remaining files with NV license * update markdown files with NV license * update more make files with NV license * update python pyx files with NV license * update cmake txt files with NV license * update cmake files with NV license * update remaining py files with NV license * update direct entry py files with NV license * update setup py files with NV license * update direct entry py files with NV license * update pyi files with NV license * update inl files with NV license * update cu files with NV License * update cc code files with NV license * update header files with NV license * revert * update py files with NV License * update cu files with NV License * update cc code files with NV license * update header files with NV license * update cu files with NV License * update legate.mk with NV License * update inl files with NV license * update cc code files with NV license * update header files with NV license * Add NV License File See merge request legate/legate.core.internal!93 --- BUILD.md | 23 +- CMakeLists.txt | 20 +- CONTRIBUTING.md | 74 ------ LICENSE | 217 ++++-------------- README.md | 27 +-- bind.sh | 21 +- cmake/Modules/cpm_helpers.cmake | 20 +- cmake/Modules/cuda_arch_helpers.cmake | 20 +- cmake/Modules/legate_core_options.cmake | 20 +- cmake/Modules/set_cpu_arch_flags.cmake | 20 +- cmake/generate_install_info_py.cmake | 20 +- cmake/legate_helper_functions.cmake | 85 +++---- cmake/thirdparty/get_legion.cmake | 20 +- cmake/thirdparty/get_nccl.cmake | 20 +- cmake/thirdparty/get_thrust.cmake | 20 +- conda/conda-build/meta.yaml | 2 +- docs/legate/core/Makefile | 21 +- docs/legate/core/source/conf.py | 22 +- examples/CMakeLists.txt | 20 +- examples/cpp/CMakeLists.txt | 20 +- examples/cpp/hello/hello.cc | 22 +- examples/cpp/hello/hello_print.cc | 22 +- examples/cpp/hello/task_hello.cc | 22 +- examples/cpp/hello/task_hello.h | 22 +- examples/cpp/io/io.cc | 22 +- examples/cpp/io/task_io.cc | 22 +- examples/cpp/io/task_io.h | 22 +- examples/cpp/main.cc | 22 +- examples/hello/CMakeLists.txt | 20 +- examples/hello/README.md | 23 +- examples/hello/examples/cunumeric-variance.py | 21 +- examples/hello/examples/hello-world.md | 23 +- examples/hello/examples/hello-world.py | 22 +- examples/hello/examples/variance.md | 23 +- examples/hello/examples/variance.py | 22 +- examples/hello/hello/__init__.py | 23 +- examples/hello/hello/hello.py | 21 +- examples/hello/setup.py | 26 +-- examples/hello/src/CMakeLists.txt | 20 +- examples/hello/src/hello_cffi.h | 22 +- examples/hello/src/hello_world.cc | 22 +- examples/hello/src/hello_world.h | 22 +- examples/hello/src/iota.cc | 22 +- examples/hello/src/square.cc | 22 +- examples/hello/src/sum.cc | 22 +- examples/io/CMakeLists.txt | 20 +- examples/io/README.md | 23 +- examples/io/editable-install.sh | 20 +- examples/io/examples/even_tiles.py | 22 +- examples/io/examples/single_file.py | 22 +- examples/io/examples/uneven_tiles.py | 22 +- examples/io/install.sh | 20 +- examples/io/legateio/__init__.py | 22 +- examples/io/legateio/legateio.py | 22 +- examples/io/setup.py | 26 +-- examples/io/src/CMakeLists.txt | 20 +- examples/io/src/legateio.cc | 22 +- examples/io/src/legateio.h | 22 +- examples/io/src/legateio_cffi.h | 22 +- examples/io/src/read_even_tiles.cc | 22 +- examples/io/src/read_file.cc | 22 +- examples/io/src/read_uneven_tiles.cc | 22 +- examples/io/src/util.cc | 22 +- examples/io/src/util.h | 22 +- examples/io/src/write_even_tiles.cc | 22 +- examples/io/src/write_file.cc | 22 +- examples/io/src/write_uneven_tiles.cc | 22 +- examples/reduction/CMakeLists.txt | 20 +- examples/reduction/README.md | 21 +- examples/reduction/editable-install.sh | 20 +- examples/reduction/examples/bincount.py | 22 +- examples/reduction/examples/histogram.py | 22 +- examples/reduction/examples/matmul.py | 22 +- examples/reduction/examples/sum_over_axis.py | 22 +- examples/reduction/examples/unique.py | 22 +- examples/reduction/install.sh | 20 +- examples/reduction/reduction/__init__.py | 22 +- examples/reduction/reduction/reduction.py | 22 +- examples/reduction/setup.py | 26 +-- examples/reduction/src/CMakeLists.txt | 20 +- examples/reduction/src/bincount.cc | 22 +- examples/reduction/src/categorize.cc | 22 +- examples/reduction/src/histogram.cc | 22 +- examples/reduction/src/matmul.cc | 22 +- examples/reduction/src/mul.cc | 22 +- examples/reduction/src/reduction_cffi.h | 22 +- examples/reduction/src/sum_over_axis.cc | 22 +- examples/reduction/src/unique.cc | 22 +- install.py | 21 +- legate/__init__.py | 22 +- legate/_sphinxext/__init__.py | 24 +- legate/_sphinxext/settings.py | 22 +- legate/core/__init__.py | 22 +- legate/core/_legion/__init__.py | 22 +- legate/core/_legion/env.py | 22 +- legate/core/_legion/field.py | 22 +- legate/core/_legion/future.py | 22 +- legate/core/_legion/geometry.py | 22 +- legate/core/_legion/operation.py | 22 +- legate/core/_legion/partition.py | 22 +- legate/core/_legion/partition_functor.py | 22 +- legate/core/_legion/pending.py | 22 +- legate/core/_legion/region.py | 22 +- legate/core/_legion/space.py | 22 +- legate/core/_legion/task.py | 22 +- legate/core/_legion/transform.py | 22 +- legate/core/_legion/util.py | 22 +- legate/core/_lib/CMakeLists.txt | 20 +- legate/core/_lib/context.pyx | 21 +- legate/core/_lib/types.pyx | 22 +- legate/core/allocation.py | 22 +- legate/core/communicator.py | 22 +- legate/core/constraints.py | 22 +- legate/core/context.py | 22 +- legate/core/corelib.py | 22 +- legate/core/cycle_detector.py | 22 +- legate/core/exception.py | 22 +- legate/core/io.py | 22 +- legate/core/launcher.py | 22 +- legate/core/legate.py | 22 +- legate/core/machine.py | 22 +- legate/core/operation.py | 22 +- legate/core/partition.py | 22 +- legate/core/projection.py | 22 +- legate/core/restriction.py | 22 +- legate/core/runtime.py | 22 +- legate/core/shape.py | 22 +- legate/core/solver.py | 22 +- legate/core/store.py | 22 +- legate/core/transform.py | 22 +- legate/core/types.py | 22 +- legate/core/types.pyi | 22 +- legate/core/utils.py | 22 +- legate/driver/__init__.py | 22 +- legate/driver/args.py | 22 +- legate/driver/command.py | 22 +- legate/driver/config.py | 22 +- legate/driver/defaults.py | 22 +- legate/driver/driver.py | 22 +- legate/driver/launcher.py | 22 +- legate/driver/logs.py | 22 +- legate/driver/main.py | 22 +- legate/jupyter/__init__.py | 22 +- legate/jupyter/_legion_kernel.py | 22 +- legate/jupyter/args.py | 22 +- legate/jupyter/config.py | 22 +- legate/jupyter/kernel.py | 22 +- legate/jupyter/magic.py | 22 +- legate/jupyter/main.py | 22 +- legate/lgpatch.py | 22 +- legate/settings.py | 22 +- legate/tester/__init__.py | 22 +- legate/tester/args.py | 22 +- legate/tester/config.py | 22 +- legate/tester/logger.py | 22 +- legate/tester/stages/__init__.py | 22 +- legate/tester/stages/_linux/__init__.py | 22 +- legate/tester/stages/_linux/cpu.py | 22 +- legate/tester/stages/_linux/eager.py | 22 +- legate/tester/stages/_linux/gpu.py | 22 +- legate/tester/stages/_linux/omp.py | 22 +- legate/tester/stages/_osx/__init__.py | 22 +- legate/tester/stages/_osx/cpu.py | 22 +- legate/tester/stages/_osx/eager.py | 22 +- legate/tester/stages/_osx/gpu.py | 22 +- legate/tester/stages/_osx/omp.py | 22 +- legate/tester/stages/test_stage.py | 22 +- legate/tester/stages/util.py | 22 +- legate/tester/test_plan.py | 22 +- legate/tester/test_system.py | 22 +- legate/timing/__init__.py | 24 +- legate/timing/timing.py | 22 +- legate/util/__init__.py | 24 +- legate/util/args.py | 22 +- legate/util/colors.py | 22 +- legate/util/fs.py | 22 +- legate/util/info.py | 22 +- legate/util/settings.py | 22 +- legate/util/shared_args.py | 22 +- legate/util/system.py | 22 +- legate/util/types.py | 22 +- legate/util/ui.py | 22 +- legate_core_cpp.cmake | 20 +- legate_core_python.cmake | 20 +- pyproject.toml | 20 +- setup.py | 25 +- src/core/comm/coll.cc | 22 +- src/core/comm/coll.h | 22 +- src/core/comm/comm.cc | 22 +- src/core/comm/comm.h | 22 +- src/core/comm/comm_cpu.cc | 22 +- src/core/comm/comm_cpu.h | 22 +- src/core/comm/comm_nccl.cu | 22 +- src/core/comm/comm_nccl.h | 22 +- src/core/comm/communicator.h | 22 +- src/core/comm/local_comm.cc | 22 +- src/core/comm/mpi_comm.cc | 22 +- src/core/comm/pthread_barrier.h | 22 +- src/core/cuda/cuda_help.h | 22 +- src/core/cuda/stream_pool.cu | 22 +- src/core/cuda/stream_pool.h | 22 +- src/core/data/allocator.cc | 22 +- src/core/data/allocator.h | 22 +- src/core/data/buffer.h | 22 +- src/core/data/detail/logical_region_field.cc | 22 +- src/core/data/detail/logical_region_field.h | 22 +- src/core/data/detail/logical_store.cc | 22 +- src/core/data/detail/logical_store.h | 22 +- src/core/data/detail/scalar.cc | 22 +- src/core/data/detail/scalar.h | 22 +- src/core/data/detail/store.cc | 22 +- src/core/data/detail/store.h | 22 +- src/core/data/detail/transform.cc | 22 +- src/core/data/detail/transform.h | 22 +- src/core/data/logical_store.cc | 22 +- src/core/data/logical_store.h | 22 +- src/core/data/scalar.cc | 22 +- src/core/data/scalar.h | 22 +- src/core/data/scalar.inl | 22 +- src/core/data/shape.cc | 22 +- src/core/data/shape.h | 22 +- src/core/data/slice.h | 22 +- src/core/data/store.cc | 22 +- src/core/data/store.h | 22 +- src/core/data/store.inl | 22 +- src/core/legate_c.cc | 22 +- src/core/legate_c.h | 22 +- src/core/mapping/detail/base_mapper.cc | 22 +- src/core/mapping/detail/base_mapper.h | 22 +- src/core/mapping/detail/core_mapper.cc | 22 +- src/core/mapping/detail/core_mapper.h | 22 +- src/core/mapping/detail/default_mapper.cc | 22 +- src/core/mapping/detail/default_mapper.h | 22 +- src/core/mapping/detail/instance_manager.cc | 22 +- src/core/mapping/detail/instance_manager.h | 22 +- src/core/mapping/detail/machine.cc | 22 +- src/core/mapping/detail/machine.h | 22 +- src/core/mapping/detail/mapping.cc | 22 +- src/core/mapping/detail/mapping.h | 22 +- src/core/mapping/detail/operation.cc | 22 +- src/core/mapping/detail/operation.h | 22 +- src/core/mapping/detail/operation.inl | 22 +- src/core/mapping/detail/store.cc | 22 +- src/core/mapping/detail/store.h | 22 +- src/core/mapping/machine.cc | 22 +- src/core/mapping/machine.h | 22 +- src/core/mapping/mapping.cc | 22 +- src/core/mapping/mapping.h | 22 +- src/core/mapping/operation.cc | 22 +- src/core/mapping/operation.h | 22 +- src/core/mapping/store.cc | 22 +- src/core/mapping/store.h | 22 +- src/core/operation/detail/copy.cc | 22 +- src/core/operation/detail/copy.h | 22 +- src/core/operation/detail/copy_launcher.cc | 22 +- src/core/operation/detail/copy_launcher.h | 22 +- src/core/operation/detail/fill.cc | 22 +- src/core/operation/detail/fill.h | 22 +- src/core/operation/detail/fill_launcher.cc | 22 +- src/core/operation/detail/fill_launcher.h | 22 +- src/core/operation/detail/gather.cc | 22 +- src/core/operation/detail/gather.h | 22 +- src/core/operation/detail/launcher_arg.cc | 22 +- src/core/operation/detail/launcher_arg.h | 22 +- src/core/operation/detail/operation.cc | 22 +- src/core/operation/detail/operation.h | 22 +- src/core/operation/detail/projection.cc | 22 +- src/core/operation/detail/projection.h | 22 +- src/core/operation/detail/req_analyzer.cc | 22 +- src/core/operation/detail/req_analyzer.h | 22 +- src/core/operation/detail/scatter.cc | 22 +- src/core/operation/detail/scatter.h | 22 +- src/core/operation/detail/scatter_gather.cc | 22 +- src/core/operation/detail/scatter_gather.h | 22 +- src/core/operation/detail/task.cc | 22 +- src/core/operation/detail/task.h | 22 +- src/core/operation/detail/task_launcher.cc | 22 +- src/core/operation/detail/task_launcher.h | 22 +- src/core/operation/task.cc | 22 +- src/core/operation/task.h | 22 +- src/core/partitioning/constraint.cc | 22 +- src/core/partitioning/constraint.h | 22 +- src/core/partitioning/detail/constraint.cc | 22 +- src/core/partitioning/detail/constraint.h | 22 +- .../partitioning/detail/constraint_solver.cc | 22 +- .../partitioning/detail/constraint_solver.h | 22 +- src/core/partitioning/detail/partitioner.cc | 22 +- src/core/partitioning/detail/partitioner.h | 22 +- src/core/partitioning/partition.cc | 22 +- src/core/partitioning/partition.h | 22 +- src/core/partitioning/restriction.cc | 22 +- src/core/partitioning/restriction.h | 22 +- .../runtime/detail/communicator_manager.cc | 22 +- .../runtime/detail/communicator_manager.h | 22 +- src/core/runtime/detail/field_manager.cc | 22 +- src/core/runtime/detail/field_manager.h | 22 +- src/core/runtime/detail/library.cc | 22 +- src/core/runtime/detail/library.h | 22 +- src/core/runtime/detail/machine_manager.cc | 22 +- src/core/runtime/detail/machine_manager.h | 22 +- src/core/runtime/detail/partition_manager.cc | 22 +- src/core/runtime/detail/partition_manager.h | 22 +- src/core/runtime/detail/projection.cc | 22 +- src/core/runtime/detail/projection.h | 22 +- src/core/runtime/detail/provenance_manager.cc | 22 +- src/core/runtime/detail/provenance_manager.h | 22 +- src/core/runtime/detail/region_manager.cc | 22 +- src/core/runtime/detail/region_manager.h | 22 +- src/core/runtime/detail/runtime.cc | 22 +- src/core/runtime/detail/runtime.h | 22 +- src/core/runtime/detail/shard.cc | 22 +- src/core/runtime/detail/shard.h | 22 +- src/core/runtime/library.cc | 22 +- src/core/runtime/library.h | 22 +- src/core/runtime/library.inl | 22 +- src/core/runtime/resource.h | 22 +- src/core/runtime/runtime.cc | 22 +- src/core/runtime/runtime.h | 22 +- src/core/runtime/runtime.inl | 22 +- src/core/runtime/tracker.cc | 22 +- src/core/runtime/tracker.h | 22 +- src/core/task/detail/return.cc | 22 +- src/core/task/detail/return.h | 22 +- src/core/task/detail/task_context.cc | 22 +- src/core/task/detail/task_context.h | 22 +- src/core/task/exception.h | 22 +- src/core/task/registrar.cc | 22 +- src/core/task/registrar.h | 22 +- src/core/task/task.cc | 22 +- src/core/task/task.h | 22 +- src/core/task/task.inl | 22 +- src/core/task/task_context.cc | 22 +- src/core/task/task_context.h | 22 +- src/core/task/task_info.cc | 22 +- src/core/task/task_info.h | 22 +- src/core/task/variant_helper.h | 22 +- src/core/task/variant_options.cc | 22 +- src/core/task/variant_options.h | 22 +- src/core/type/detail/type_info.cc | 22 +- src/core/type/detail/type_info.h | 22 +- src/core/type/type_info.cc | 22 +- src/core/type/type_info.h | 24 +- src/core/type/type_traits.h | 22 +- src/core/utilities/debug.cc | 22 +- src/core/utilities/debug.h | 22 +- src/core/utilities/deserializer.cc | 22 +- src/core/utilities/deserializer.h | 22 +- src/core/utilities/deserializer.inl | 22 +- src/core/utilities/detail/buffer_builder.cc | 22 +- src/core/utilities/detail/buffer_builder.h | 22 +- src/core/utilities/detail/buffer_builder.inl | 22 +- src/core/utilities/dispatch.h | 22 +- src/core/utilities/linearize.cc | 22 +- src/core/utilities/linearize.h | 22 +- src/core/utilities/machine.cc | 22 +- src/core/utilities/machine.h | 22 +- src/core/utilities/multi_set.h | 22 +- src/core/utilities/multi_set.inl | 22 +- src/core/utilities/nvtx_help.h | 22 +- src/core/utilities/ordered_set.h | 22 +- src/core/utilities/span.h | 22 +- src/core/utilities/tuple.h | 22 +- src/core/utilities/tuple.inl | 22 +- src/core/utilities/typedefs.h | 24 +- src/env_defaults.h | 22 +- src/legate.h | 22 +- src/legate.mk | 21 +- src/legate_defines.h | 22 +- src/legate_preamble.h | 22 +- tests/cpp/CMakeLists.txt | 20 +- tests/cpp/cmake/thirdparty/get_nccl.cmake | 20 +- .../cpp/integration/alignment_constraints.cc | 22 +- .../cpp/integration/broadcast_constraints.cc | 22 +- tests/cpp/integration/copy_failure.cc | 22 +- tests/cpp/integration/copy_gather.cc | 22 +- tests/cpp/integration/copy_gather_scatter.cc | 22 +- tests/cpp/integration/copy_normal.cc | 22 +- tests/cpp/integration/copy_scatter.cc | 22 +- tests/cpp/integration/copy_util.inl | 22 +- tests/cpp/integration/cpu_communicator.cc | 22 +- tests/cpp/integration/exception.cc | 22 +- tests/cpp/integration/fill.cc | 22 +- tests/cpp/integration/image_constraints.cc | 22 +- tests/cpp/integration/inline_map.cc | 22 +- tests/cpp/integration/inout.cc | 22 +- tests/cpp/integration/machine_scope.cc | 22 +- tests/cpp/integration/manual_simple.cc | 22 +- tests/cpp/integration/multi_scalar_out.cc | 22 +- tests/cpp/integration/nccl.cu | 22 +- tests/cpp/integration/provenance.cc | 22 +- tests/cpp/integration/region_manager.cc | 22 +- .../integration/tasks/task_region_manager.cc | 22 +- .../integration/tasks/task_region_manager.h | 22 +- tests/cpp/integration/tasks/task_simple.cc | 22 +- tests/cpp/integration/tasks/task_simple.h | 22 +- tests/cpp/integration/weighted.cc | 22 +- tests/cpp/main.cc | 22 +- tests/cpp/run.py | 22 +- tests/cpp/unit/library.cc | 22 +- tests/cpp/unit/machine.cc | 22 +- tests/cpp/unit/registration.cc | 22 +- tests/cpp/unit/scalar.cc | 22 +- tests/cpp/unit/span.cc | 22 +- tests/cpp/unit/store.cc | 22 +- tests/integration/CMakeLists.txt | 20 +- tests/integration/collective/CMakeLists.txt | 20 +- .../collective/collective/__init__.py | 22 +- .../collective/collective/collective.py | 22 +- .../integration/collective/collective/lib.py | 22 +- .../collective/editable-install.sh | 20 +- .../collective/examples/test_collective.py | 22 +- tests/integration/collective/install.sh | 20 +- tests/integration/collective/setup.py | 26 +-- .../integration/collective/src/CMakeLists.txt | 20 +- .../collective/src/collective_cffi.h | 22 +- .../collective/src/collective_test.h | 24 +- tests/integration/collective/src/library.cc | 22 +- tests/integration/collective/src/library.h | 22 +- .../integration/region_manager/CMakeLists.txt | 20 +- .../examples/test_region_manager.py | 22 +- .../region_manager/region_manager/__init__.py | 24 +- .../region_manager/region_manager/lib.py | 22 +- tests/integration/region_manager/setup.py | 26 +-- .../region_manager/src/CMakeLists.txt | 20 +- .../integration/region_manager/src/library.cc | 22 +- .../integration/region_manager/src/library.h | 22 +- .../region_manager/src/region_manager_cffi.h | 22 +- .../integration/region_manager/src/tester.cc | 22 +- tests/integration/region_manager/src/tester.h | 22 +- tests/integration/registry/CMakeLists.txt | 20 +- .../registry/examples/test_registry.py | 22 +- .../integration/registry/registry/__init__.py | 24 +- tests/integration/registry/registry/lib.py | 22 +- tests/integration/registry/setup.py | 26 +-- tests/integration/registry/src/CMakeLists.txt | 20 +- tests/integration/registry/src/hello.cc | 22 +- tests/integration/registry/src/library.cc | 22 +- tests/integration/registry/src/library.h | 22 +- tests/integration/registry/src/no_variant.cc | 22 +- .../integration/registry/src/registry_cffi.h | 22 +- tests/integration/registry/src/world.cc | 22 +- tests/integration/registry/src/world.h | 22 +- tests/integration/scoping/CMakeLists.txt | 20 +- .../scoping/examples/test_scoping.py | 22 +- tests/integration/scoping/scoping/__init__.py | 24 +- tests/integration/scoping/scoping/lib.py | 22 +- tests/integration/scoping/setup.py | 26 +-- tests/integration/scoping/src/CMakeLists.txt | 20 +- tests/integration/scoping/src/library.cc | 22 +- tests/integration/scoping/src/scoping_cffi.h | 22 +- tests/integration/tree_reduce/CMakeLists.txt | 20 +- .../tree_reduce/examples/test_tree_reduce.py | 22 +- tests/integration/tree_reduce/setup.py | 26 +-- .../tree_reduce/src/CMakeLists.txt | 20 +- tests/integration/tree_reduce/src/library.cc | 22 +- tests/integration/tree_reduce/src/library.h | 22 +- .../tree_reduce/src/produce_normal.cc | 22 +- .../tree_reduce/src/produce_normal.h | 22 +- .../tree_reduce/src/produce_unbound.cc | 22 +- .../tree_reduce/src/produce_unbound.h | 22 +- .../tree_reduce/src/reduce_normal.cc | 22 +- .../tree_reduce/src/reduce_normal.h | 22 +- .../tree_reduce/src/reduce_unbound.cc | 22 +- .../tree_reduce/src/reduce_unbound.h | 22 +- .../tree_reduce/src/tree_reduce_cffi.h | 22 +- .../tree_reduce/tree_reduce/__init__.py | 24 +- .../tree_reduce/tree_reduce/lib.py | 22 +- tests/unit/__init__.py | 24 +- tests/unit/legate/__init__.py | 24 +- .../legate/core/test_invalid_scalar_arg.py | 22 +- tests/unit/legate/core/test_machine.py | 22 +- tests/unit/legate/core/test_store.py | 22 +- tests/unit/legate/core/test_type_uid.py | 22 +- tests/unit/legate/driver/__init__.py | 24 +- tests/unit/legate/driver/conftest.py | 22 +- tests/unit/legate/driver/test_args.py | 22 +- tests/unit/legate/driver/test_command.py | 22 +- tests/unit/legate/driver/test_config.py | 22 +- tests/unit/legate/driver/test_defaults.py | 22 +- tests/unit/legate/driver/test_driver.py | 22 +- tests/unit/legate/driver/test_launcher.py | 22 +- tests/unit/legate/driver/test_logs.py | 22 +- tests/unit/legate/driver/test_main.py | 22 +- tests/unit/legate/driver/util.py | 22 +- tests/unit/legate/jupyter/__init__.py | 24 +- tests/unit/legate/jupyter/test_args.py | 22 +- tests/unit/legate/jupyter/test_config.py | 22 +- tests/unit/legate/jupyter/test_kernel.py | 22 +- tests/unit/legate/jupyter/test_main.py | 22 +- tests/unit/legate/test_cycle_check.py | 22 +- tests/unit/legate/test_settings.py | 22 +- tests/unit/legate/tester/__init__.py | 24 +- tests/unit/legate/tester/stages/__init__.py | 22 +- .../legate/tester/stages/_linux/__init__.py | 22 +- .../legate/tester/stages/_linux/test_cpu.py | 22 +- .../legate/tester/stages/_linux/test_eager.py | 22 +- .../legate/tester/stages/_linux/test_gpu.py | 22 +- .../legate/tester/stages/_linux/test_omp.py | 22 +- .../legate/tester/stages/test_test_stage.py | 22 +- tests/unit/legate/tester/stages/test_util.py | 22 +- tests/unit/legate/tester/test___init__.py | 22 +- tests/unit/legate/tester/test_args.py | 22 +- tests/unit/legate/tester/test_config.py | 22 +- tests/unit/legate/tester/test_logger.py | 22 +- tests/unit/legate/tester/test_test_system.py | 22 +- tests/unit/legate/util/__init__.py | 24 +- tests/unit/legate/util/test_args.py | 22 +- tests/unit/legate/util/test_colors.py | 22 +- tests/unit/legate/util/test_fs.py | 22 +- tests/unit/legate/util/test_settings.py | 22 +- tests/unit/legate/util/test_system.py | 22 +- tests/unit/legate/util/test_types.py | 22 +- tests/unit/legate/util/test_ui.py | 22 +- tests/unit/util.py | 22 +- typings/IPython/__init__.pyi | 22 +- typings/IPython/core/magic.pyi | 22 +- typings/jupyter_client/kernelspec.pyi | 22 +- typings/legion_cffi/__init__.pyi | 22 +- typings/legion_cffi/lib.pyi | 22 +- typings/legion_top/__init__.pyi | 22 +- typings/pyarrow/__init__.pyi | 22 +- typings/pyarrow/lib.pyi | 22 +- 522 files changed, 4720 insertions(+), 7044 deletions(-) delete mode 100644 CONTRIBUTING.md diff --git a/BUILD.md b/BUILD.md index 6691d49169..b188a4ecb2 100644 --- a/BUILD.md +++ b/BUILD.md @@ -1,18 +1,13 @@ # Basic build diff --git a/CMakeLists.txt b/CMakeLists.txt index a056ea3405..a208741e75 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2022 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= cmake_minimum_required(VERSION 3.22.1 FATAL_ERROR) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 3da71fca14..0000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,74 +0,0 @@ -# Contributing to Legate Core - -Legate Core is an open-source project released under the [Apache license, version 2.0](https://www.apache.org/licenses/LICENSE-2.0). We welcome any and all contributions, and we hope that you can help us develop a strong community. - -## How to begin - -Most of the time, the best thing is to begin by [opening an issue](https://github.com/nv-legate/Legate Core/issues). This gives us a chance to discuss the contribution and to define the problem or feature that it addresses. Often, opening of the issue first may help prevent you from doing unnecessary work or to enhance and further develop your idea. - -Once you are ready to start development, we ask you to work on a [fork](https://docs.github.com/en/get-started/quickstart/fork-a-repo) of our repository. The next step is to create a (pull request)[https://help.github.com/en/articles/about-pull-requests]. Feel free to open the pull request as soon as you begin your development (just mark it [as a draft](https://github.blog/2019-02-14-introducing-draft-pull-requests/)) or when you are ready to have your contribution merged. - -## The Legalese: Developer Certificate of Origin - -Legate Core is released under the [Apache license, version 2.0](https://www.apache.org/licenses/LICENSE-2.0) and is free to use, modify, and redistribute. To ensure that the license can be exercised without encumbrance, we ask you that you only contribute your own work or work to which you have the intellectual rights. To that end, we employ the Developer's Certificate of Origin (DCO), which is the lightweight mechanism for you to certify that you are legally able to make your contribution. Here is the full text of the certificate (also available at [DeveloperCertificate.org](https://developercertificate.org/)): - -```` -Developer Certificate of Origin -Version 1.1 - -Copyright (C) 2004, 2006 The Linux Foundation and its contributors. - -Everyone is permitted to copy and distribute verbatim copies of this -license document, but changing it is not allowed. - - -Developer's Certificate of Origin 1.1 - -By making a contribution to this project, I certify that: - -(a) The contribution was created in whole or in part by me and I - have the right to submit it under the open source license - indicated in the file; or - -(b) The contribution is based upon previous work that, to the best - of my knowledge, is covered under an appropriate open source - license and I have the right under that license to submit that - work with modifications, whether created in whole or in part - by me, under the same open source license (unless I am - permitted to submit under a different license), as indicated - in the file; or - -(c) The contribution was provided directly to me by some other - person who certified (a), (b) or (c) and I have not modified - it. - -(d) I understand and agree that this project and the contribution - are public and that a record of the contribution (including all - personal information I submit with it, including my sign-off) is - maintained indefinitely and may be redistributed consistent with - this project or the open source license(s) involved. -```` - -### How Do I Sign the DCO? - -Fortunately, it does not take much work to sign the DCO. The only thing that you have to do is to mark all your commits with a `Signed-off-by` line that looks like that: - -```` -Signed-off-by: Your Name -```` - -Please use your real name and a valid email address at which you can be reached. For legal reasons, we will not be able to accept contributions that use pseudonyms in the signature. You can simply add this line at the end of all your commits manually, or you can use the `-s` or the `--signoff` options provided by Git to automatically tack on the signature. - -## Review Process - -We are really grateful that you are thinking of contributing to Legate Core. We will make every effort to review your contributions as soon as possible. - -As we suggested at the beginning of this document, it will be really helpful to start with an issue unless your proposed change is really trivial. An issue will help to save work in the review process (e.g., maybe somebody is already working on exactly the same thing you want to work on). After you open your pull request (PR), there usually will be a community feedback that often will require further changes to your contribution (the usual open-source process). Usually, this will conclude in the PR being merged by a maintainer, but on rare occasions a PR may be rejected. This may happen, for example, if the PR appears abandoned (no response to the community feedback) or if the PR does not seem to be approaching community acceptance in a reasonable time frame. In any case, an explanation will always be given why a PR is closed. Even if a PR is closed for some reason, it may always be reopened if the situation evolves (feel free to comment on closed PRs to discuss reopening them). - -## Code Formatting Requirements - -Legate Core has a set of coding standards that are expected from all the code merged into the project. The coding standards are defined by the set of tools we use to format our code. We use the [pre-commit](https://pre-commit.com/) framework to run our formatting tools. The easiest way to meet the coding standards is to simply use the pre-commit framework to run all the checks for you. Please visit the [pre-commit project page](https://pre-commit.com/) for pre-commit installation and usage instructions. Once pre-commit is installed in the cuNumeric repo, all the checks and formatting will be run on every commit, but one can also run the checks explicitly as detailed in pre-commit documentation. - - - -We hope that the automation of our formatting checks will make it easy to comply with our coding standards. If you encounter problems with code formatting, however, please let us know in a comment on your PR, and we will do our best to help. diff --git a/LICENSE b/LICENSE index f433b1a53f..e84be4a9ee 100644 --- a/LICENSE +++ b/LICENSE @@ -1,177 +1,42 @@ +NVIDIA Source Code License (1-Way Commercial) – NVIDIA CONFIDENTIAL +1. Definitions - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS +“Licensor” means any person or entity that distributes its Work. + +“Software” means the original work of authorship made available under this License. + +“Work” means the Software and any additions to or derivative works of the Software that are made available under this License. + +The terms “reproduce,” “reproduction,” “derivative works,” and “distribution” have the meaning as provided under U.S. copyright law; provided, however, that for the purposes of this License, derivative works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work. + +Works, including the Software, are “made available” under this License by including in or with the Work either (a) a copyright notice referencing the applicability of this License to the Work, or (b) a copy of this License. + + +2. License Grants + +2.1 Copyright Grant. Subject to the terms and conditions of this License, each Licensor grants to you a perpetual, worldwide, non-exclusive, royalty-free, copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, sublicense and distribute its Work and any resulting derivative works in any form. + + +3. Limitations + +3.1 Redistribution. You may reproduce or distribute the Work only if (a) you do so under this License, (b) you include a complete copy of this License with your distribution, and (c) you retain without modification any copyright, patent, trademark, or attribution notices that are present in the Work. + +3.2 Derivative Works. You may specify that additional or different terms apply to the use, reproduction, and distribution of your derivative works of the Work (“Your Terms”) only if (a) Your Terms provide that the use limitation in Section 3.3 applies to your derivative works, and (b) you identify the specific derivative works that are subject to Your Terms. Notwithstanding Your Terms, this License (including the redistribution requirements in Section 3.1) will continue to apply to the Work itself. + +3.3 Use Limitation. The Work and any derivative works thereof only may be used or intended for use non-commercially. The Work or derivative works thereof may be used or intended for use by NVIDIA or it’s affiliates commercially or non-commercially. As used herein, “non-commercially” means for research or evaluation purposes only and not for any direct or indirect monetary gain. + +3.4 Patent Claims. If you bring or threaten to bring a patent claim against any Licensor (including any claim, cross-claim or counterclaim in a lawsuit) to enforce any patents that you allege are infringed by any Work, then your rights under this License from such Licensor (including the grants in Sections 2.1) will terminate immediately. + +3.5 Trademarks. This License does not grant any rights to use any Licensor’s or its affiliates’ names, logos, or trademarks, except as necessary to reproduce the notices described in this License. + +3.6 Termination. If you violate any term of this License, then your rights under this License (including the grants in Sections 2.1) will terminate immediately. + + +4. Disclaimer of Warranty. + +THE WORK IS PROVIDED “AS IS” WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WARRANTIES OR CONDITIONS OF M ERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NON-INFRINGEMENT. YOU BEAR THE RISK OF UNDERTAKING ANY ACTIVITIES UNDER THIS LICENSE. + + +5. Limitation of Liability. + +EXCEPT AS PROHIBITED BY APPLICABLE LAW, IN NO EVENT AND UNDER NO LEGAL THEORY, WHETHER IN TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE SHALL ANY LICENSOR BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF OR RELATED TO THIS LICENSE, THE USE OR INABILITY TO USE THE WORK (INCLUDING BUT NOT LIMITED TO LOSS OF GOODWILL, BUSINESS INTERRUPTION, LOST PROFITS OR DATA, COMPUTER FAILURE OR MALFUNCTION, OR ANY OTHER COMM ERCIAL DAMAGES OR LOSSES), EVEN IF THE LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. diff --git a/README.md b/README.md index 87d001510c..fbee060e02 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,13 @@ # Legate @@ -459,10 +454,6 @@ Memory: on the [Legion github issue tracker](https://github.com/StanfordLegion/legion/issues) as it will be almost entirely orthogonal to how you use Legate. -## Contributing - -See the discussion of contributing in [CONTRIBUTING.md](CONTRIBUTING.md). - ## Documentation A complete list of available features can is found in the [Legate Core diff --git a/bind.sh b/bind.sh index 8e00a7d5ae..a48d875005 100755 --- a/bind.sh +++ b/bind.sh @@ -1,19 +1,14 @@ #!/bin/bash -# Copyright 2021 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. set -euo pipefail diff --git a/cmake/Modules/cpm_helpers.cmake b/cmake/Modules/cpm_helpers.cmake index 9fc28633d8..672c43f0ad 100644 --- a/cmake/Modules/cpm_helpers.cmake +++ b/cmake/Modules/cpm_helpers.cmake @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2022 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= function(get_cpm_git_args _out_var) diff --git a/cmake/Modules/cuda_arch_helpers.cmake b/cmake/Modules/cuda_arch_helpers.cmake index 9a2206f69d..5dc937b726 100644 --- a/cmake/Modules/cuda_arch_helpers.cmake +++ b/cmake/Modules/cuda_arch_helpers.cmake @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2022 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= function(set_cuda_arch_from_names) diff --git a/cmake/Modules/legate_core_options.cmake b/cmake/Modules/legate_core_options.cmake index 96a8d7ec5f..449521df15 100644 --- a/cmake/Modules/legate_core_options.cmake +++ b/cmake/Modules/legate_core_options.cmake @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2022 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= option(BUILD_SHARED_LIBS "Build legate.core shared libraries" ON) diff --git a/cmake/Modules/set_cpu_arch_flags.cmake b/cmake/Modules/set_cpu_arch_flags.cmake index ff3e35ca39..dc82265330 100644 --- a/cmake/Modules/set_cpu_arch_flags.cmake +++ b/cmake/Modules/set_cpu_arch_flags.cmake @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2022 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= #------------------------------------------------------------------------------# diff --git a/cmake/generate_install_info_py.cmake b/cmake/generate_install_info_py.cmake index 408500ac91..4a8a76703e 100644 --- a/cmake/generate_install_info_py.cmake +++ b/cmake/generate_install_info_py.cmake @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2022 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= execute_process( diff --git a/cmake/legate_helper_functions.cmake b/cmake/legate_helper_functions.cmake index de2216c37f..7ddd7d3002 100644 --- a/cmake/legate_helper_functions.cmake +++ b/cmake/legate_helper_functions.cmake @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= macro(legate_include_rapids) @@ -280,20 +276,16 @@ endfunction() function(legate_cpp_library_template target output_sources_variable) set(file_template [=[ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once @@ -319,20 +311,16 @@ struct Task : public legate::LegateTask { set(file_template [=[ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "legate_library.h" @@ -408,20 +396,15 @@ set(fn_library "${CMAKE_CURRENT_SOURCE_DIR}/${py_path}/library.py") set(file_template [=[ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. from legate.core import ( Library, diff --git a/cmake/thirdparty/get_legion.cmake b/cmake/thirdparty/get_legion.cmake index 76c4c86857..54c83a6f19 100644 --- a/cmake/thirdparty/get_legion.cmake +++ b/cmake/thirdparty/get_legion.cmake @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2022-2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= include_guard(GLOBAL) diff --git a/cmake/thirdparty/get_nccl.cmake b/cmake/thirdparty/get_nccl.cmake index 1aee52b6f5..5269e68d3c 100644 --- a/cmake/thirdparty/get_nccl.cmake +++ b/cmake/thirdparty/get_nccl.cmake @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2022 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= function(find_or_configure_nccl) diff --git a/cmake/thirdparty/get_thrust.cmake b/cmake/thirdparty/get_thrust.cmake index 84784a1cef..25e09a7ac0 100644 --- a/cmake/thirdparty/get_thrust.cmake +++ b/cmake/thirdparty/get_thrust.cmake @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2022 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= # Use CPM to find or clone thrust diff --git a/conda/conda-build/meta.yaml b/conda/conda-build/meta.yaml index e384affcf7..427a46a39e 100644 --- a/conda/conda-build/meta.yaml +++ b/conda/conda-build/meta.yaml @@ -138,7 +138,7 @@ test: about: home: https://github.com/nv-legate/legate.core - license: Apache-2.0 + license: NVIDIA Proprietary license_file: LICENSE summary: 'Scalable Computational Code' description: | diff --git a/docs/legate/core/Makefile b/docs/legate/core/Makefile index f17ad74491..24a8d95142 100644 --- a/docs/legate/core/Makefile +++ b/docs/legate/core/Makefile @@ -1,17 +1,12 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. # Minimal makefile for Sphinx documentation # diff --git a/docs/legate/core/source/conf.py b/docs/legate/core/source/conf.py index 7abba3f931..4a4498ec27 100644 --- a/docs/legate/core/source/conf.py +++ b/docs/legate/core/source/conf.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + # Configuration file for the Sphinx documentation builder. # diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 4000994006..0c90aa2422 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= # We abuse find package for testing purposes here to diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt index f41886acbe..0a67f2e7c5 100644 --- a/examples/cpp/CMakeLists.txt +++ b/examples/cpp/CMakeLists.txt @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= cmake_minimum_required(VERSION 3.22.1 FATAL_ERROR) diff --git a/examples/cpp/hello/hello.cc b/examples/cpp/hello/hello.cc index ce613974e0..fa4cc85776 100644 --- a/examples/cpp/hello/hello.cc +++ b/examples/cpp/hello/hello.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/examples/cpp/hello/hello_print.cc b/examples/cpp/hello/hello_print.cc index dd730e2c21..821d810248 100644 --- a/examples/cpp/hello/hello_print.cc +++ b/examples/cpp/hello/hello_print.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/examples/cpp/hello/task_hello.cc b/examples/cpp/hello/task_hello.cc index 269579a9d8..ff5323ff14 100644 --- a/examples/cpp/hello/task_hello.cc +++ b/examples/cpp/hello/task_hello.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "task_hello.h" diff --git a/examples/cpp/hello/task_hello.h b/examples/cpp/hello/task_hello.h index c88cebd26b..8333800eb7 100644 --- a/examples/cpp/hello/task_hello.h +++ b/examples/cpp/hello/task_hello.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/examples/cpp/io/io.cc b/examples/cpp/io/io.cc index 170cdf810c..6ed1ae0d58 100644 --- a/examples/cpp/io/io.cc +++ b/examples/cpp/io/io.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/examples/cpp/io/task_io.cc b/examples/cpp/io/task_io.cc index 897425a4ab..83f9c3bc73 100644 --- a/examples/cpp/io/task_io.cc +++ b/examples/cpp/io/task_io.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/examples/cpp/io/task_io.h b/examples/cpp/io/task_io.h index a3e16c62ce..d9bbe63507 100644 --- a/examples/cpp/io/task_io.h +++ b/examples/cpp/io/task_io.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/examples/cpp/main.cc b/examples/cpp/main.cc index 34dcccd138..17ee5040da 100644 --- a/examples/cpp/main.cc +++ b/examples/cpp/main.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/examples/hello/CMakeLists.txt b/examples/hello/CMakeLists.txt index 2e618a9ad9..7630678b99 100644 --- a/examples/hello/CMakeLists.txt +++ b/examples/hello/CMakeLists.txt @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= cmake_minimum_required(VERSION 3.22.1 FATAL_ERROR) diff --git a/examples/hello/README.md b/examples/hello/README.md index d4ca94fad1..44ac5ccffa 100644 --- a/examples/hello/README.md +++ b/examples/hello/README.md @@ -1,18 +1,13 @@ # Legate Hello World Application diff --git a/examples/hello/examples/cunumeric-variance.py b/examples/hello/examples/cunumeric-variance.py index 11f1d84049..0182e80641 100644 --- a/examples/hello/examples/cunumeric-variance.py +++ b/examples/hello/examples/cunumeric-variance.py @@ -1,19 +1,14 @@ #!/usr/bin/env python3 -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. from typing import Any diff --git a/examples/hello/examples/hello-world.md b/examples/hello/examples/hello-world.md index 463f77358c..fb0f0dba03 100644 --- a/examples/hello/examples/hello-world.md +++ b/examples/hello/examples/hello-world.md @@ -1,18 +1,13 @@ # Basic Hello, World Application diff --git a/examples/hello/examples/hello-world.py b/examples/hello/examples/hello-world.py index a35914c7e3..12c1429a5e 100644 --- a/examples/hello/examples/hello-world.py +++ b/examples/hello/examples/hello-world.py @@ -1,20 +1,14 @@ #!/usr/bin/env python3 -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. from hello import print_hello, print_hellos diff --git a/examples/hello/examples/variance.md b/examples/hello/examples/variance.md index 5352694d23..22e3feb5ea 100644 --- a/examples/hello/examples/variance.md +++ b/examples/hello/examples/variance.md @@ -1,18 +1,13 @@ # Variance Example diff --git a/examples/hello/examples/variance.py b/examples/hello/examples/variance.py index efa71cbdc0..7675fa707e 100644 --- a/examples/hello/examples/variance.py +++ b/examples/hello/examples/variance.py @@ -1,22 +1,16 @@ #!/usr/bin/env python3 -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. from typing import Any - from hello import iota, square, sum, to_scalar from legate.core import Store diff --git a/examples/hello/hello/__init__.py b/examples/hello/hello/__init__.py index 69725303e1..606da68e68 100644 --- a/examples/hello/hello/__init__.py +++ b/examples/hello/hello/__init__.py @@ -1,19 +1,14 @@ #!/usr/bin/env python3 -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. from .library import user_lib from .hello import iota, print_hello, print_hellos, square, sum, to_scalar diff --git a/examples/hello/hello/hello.py b/examples/hello/hello/hello.py index f446584df5..239eeb4829 100644 --- a/examples/hello/hello/hello.py +++ b/examples/hello/hello/hello.py @@ -1,19 +1,14 @@ #!/usr/bin/env python3 -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. import struct from enum import IntEnum diff --git a/examples/hello/setup.py b/examples/hello/setup.py index 3dff38ed7b..12cc4d3f94 100644 --- a/examples/hello/setup.py +++ b/examples/hello/setup.py @@ -1,19 +1,15 @@ #!/usr/bin/env python3 -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import os from pathlib import Path @@ -39,12 +35,12 @@ version="0.1", description="A Hello World for Legate", author="NVIDIA Corporation", - license="Apache 2.0", + license="Proprietary", classifiers=[ "Intended Audience :: Developers", "Topic :: Database", "Topic :: Scientific/Engineering", - "License :: OSI Approved :: Apache Software License", + "License :: Proprietary :: Nvidia Proprietary", "Programming Language :: Python", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", diff --git a/examples/hello/src/CMakeLists.txt b/examples/hello/src/CMakeLists.txt index c36e7c8638..c3cdd5ed10 100644 --- a/examples/hello/src/CMakeLists.txt +++ b/examples/hello/src/CMakeLists.txt @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= legate_cpp_library_template(hello TEMPLATE_SOURCES) diff --git a/examples/hello/src/hello_cffi.h b/examples/hello/src/hello_cffi.h index f7ae54834e..1812d1271b 100644 --- a/examples/hello/src/hello_cffi.h +++ b/examples/hello/src/hello_cffi.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ enum HelloOpCode { diff --git a/examples/hello/src/hello_world.cc b/examples/hello/src/hello_world.cc index 8e2f8809ac..9d06e941c3 100644 --- a/examples/hello/src/hello_world.cc +++ b/examples/hello/src/hello_world.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "hello_world.h" diff --git a/examples/hello/src/hello_world.h b/examples/hello/src/hello_world.h index 0a9ab10759..f2c569d6f3 100644 --- a/examples/hello/src/hello_world.h +++ b/examples/hello/src/hello_world.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/examples/hello/src/iota.cc b/examples/hello/src/iota.cc index 282e418bec..5a4b39bcdd 100644 --- a/examples/hello/src/iota.cc +++ b/examples/hello/src/iota.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "hello_world.h" diff --git a/examples/hello/src/square.cc b/examples/hello/src/square.cc index b13ba088a3..d4a569cc37 100644 --- a/examples/hello/src/square.cc +++ b/examples/hello/src/square.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "hello_world.h" diff --git a/examples/hello/src/sum.cc b/examples/hello/src/sum.cc index a51dc5193a..dfbfedc8ad 100644 --- a/examples/hello/src/sum.cc +++ b/examples/hello/src/sum.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "hello_world.h" diff --git a/examples/io/CMakeLists.txt b/examples/io/CMakeLists.txt index 0b3c08844f..b4eaa9f656 100644 --- a/examples/io/CMakeLists.txt +++ b/examples/io/CMakeLists.txt @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= cmake_minimum_required(VERSION 3.22.1 FATAL_ERROR) diff --git a/examples/io/README.md b/examples/io/README.md index e372af5931..dbb2c3d238 100644 --- a/examples/io/README.md +++ b/examples/io/README.md @@ -1,18 +1,13 @@ # Legate IO example diff --git a/examples/io/editable-install.sh b/examples/io/editable-install.sh index 507e2445b9..56e3e4d02a 100755 --- a/examples/io/editable-install.sh +++ b/examples/io/editable-install.sh @@ -1,18 +1,14 @@ #!/bin/bash -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. legate_root=`python -c 'import legate.install_info as i; from pathlib import Path; print(Path(i.libpath).parent.resolve())'` echo "Using Legate at $legate_root" diff --git a/examples/io/examples/even_tiles.py b/examples/io/examples/even_tiles.py index 4a0885ff4a..c5af9905d1 100644 --- a/examples/io/examples/even_tiles.py +++ b/examples/io/examples/even_tiles.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import argparse diff --git a/examples/io/examples/single_file.py b/examples/io/examples/single_file.py index e6bfd406e8..8d8eecdffe 100644 --- a/examples/io/examples/single_file.py +++ b/examples/io/examples/single_file.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import argparse diff --git a/examples/io/examples/uneven_tiles.py b/examples/io/examples/uneven_tiles.py index ea54f79eff..567187dfb0 100644 --- a/examples/io/examples/uneven_tiles.py +++ b/examples/io/examples/uneven_tiles.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import argparse diff --git a/examples/io/install.sh b/examples/io/install.sh index 9e9ff542b0..2ad82c5835 100755 --- a/examples/io/install.sh +++ b/examples/io/install.sh @@ -1,17 +1,13 @@ #!/bin/bash -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. python -m pip install . diff --git a/examples/io/legateio/__init__.py b/examples/io/legateio/__init__.py index 27570fa55e..4a17e831d2 100644 --- a/examples/io/legateio/__init__.py +++ b/examples/io/legateio/__init__.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from .library import user_lib from .legateio import ( diff --git a/examples/io/legateio/legateio.py b/examples/io/legateio/legateio.py index ea2a33efb4..82c58ace11 100644 --- a/examples/io/legateio/legateio.py +++ b/examples/io/legateio/legateio.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import os import struct diff --git a/examples/io/setup.py b/examples/io/setup.py index 8c65f5b0bd..a62872d42d 100644 --- a/examples/io/setup.py +++ b/examples/io/setup.py @@ -1,19 +1,15 @@ #!/usr/bin/env python3 -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import os from pathlib import Path @@ -39,12 +35,12 @@ version="0.1", description="An IO example for Legate", author="NVIDIA Corporation", - license="Apache 2.0", + license="Proprietary", classifiers=[ "Intended Audience :: Developers", "Topic :: Database", "Topic :: Scientific/Engineering", - "License :: OSI Approved :: Apache Software License", + "License :: Proprietary :: Nvidia Proprietary", "Programming Language :: Python", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", diff --git a/examples/io/src/CMakeLists.txt b/examples/io/src/CMakeLists.txt index 028003b1e2..01d200db7b 100644 --- a/examples/io/src/CMakeLists.txt +++ b/examples/io/src/CMakeLists.txt @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= legate_cpp_library_template(legateio TEMPLATE_SOURCES) diff --git a/examples/io/src/legateio.cc b/examples/io/src/legateio.cc index 3570d26137..7d0a0e2440 100644 --- a/examples/io/src/legateio.cc +++ b/examples/io/src/legateio.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "legateio.h" diff --git a/examples/io/src/legateio.h b/examples/io/src/legateio.h index 300d9e0159..49e457c0e5 100644 --- a/examples/io/src/legateio.h +++ b/examples/io/src/legateio.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/examples/io/src/legateio_cffi.h b/examples/io/src/legateio_cffi.h index 0394a90c21..1d0283402b 100644 --- a/examples/io/src/legateio_cffi.h +++ b/examples/io/src/legateio_cffi.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #ifndef __LEGATE_IO_C_H__ diff --git a/examples/io/src/read_even_tiles.cc b/examples/io/src/read_even_tiles.cc index eb50a69974..80ebd5b21b 100644 --- a/examples/io/src/read_even_tiles.cc +++ b/examples/io/src/read_even_tiles.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/examples/io/src/read_file.cc b/examples/io/src/read_file.cc index 9d2c3f5601..7e67362293 100644 --- a/examples/io/src/read_file.cc +++ b/examples/io/src/read_file.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/examples/io/src/read_uneven_tiles.cc b/examples/io/src/read_uneven_tiles.cc index 68c024060d..559c3f70bb 100644 --- a/examples/io/src/read_uneven_tiles.cc +++ b/examples/io/src/read_uneven_tiles.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/examples/io/src/util.cc b/examples/io/src/util.cc index 9a6fc4bd19..11ad75fa63 100644 --- a/examples/io/src/util.cc +++ b/examples/io/src/util.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/examples/io/src/util.h b/examples/io/src/util.h index 86e6e1fa93..bbe29dc45d 100644 --- a/examples/io/src/util.h +++ b/examples/io/src/util.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/examples/io/src/write_even_tiles.cc b/examples/io/src/write_even_tiles.cc index 5c4bd4f584..d9bd42921a 100644 --- a/examples/io/src/write_even_tiles.cc +++ b/examples/io/src/write_even_tiles.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/examples/io/src/write_file.cc b/examples/io/src/write_file.cc index 8b8ea57245..69aedc88a9 100644 --- a/examples/io/src/write_file.cc +++ b/examples/io/src/write_file.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/examples/io/src/write_uneven_tiles.cc b/examples/io/src/write_uneven_tiles.cc index dc585400c5..ef62dc0a81 100644 --- a/examples/io/src/write_uneven_tiles.cc +++ b/examples/io/src/write_uneven_tiles.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/examples/reduction/CMakeLists.txt b/examples/reduction/CMakeLists.txt index ffa1a60dc4..75c989c970 100644 --- a/examples/reduction/CMakeLists.txt +++ b/examples/reduction/CMakeLists.txt @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= cmake_minimum_required(VERSION 3.22.1 FATAL_ERROR) diff --git a/examples/reduction/README.md b/examples/reduction/README.md index b9edd0e268..3c83c5cf2e 100644 --- a/examples/reduction/README.md +++ b/examples/reduction/README.md @@ -1,18 +1,13 @@ # Reduction examples diff --git a/examples/reduction/editable-install.sh b/examples/reduction/editable-install.sh index 402c903cf5..b4542cacc2 100755 --- a/examples/reduction/editable-install.sh +++ b/examples/reduction/editable-install.sh @@ -1,18 +1,14 @@ #!/bin/bash -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. legate_root=`python -c 'import legate.install_info as i; from pathlib import Path; print(Path(i.libpath).parent.resolve())'` echo "Using Legate at $legate_root" diff --git a/examples/reduction/examples/bincount.py b/examples/reduction/examples/bincount.py index fbc6b79d88..260f529ca5 100644 --- a/examples/reduction/examples/bincount.py +++ b/examples/reduction/examples/bincount.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import cunumeric as np from reduction import bincount, user_context diff --git a/examples/reduction/examples/histogram.py b/examples/reduction/examples/histogram.py index 845536bdd2..ae5e077b2c 100644 --- a/examples/reduction/examples/histogram.py +++ b/examples/reduction/examples/histogram.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import argparse diff --git a/examples/reduction/examples/matmul.py b/examples/reduction/examples/matmul.py index a01647ad68..598200b97c 100644 --- a/examples/reduction/examples/matmul.py +++ b/examples/reduction/examples/matmul.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import argparse diff --git a/examples/reduction/examples/sum_over_axis.py b/examples/reduction/examples/sum_over_axis.py index 02327c8b4a..5892beeb42 100644 --- a/examples/reduction/examples/sum_over_axis.py +++ b/examples/reduction/examples/sum_over_axis.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import cunumeric as np from reduction import sum_over_axis, user_context diff --git a/examples/reduction/examples/unique.py b/examples/reduction/examples/unique.py index 7809ea37cf..df8db764e8 100644 --- a/examples/reduction/examples/unique.py +++ b/examples/reduction/examples/unique.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import argparse diff --git a/examples/reduction/install.sh b/examples/reduction/install.sh index 9e9ff542b0..2ad82c5835 100755 --- a/examples/reduction/install.sh +++ b/examples/reduction/install.sh @@ -1,17 +1,13 @@ #!/bin/bash -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. python -m pip install . diff --git a/examples/reduction/reduction/__init__.py b/examples/reduction/reduction/__init__.py index e17ce1f73f..7c3cf9f1c9 100644 --- a/examples/reduction/reduction/__init__.py +++ b/examples/reduction/reduction/__init__.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from .library import user_lib, user_context from .reduction import ( diff --git a/examples/reduction/reduction/reduction.py b/examples/reduction/reduction/reduction.py index 58167422fa..30ebd928b1 100644 --- a/examples/reduction/reduction/reduction.py +++ b/examples/reduction/reduction/reduction.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from enum import IntEnum diff --git a/examples/reduction/setup.py b/examples/reduction/setup.py index 7f385fd76f..0561ca423d 100644 --- a/examples/reduction/setup.py +++ b/examples/reduction/setup.py @@ -1,19 +1,15 @@ #!/usr/bin/env python3 -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import os from pathlib import Path @@ -39,12 +35,12 @@ version="0.1", description="Reduction examples for Legate", author="NVIDIA Corporation", - license="Apache 2.0", + license="Proprietary", classifiers=[ "Intended Audience :: Developers", "Topic :: Database", "Topic :: Scientific/Engineering", - "License :: OSI Approved :: Apache Software License", + "License :: Proprietary :: Nvidia Proprietary", "Programming Language :: Python", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", diff --git a/examples/reduction/src/CMakeLists.txt b/examples/reduction/src/CMakeLists.txt index b18cac3dae..ec7af99049 100644 --- a/examples/reduction/src/CMakeLists.txt +++ b/examples/reduction/src/CMakeLists.txt @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= legate_cpp_library_template(reduction TEMPLATE_SOURCES) diff --git a/examples/reduction/src/bincount.cc b/examples/reduction/src/bincount.cc index ef93c10b50..266dadd6b3 100644 --- a/examples/reduction/src/bincount.cc +++ b/examples/reduction/src/bincount.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "legate_library.h" diff --git a/examples/reduction/src/categorize.cc b/examples/reduction/src/categorize.cc index f753b31e27..94ac13cb6a 100644 --- a/examples/reduction/src/categorize.cc +++ b/examples/reduction/src/categorize.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "legate_library.h" diff --git a/examples/reduction/src/histogram.cc b/examples/reduction/src/histogram.cc index 36e87ed8e7..f922126238 100644 --- a/examples/reduction/src/histogram.cc +++ b/examples/reduction/src/histogram.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "legate_library.h" diff --git a/examples/reduction/src/matmul.cc b/examples/reduction/src/matmul.cc index 7707cf3c93..5fb1064eb9 100644 --- a/examples/reduction/src/matmul.cc +++ b/examples/reduction/src/matmul.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "legate_library.h" diff --git a/examples/reduction/src/mul.cc b/examples/reduction/src/mul.cc index 72c1db5d8e..d203b5b1a4 100644 --- a/examples/reduction/src/mul.cc +++ b/examples/reduction/src/mul.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "legate_library.h" diff --git a/examples/reduction/src/reduction_cffi.h b/examples/reduction/src/reduction_cffi.h index c946e5b9e9..c2638d0784 100644 --- a/examples/reduction/src/reduction_cffi.h +++ b/examples/reduction/src/reduction_cffi.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #ifndef __REDUCTION_C_H__ diff --git a/examples/reduction/src/sum_over_axis.cc b/examples/reduction/src/sum_over_axis.cc index ae65ba6ad8..abd9631bf6 100644 --- a/examples/reduction/src/sum_over_axis.cc +++ b/examples/reduction/src/sum_over_axis.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "legate_library.h" diff --git a/examples/reduction/src/unique.cc b/examples/reduction/src/unique.cc index 515e801e87..ab44360e0e 100644 --- a/examples/reduction/src/unique.cc +++ b/examples/reduction/src/unique.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/install.py b/install.py index 22a6e2cc3a..6ca1dbc02b 100755 --- a/install.py +++ b/install.py @@ -1,19 +1,14 @@ #!/usr/bin/env python3 -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. import argparse import multiprocessing diff --git a/legate/__init__.py b/legate/__init__.py index 2c9b706ce1..65ea191050 100644 --- a/legate/__init__.py +++ b/legate/__init__.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from pkgutil import extend_path diff --git a/legate/_sphinxext/__init__.py b/legate/_sphinxext/__init__.py index dca0f13159..167966a07f 100644 --- a/legate/_sphinxext/__init__.py +++ b/legate/_sphinxext/__init__.py @@ -1,14 +1,10 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + diff --git a/legate/_sphinxext/settings.py b/legate/_sphinxext/settings.py index b50f2b5189..8c9652a0a7 100644 --- a/legate/_sphinxext/settings.py +++ b/legate/_sphinxext/settings.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import importlib diff --git a/legate/core/__init__.py b/legate/core/__init__.py index 1f182c9b82..256d5ac527 100644 --- a/legate/core/__init__.py +++ b/legate/core/__init__.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from legion_cffi import is_legion_python, ffi, lib as legion diff --git a/legate/core/_legion/__init__.py b/legate/core/_legion/__init__.py index f67fa67930..ab21d8595b 100644 --- a/legate/core/_legion/__init__.py +++ b/legate/core/_legion/__init__.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from .env import LEGATE_MAX_DIM, LEGATE_MAX_FIELDS diff --git a/legate/core/_legion/env.py b/legate/core/_legion/env.py index 3bb1a258d3..7d054db818 100644 --- a/legate/core/_legion/env.py +++ b/legate/core/_legion/env.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import os diff --git a/legate/core/_legion/field.py b/legate/core/_legion/field.py index c31031f16c..06c407ec22 100644 --- a/legate/core/_legion/field.py +++ b/legate/core/_legion/field.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations diff --git a/legate/core/_legion/future.py b/legate/core/_legion/future.py index ae89931034..a57e5ecd6f 100644 --- a/legate/core/_legion/future.py +++ b/legate/core/_legion/future.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from typing import Any, Optional diff --git a/legate/core/_legion/geometry.py b/legate/core/_legion/geometry.py index 066c62969b..41f2303338 100644 --- a/legate/core/_legion/geometry.py +++ b/legate/core/_legion/geometry.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from typing import Any, Collection, Iterator, Optional, Union diff --git a/legate/core/_legion/operation.py b/legate/core/_legion/operation.py index acafd80cf1..0e6e323983 100644 --- a/legate/core/_legion/operation.py +++ b/legate/core/_legion/operation.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from typing import TYPE_CHECKING, Any, Optional, Union diff --git a/legate/core/_legion/partition.py b/legate/core/_legion/partition.py index c0d4c5c24d..c86ccfe584 100644 --- a/legate/core/_legion/partition.py +++ b/legate/core/_legion/partition.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from typing import TYPE_CHECKING, Any, Optional diff --git a/legate/core/_legion/partition_functor.py b/legate/core/_legion/partition_functor.py index 7bd380e78f..37d10bbc36 100644 --- a/legate/core/_legion/partition_functor.py +++ b/legate/core/_legion/partition_functor.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from typing import TYPE_CHECKING, Any, Optional, Tuple, Union diff --git a/legate/core/_legion/pending.py b/legate/core/_legion/pending.py index 2f536eec01..b841a69ee1 100644 --- a/legate/core/_legion/pending.py +++ b/legate/core/_legion/pending.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from typing import Any diff --git a/legate/core/_legion/region.py b/legate/core/_legion/region.py index 60577ad1d0..1d8a616feb 100644 --- a/legate/core/_legion/region.py +++ b/legate/core/_legion/region.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import weakref diff --git a/legate/core/_legion/space.py b/legate/core/_legion/space.py index 77c529c1ea..b7cab91ee0 100644 --- a/legate/core/_legion/space.py +++ b/legate/core/_legion/space.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from typing import TYPE_CHECKING, Any, Optional, Union diff --git a/legate/core/_legion/task.py b/legate/core/_legion/task.py index 074a61a6c4..f5cd980b8e 100644 --- a/legate/core/_legion/task.py +++ b/legate/core/_legion/task.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from typing import TYPE_CHECKING, Any, Optional, Union diff --git a/legate/core/_legion/transform.py b/legate/core/_legion/transform.py index d176ef6aee..a035813516 100644 --- a/legate/core/_legion/transform.py +++ b/legate/core/_legion/transform.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from typing import TYPE_CHECKING, Any, Optional diff --git a/legate/core/_legion/util.py b/legate/core/_legion/util.py index 240e32e56a..e74469d5be 100644 --- a/legate/core/_legion/util.py +++ b/legate/core/_legion/util.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import struct diff --git a/legate/core/_lib/CMakeLists.txt b/legate/core/_lib/CMakeLists.txt index db69be4735..df69f7a2bf 100644 --- a/legate/core/_lib/CMakeLists.txt +++ b/legate/core/_lib/CMakeLists.txt @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= set(cython_sources context.pyx types.pyx) diff --git a/legate/core/_lib/context.pyx b/legate/core/_lib/context.pyx index cb16a1ef6c..79d1b42bbc 100644 --- a/legate/core/_lib/context.pyx +++ b/legate/core/_lib/context.pyx @@ -1,17 +1,12 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. import cython from libcpp cimport bool diff --git a/legate/core/_lib/types.pyx b/legate/core/_lib/types.pyx index 2ea705963f..062e539e4f 100644 --- a/legate/core/_lib/types.pyx +++ b/legate/core/_lib/types.pyx @@ -1,18 +1,12 @@ -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. from libcpp cimport bool from libcpp.memory cimport shared_ptr diff --git a/legate/core/allocation.py b/legate/core/allocation.py index b2b86f6c28..44c862edbf 100644 --- a/legate/core/allocation.py +++ b/legate/core/allocation.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from typing import TYPE_CHECKING, Any, Callable, Union diff --git a/legate/core/communicator.py b/legate/core/communicator.py index ad05620388..01615adca2 100644 --- a/legate/core/communicator.py +++ b/legate/core/communicator.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import struct diff --git a/legate/core/constraints.py b/legate/core/constraints.py index 5b910c3e32..98c773c307 100644 --- a/legate/core/constraints.py +++ b/legate/core/constraints.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from collections.abc import Iterable diff --git a/legate/core/context.py b/legate/core/context.py index 5cc15b1a3c..57245abdb7 100644 --- a/legate/core/context.py +++ b/legate/core/context.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union diff --git a/legate/core/corelib.py b/legate/core/corelib.py index 224cf8d833..4d30274ee0 100644 --- a/legate/core/corelib.py +++ b/legate/core/corelib.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import os diff --git a/legate/core/cycle_detector.py b/legate/core/cycle_detector.py index 14fb73e5ea..3f75e24bce 100644 --- a/legate/core/cycle_detector.py +++ b/legate/core/cycle_detector.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import gc import inspect diff --git a/legate/core/exception.py b/legate/core/exception.py index 5b8bace1e3..483adf2bc5 100644 --- a/legate/core/exception.py +++ b/legate/core/exception.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations diff --git a/legate/core/io.py b/legate/core/io.py index fffbd306c5..8080116b47 100644 --- a/legate/core/io.py +++ b/legate/core/io.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from typing import TYPE_CHECKING, Callable, Iterable, Optional, Union diff --git a/legate/core/launcher.py b/legate/core/launcher.py index 128234a719..53db9555b5 100644 --- a/legate/core/launcher.py +++ b/legate/core/launcher.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from dataclasses import dataclass diff --git a/legate/core/legate.py b/legate/core/legate.py index 3492a17aa5..561ce77c53 100644 --- a/legate/core/legate.py +++ b/legate/core/legate.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import platform diff --git a/legate/core/machine.py b/legate/core/machine.py index ac64b0dc1d..87c1ced9fe 100644 --- a/legate/core/machine.py +++ b/legate/core/machine.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from dataclasses import dataclass diff --git a/legate/core/operation.py b/legate/core/operation.py index 04df101a22..91baa2d5eb 100644 --- a/legate/core/operation.py +++ b/legate/core/operation.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from typing import ( diff --git a/legate/core/partition.py b/legate/core/partition.py index 6fff9cf91e..ee8d16c327 100644 --- a/legate/core/partition.py +++ b/legate/core/partition.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from abc import ABC, abstractmethod, abstractproperty diff --git a/legate/core/projection.py b/legate/core/projection.py index 18d21a8cf5..2e97d5eaac 100644 --- a/legate/core/projection.py +++ b/legate/core/projection.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from typing import Any, Callable, Optional, Tuple, Union diff --git a/legate/core/restriction.py b/legate/core/restriction.py index 8feb6a4960..e244ded10c 100644 --- a/legate/core/restriction.py +++ b/legate/core/restriction.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from enum import IntEnum, unique diff --git a/legate/core/runtime.py b/legate/core/runtime.py index 6ed5586a8c..70344d76d7 100644 --- a/legate/core/runtime.py +++ b/legate/core/runtime.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import gc diff --git a/legate/core/shape.py b/legate/core/shape.py index af84a61323..fa70e97605 100644 --- a/legate/core/shape.py +++ b/legate/core/shape.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from functools import reduce diff --git a/legate/core/solver.py b/legate/core/solver.py index 8efe888664..5fc5f4c936 100644 --- a/legate/core/solver.py +++ b/legate/core/solver.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from typing import TYPE_CHECKING, Generic, List, Optional, Tuple, TypeVar diff --git a/legate/core/store.py b/legate/core/store.py index 554623267f..6e5d7d2f31 100644 --- a/legate/core/store.py +++ b/legate/core/store.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import weakref diff --git a/legate/core/transform.py b/legate/core/transform.py index 9a7d5baa0e..ad5fe06fbe 100644 --- a/legate/core/transform.py +++ b/legate/core/transform.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from typing import TYPE_CHECKING, Protocol, Tuple diff --git a/legate/core/types.py b/legate/core/types.py index fcb8e34e57..820cd9df7a 100644 --- a/legate/core/types.py +++ b/legate/core/types.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from enum import IntEnum, unique diff --git a/legate/core/types.pyi b/legate/core/types.pyi index 9a44c9b366..15db64bb57 100644 --- a/legate/core/types.pyi +++ b/legate/core/types.pyi @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from typing import Any diff --git a/legate/core/utils.py b/legate/core/utils.py index f59277ca47..7150692257 100644 --- a/legate/core/utils.py +++ b/legate/core/utils.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import traceback diff --git a/legate/driver/__init__.py b/legate/driver/__init__.py index 786b0f069e..f3a9686c1c 100644 --- a/legate/driver/__init__.py +++ b/legate/driver/__init__.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from .config import Config diff --git a/legate/driver/args.py b/legate/driver/args.py index 25e712970a..8b715f87bd 100644 --- a/legate/driver/args.py +++ b/legate/driver/args.py @@ -1,19 +1,15 @@ #!/usr/bin/env python -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from argparse import REMAINDER, ArgumentDefaultsHelpFormatter, ArgumentParser diff --git a/legate/driver/command.py b/legate/driver/command.py index 3ac8f9ec20..6e92b0b5e1 100644 --- a/legate/driver/command.py +++ b/legate/driver/command.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import argparse diff --git a/legate/driver/config.py b/legate/driver/config.py index d5e6ebf3a0..5efe3ccb73 100644 --- a/legate/driver/config.py +++ b/legate/driver/config.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """Consolidate driver configuration from command-line and environment. """ diff --git a/legate/driver/defaults.py b/legate/driver/defaults.py index adddd23b26..698b3df6ae 100644 --- a/legate/driver/defaults.py +++ b/legate/driver/defaults.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from os import environ, getcwd diff --git a/legate/driver/driver.py b/legate/driver/driver.py index a534bf09ff..f605a2614c 100644 --- a/legate/driver/driver.py +++ b/legate/driver/driver.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from dataclasses import dataclass diff --git a/legate/driver/launcher.py b/legate/driver/launcher.py index 278345bcea..8f72c65a9e 100644 --- a/legate/driver/launcher.py +++ b/legate/driver/launcher.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import os diff --git a/legate/driver/logs.py b/legate/driver/logs.py index 2d9b29410f..7a0d6d052b 100644 --- a/legate/driver/logs.py +++ b/legate/driver/logs.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """ """ diff --git a/legate/driver/main.py b/legate/driver/main.py index bb02c7cbba..e41b755548 100644 --- a/legate/driver/main.py +++ b/legate/driver/main.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """ """ diff --git a/legate/jupyter/__init__.py b/legate/jupyter/__init__.py index c9530a071d..27a26d8cf2 100644 --- a/legate/jupyter/__init__.py +++ b/legate/jupyter/__init__.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from typing import TYPE_CHECKING diff --git a/legate/jupyter/_legion_kernel.py b/legate/jupyter/_legion_kernel.py index 812f81cf60..60ad831841 100644 --- a/legate/jupyter/_legion_kernel.py +++ b/legate/jupyter/_legion_kernel.py @@ -1,19 +1,15 @@ #!/usr/bin/env python -# Copyright 2022 Los Alamos National Laboratory, NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import sys diff --git a/legate/jupyter/args.py b/legate/jupyter/args.py index 7f80a49a13..e1c95ffc3c 100644 --- a/legate/jupyter/args.py +++ b/legate/jupyter/args.py @@ -1,19 +1,15 @@ #!/usr/bin/env python -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser diff --git a/legate/jupyter/config.py b/legate/jupyter/config.py index 141a89a5d3..7f20d06d5a 100644 --- a/legate/jupyter/config.py +++ b/legate/jupyter/config.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """Consolidate driver configuration from command-line and environment. """ diff --git a/legate/jupyter/kernel.py b/legate/jupyter/kernel.py index ae371e28f7..e95b429585 100644 --- a/legate/jupyter/kernel.py +++ b/legate/jupyter/kernel.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """Consolidate driver configuration from command-line and environment. """ diff --git a/legate/jupyter/magic.py b/legate/jupyter/magic.py index ba80f9c205..54b5c9a1fa 100644 --- a/legate/jupyter/magic.py +++ b/legate/jupyter/magic.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import os diff --git a/legate/jupyter/main.py b/legate/jupyter/main.py index d287022d3b..0423487554 100644 --- a/legate/jupyter/main.py +++ b/legate/jupyter/main.py @@ -1,19 +1,15 @@ #!/usr/bin/env python -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from legate.driver import LegateDriver diff --git a/legate/lgpatch.py b/legate/lgpatch.py index dd07da876b..95cedb8411 100644 --- a/legate/lgpatch.py +++ b/legate/lgpatch.py @@ -1,18 +1,14 @@ #! /usr/bin/env legate -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import sys import textwrap from argparse import REMAINDER, ArgumentParser, RawDescriptionHelpFormatter diff --git a/legate/settings.py b/legate/settings.py index 72b2a0b228..25a6cd6042 100644 --- a/legate/settings.py +++ b/legate/settings.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from .util.settings import ( diff --git a/legate/tester/__init__.py b/legate/tester/__init__.py index f49220c6da..b0c583337c 100644 --- a/legate/tester/__init__.py +++ b/legate/tester/__init__.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """Utilities and helpers for implementing the Cunumeric custom test runner. """ diff --git a/legate/tester/args.py b/legate/tester/args.py index 0aeee4954e..3a878acfe5 100644 --- a/legate/tester/args.py +++ b/legate/tester/args.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """Provide an argparse ArgumentParser for the test runner. """ diff --git a/legate/tester/config.py b/legate/tester/config.py index 274bcc175a..ef6a8b59a3 100644 --- a/legate/tester/config.py +++ b/legate/tester/config.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """Consolidate test configuration from command-line and environment. """ diff --git a/legate/tester/logger.py b/legate/tester/logger.py index f409042192..3430dab7ba 100644 --- a/legate/tester/logger.py +++ b/legate/tester/logger.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """Provide a basic logger that can scrub ANSI color codes. """ diff --git a/legate/tester/stages/__init__.py b/legate/tester/stages/__init__.py index fa8f916d58..dc61052328 100644 --- a/legate/tester/stages/__init__.py +++ b/legate/tester/stages/__init__.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """Provide TestStage subclasses for running configured test files using specific features. diff --git a/legate/tester/stages/_linux/__init__.py b/legate/tester/stages/_linux/__init__.py index 032305f9ca..1fbfeb8263 100644 --- a/legate/tester/stages/_linux/__init__.py +++ b/legate/tester/stages/_linux/__init__.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """Provide TestStage subclasses for running configured test files using specific features on linux platforms. diff --git a/legate/tester/stages/_linux/cpu.py b/legate/tester/stages/_linux/cpu.py index 11047c163c..6f26ebcabe 100644 --- a/legate/tester/stages/_linux/cpu.py +++ b/legate/tester/stages/_linux/cpu.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from itertools import chain diff --git a/legate/tester/stages/_linux/eager.py b/legate/tester/stages/_linux/eager.py index 1260d394bd..cd067389f5 100644 --- a/legate/tester/stages/_linux/eager.py +++ b/legate/tester/stages/_linux/eager.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from typing import TYPE_CHECKING diff --git a/legate/tester/stages/_linux/gpu.py b/legate/tester/stages/_linux/gpu.py index 0187d687be..584f3fc417 100644 --- a/legate/tester/stages/_linux/gpu.py +++ b/legate/tester/stages/_linux/gpu.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import time diff --git a/legate/tester/stages/_linux/omp.py b/legate/tester/stages/_linux/omp.py index 55ad8ffbe8..1b77d5d4b4 100644 --- a/legate/tester/stages/_linux/omp.py +++ b/legate/tester/stages/_linux/omp.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from itertools import chain diff --git a/legate/tester/stages/_osx/__init__.py b/legate/tester/stages/_osx/__init__.py index 80a7c368de..8974244eff 100644 --- a/legate/tester/stages/_osx/__init__.py +++ b/legate/tester/stages/_osx/__init__.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """Provide TestStage subclasses for running configured test files using specific features on OSX. diff --git a/legate/tester/stages/_osx/cpu.py b/legate/tester/stages/_osx/cpu.py index aaedaa6cc4..f9a65a8820 100644 --- a/legate/tester/stages/_osx/cpu.py +++ b/legate/tester/stages/_osx/cpu.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from typing import TYPE_CHECKING diff --git a/legate/tester/stages/_osx/eager.py b/legate/tester/stages/_osx/eager.py index b701ebd677..f2a302bd3c 100644 --- a/legate/tester/stages/_osx/eager.py +++ b/legate/tester/stages/_osx/eager.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from typing import TYPE_CHECKING diff --git a/legate/tester/stages/_osx/gpu.py b/legate/tester/stages/_osx/gpu.py index 789a15aa36..cd004a0fb6 100644 --- a/legate/tester/stages/_osx/gpu.py +++ b/legate/tester/stages/_osx/gpu.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import time diff --git a/legate/tester/stages/_osx/omp.py b/legate/tester/stages/_osx/omp.py index a13f4f3443..6835c6c097 100644 --- a/legate/tester/stages/_osx/omp.py +++ b/legate/tester/stages/_osx/omp.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from typing import TYPE_CHECKING diff --git a/legate/tester/stages/test_stage.py b/legate/tester/stages/test_stage.py index 51ad419315..523e0ff32b 100644 --- a/legate/tester/stages/test_stage.py +++ b/legate/tester/stages/test_stage.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import multiprocessing diff --git a/legate/tester/stages/util.py b/legate/tester/stages/util.py index 94fb14e78e..ce6506cbbc 100644 --- a/legate/tester/stages/util.py +++ b/legate/tester/stages/util.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from dataclasses import dataclass diff --git a/legate/tester/test_plan.py b/legate/tester/test_plan.py index e62ffc2619..561fb9ad7f 100644 --- a/legate/tester/test_plan.py +++ b/legate/tester/test_plan.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """Provide a TestPlan class to coordinate multiple feature test stages. """ diff --git a/legate/tester/test_system.py b/legate/tester/test_system.py index 85c4b00d6f..2d0a271b02 100644 --- a/legate/tester/test_system.py +++ b/legate/tester/test_system.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """Provide a System class to encapsulate process execution and reporting system information (number of CPUs present, etc). diff --git a/legate/timing/__init__.py b/legate/timing/__init__.py index 8610f73dda..4103d6320a 100644 --- a/legate/timing/__init__.py +++ b/legate/timing/__init__.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from legate.timing.timing import time diff --git a/legate/timing/timing.py b/legate/timing/timing.py index f5de1d5a24..1ac9ee67ad 100644 --- a/legate/timing/timing.py +++ b/legate/timing/timing.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import struct diff --git a/legate/util/__init__.py b/legate/util/__init__.py index 98636f9f74..19271d856c 100644 --- a/legate/util/__init__.py +++ b/legate/util/__init__.py @@ -1,15 +1,11 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations diff --git a/legate/util/args.py b/legate/util/args.py index cce539e67a..3a2431fe5e 100644 --- a/legate/util/args.py +++ b/legate/util/args.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import sys diff --git a/legate/util/colors.py b/legate/util/colors.py index 547f0e0159..81f1a17819 100644 --- a/legate/util/colors.py +++ b/legate/util/colors.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """Helper functions for adding colors to simple text UI output. The color functions in this module require ``colorama`` to be installed in diff --git a/legate/util/fs.py b/legate/util/fs.py index e12fb4151d..2e1de8906e 100644 --- a/legate/util/fs.py +++ b/legate/util/fs.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import re diff --git a/legate/util/info.py b/legate/util/info.py index 2ebb37e090..81703063a5 100644 --- a/legate/util/info.py +++ b/legate/util/info.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """ """ from .. import install_info as info diff --git a/legate/util/settings.py b/legate/util/settings.py index ca8c9578e1..8c25fea1b9 100644 --- a/legate/util/settings.py +++ b/legate/util/settings.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """ Control global configuration options with environment variables. Precedence diff --git a/legate/util/shared_args.py b/legate/util/shared_args.py index ac3a561f6b..f9181cf011 100644 --- a/legate/util/shared_args.py +++ b/legate/util/shared_args.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from ..driver import defaults diff --git a/legate/util/system.py b/legate/util/system.py index ba48e6ac5b..a6b6307a7a 100644 --- a/legate/util/system.py +++ b/legate/util/system.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import multiprocessing diff --git a/legate/util/types.py b/legate/util/types.py index 8c7c06bdf8..5524e53f5f 100644 --- a/legate/util/types.py +++ b/legate/util/types.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """Provide types that are useful throughout the test driver code. """ diff --git a/legate/util/ui.py b/legate/util/ui.py index 5a5927e1db..7210ba072b 100644 --- a/legate/util/ui.py +++ b/legate/util/ui.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """Helper functions for simple text UI output. The color functions in this module require ``colorama`` to be installed in diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index 46871264a9..41803a2c38 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2022 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= ############################################################################## diff --git a/legate_core_python.cmake b/legate_core_python.cmake index d23296d86f..23515e2b2d 100644 --- a/legate_core_python.cmake +++ b/legate_core_python.cmake @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2022 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= ############################################################################## diff --git a/pyproject.toml b/pyproject.toml index 7157e1e73c..c4fd86542c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,16 +1,12 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. [build-system] requires = [ diff --git a/setup.py b/setup.py index 83912f31f8..9c0bf9c8a3 100755 --- a/setup.py +++ b/setup.py @@ -1,19 +1,14 @@ #!/usr/bin/env python3 -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. from setuptools import find_packages from skbuild import setup @@ -26,12 +21,12 @@ description="legate.core - The Foundation for All Legate Libraries", url="https://github.com/nv-legate/legate.core", author="NVIDIA Corporation", - license="Apache 2.0", + license="Proprietary", classifiers=[ "Intended Audience :: Developers", "Topic :: Database", "Topic :: Scientific/Engineering", - "License :: OSI Approved :: Apache Software License", + "License :: Proprietary :: Nvidia Proprietary", "Programming Language :: Python", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", diff --git a/src/core/comm/coll.cc b/src/core/comm/coll.cc index 035ba70a7d..aa23ba82e2 100644 --- a/src/core/comm/coll.cc +++ b/src/core/comm/coll.cc @@ -1,17 +1,13 @@ -/* Copyright 2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/src/core/comm/coll.h b/src/core/comm/coll.h index 130906ec82..a917ff8974 100644 --- a/src/core/comm/coll.h +++ b/src/core/comm/coll.h @@ -1,17 +1,13 @@ -/* Copyright 2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/comm/comm.cc b/src/core/comm/comm.cc index 9507d8afe6..0d59b2c9fd 100644 --- a/src/core/comm/comm.cc +++ b/src/core/comm/comm.cc @@ -1,17 +1,13 @@ -/* Copyright 2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/comm/comm.h" diff --git a/src/core/comm/comm.h b/src/core/comm/comm.h index 9d5317db54..29f3117857 100644 --- a/src/core/comm/comm.h +++ b/src/core/comm/comm.h @@ -1,17 +1,13 @@ -/* Copyright 2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/comm/comm_cpu.cc b/src/core/comm/comm_cpu.cc index d7860e9104..7c0ac3cc06 100644 --- a/src/core/comm/comm_cpu.cc +++ b/src/core/comm/comm_cpu.cc @@ -1,17 +1,13 @@ -/* Copyright 2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/comm/comm_cpu.h" diff --git a/src/core/comm/comm_cpu.h b/src/core/comm/comm_cpu.h index b80265eef3..0744de0506 100644 --- a/src/core/comm/comm_cpu.h +++ b/src/core/comm/comm_cpu.h @@ -1,17 +1,13 @@ -/* Copyright 2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/comm/comm_nccl.cu b/src/core/comm/comm_nccl.cu index 50fe7fadfc..a10e2cb4a9 100644 --- a/src/core/comm/comm_nccl.cu +++ b/src/core/comm/comm_nccl.cu @@ -1,17 +1,13 @@ -/* Copyright 2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/comm/comm_nccl.h" diff --git a/src/core/comm/comm_nccl.h b/src/core/comm/comm_nccl.h index dca3e0cef4..04ec388d97 100644 --- a/src/core/comm/comm_nccl.h +++ b/src/core/comm/comm_nccl.h @@ -1,17 +1,13 @@ -/* Copyright 2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/comm/communicator.h b/src/core/comm/communicator.h index 67dd280c02..5241f8a782 100644 --- a/src/core/comm/communicator.h +++ b/src/core/comm/communicator.h @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/comm/local_comm.cc b/src/core/comm/local_comm.cc index e38d9e6eb4..b86ce73b59 100644 --- a/src/core/comm/local_comm.cc +++ b/src/core/comm/local_comm.cc @@ -1,17 +1,13 @@ -/* Copyright 2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/src/core/comm/mpi_comm.cc b/src/core/comm/mpi_comm.cc index 6754f7cc0c..400963203a 100644 --- a/src/core/comm/mpi_comm.cc +++ b/src/core/comm/mpi_comm.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "coll.h" diff --git a/src/core/comm/pthread_barrier.h b/src/core/comm/pthread_barrier.h index 4c62ac83b7..32a438e103 100644 --- a/src/core/comm/pthread_barrier.h +++ b/src/core/comm/pthread_barrier.h @@ -1,17 +1,13 @@ -/* Copyright 2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #if !defined(_POSIX_BARRIERS) || (_POSIX_BARRIERS < 0) diff --git a/src/core/cuda/cuda_help.h b/src/core/cuda/cuda_help.h index eee0856c1f..b7171e96ec 100644 --- a/src/core/cuda/cuda_help.h +++ b/src/core/cuda/cuda_help.h @@ -1,17 +1,13 @@ -/* Copyright 2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/cuda/stream_pool.cu b/src/core/cuda/stream_pool.cu index 49214ea0bc..5a1c920e76 100644 --- a/src/core/cuda/stream_pool.cu +++ b/src/core/cuda/stream_pool.cu @@ -1,17 +1,13 @@ -/* Copyright 2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/cuda/cuda_help.h" diff --git a/src/core/cuda/stream_pool.h b/src/core/cuda/stream_pool.h index aecd2e6af9..b79923f902 100644 --- a/src/core/cuda/stream_pool.h +++ b/src/core/cuda/stream_pool.h @@ -1,17 +1,13 @@ -/* Copyright 2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/data/allocator.cc b/src/core/data/allocator.cc index ee0a269556..42c5388cb4 100644 --- a/src/core/data/allocator.cc +++ b/src/core/data/allocator.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/data/allocator.h" diff --git a/src/core/data/allocator.h b/src/core/data/allocator.h index ec9143e599..c2b57d1a88 100644 --- a/src/core/data/allocator.h +++ b/src/core/data/allocator.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/data/buffer.h b/src/core/data/buffer.h index f706c232c4..0a0f3b72c5 100644 --- a/src/core/data/buffer.h +++ b/src/core/data/buffer.h @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/data/detail/logical_region_field.cc b/src/core/data/detail/logical_region_field.cc index 739cdcee11..42ec25b5a2 100644 --- a/src/core/data/detail/logical_region_field.cc +++ b/src/core/data/detail/logical_region_field.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/data/detail/logical_region_field.h" diff --git a/src/core/data/detail/logical_region_field.h b/src/core/data/detail/logical_region_field.h index 0accd7492f..cd9ce6a024 100644 --- a/src/core/data/detail/logical_region_field.h +++ b/src/core/data/detail/logical_region_field.h @@ -1,17 +1,13 @@ -/* Copyright 2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/data/detail/logical_store.cc b/src/core/data/detail/logical_store.cc index d063dd03cd..0b3af76da3 100644 --- a/src/core/data/detail/logical_store.cc +++ b/src/core/data/detail/logical_store.cc @@ -1,17 +1,13 @@ -/* Copyright 2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/data/detail/logical_store.h" diff --git a/src/core/data/detail/logical_store.h b/src/core/data/detail/logical_store.h index 097b8fde95..0bc08f463e 100644 --- a/src/core/data/detail/logical_store.h +++ b/src/core/data/detail/logical_store.h @@ -1,17 +1,13 @@ -/* Copyright 2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/data/detail/scalar.cc b/src/core/data/detail/scalar.cc index 01fd4b5738..e8de5bebc6 100644 --- a/src/core/data/detail/scalar.cc +++ b/src/core/data/detail/scalar.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/data/detail/scalar.h" diff --git a/src/core/data/detail/scalar.h b/src/core/data/detail/scalar.h index 904c45d2c3..25e6b1be68 100644 --- a/src/core/data/detail/scalar.h +++ b/src/core/data/detail/scalar.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/data/detail/store.cc b/src/core/data/detail/store.cc index 7599d1892c..4e8a256dd3 100644 --- a/src/core/data/detail/store.cc +++ b/src/core/data/detail/store.cc @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/data/detail/store.h" diff --git a/src/core/data/detail/store.h b/src/core/data/detail/store.h index 0afad392e7..9cb9d0c1a4 100644 --- a/src/core/data/detail/store.h +++ b/src/core/data/detail/store.h @@ -1,17 +1,13 @@ -/* Cop)yright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/data/detail/transform.cc b/src/core/data/detail/transform.cc index 1ce0fa9134..59c740219d 100644 --- a/src/core/data/detail/transform.cc +++ b/src/core/data/detail/transform.cc @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/data/detail/transform.h" diff --git a/src/core/data/detail/transform.h b/src/core/data/detail/transform.h index 44b3e69b53..14add35bc8 100644 --- a/src/core/data/detail/transform.h +++ b/src/core/data/detail/transform.h @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index ec498b9f21..8d5dcedbb5 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -1,17 +1,13 @@ -/* Copyright 2021 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index 18539872ee..65ac7fd8e7 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -1,17 +1,13 @@ -/* Copyright 2021 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/data/scalar.cc b/src/core/data/scalar.cc index 7e6dc5221f..0e56826dd5 100644 --- a/src/core/data/scalar.cc +++ b/src/core/data/scalar.cc @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/data/scalar.h" diff --git a/src/core/data/scalar.h b/src/core/data/scalar.h index 542c5f3179..5b178773b6 100644 --- a/src/core/data/scalar.h +++ b/src/core/data/scalar.h @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/data/scalar.inl b/src/core/data/scalar.inl index a492eb2ad1..3f1b8b8543 100644 --- a/src/core/data/scalar.inl +++ b/src/core/data/scalar.inl @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/data/shape.cc b/src/core/data/shape.cc index a415a88d78..77d83c3739 100644 --- a/src/core/data/shape.cc +++ b/src/core/data/shape.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/data/shape.h" diff --git a/src/core/data/shape.h b/src/core/data/shape.h index 594415ea0c..c39276920d 100644 --- a/src/core/data/shape.h +++ b/src/core/data/shape.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/data/slice.h b/src/core/data/slice.h index 72280b8ddb..26bd5a330d 100644 --- a/src/core/data/slice.h +++ b/src/core/data/slice.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/data/store.cc b/src/core/data/store.cc index 8b89cd7cf3..c78e749e50 100644 --- a/src/core/data/store.cc +++ b/src/core/data/store.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/data/store.h" diff --git a/src/core/data/store.h b/src/core/data/store.h index 790f16e73b..df248a62fb 100644 --- a/src/core/data/store.h +++ b/src/core/data/store.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/data/store.inl b/src/core/data/store.inl index c294349057..27439cbe09 100644 --- a/src/core/data/store.inl +++ b/src/core/data/store.inl @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/legate_c.cc b/src/core/legate_c.cc index 7d3f4667d0..5c84235b7f 100644 --- a/src/core/legate_c.cc +++ b/src/core/legate_c.cc @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/runtime/runtime.h" diff --git a/src/core/legate_c.h b/src/core/legate_c.h index 7ac8c2a727..8506e5c94b 100644 --- a/src/core/legate_c.h +++ b/src/core/legate_c.h @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #ifndef __LEGATE_C_H__ diff --git a/src/core/mapping/detail/base_mapper.cc b/src/core/mapping/detail/base_mapper.cc index ebcc09fe34..80d2f8efc4 100644 --- a/src/core/mapping/detail/base_mapper.cc +++ b/src/core/mapping/detail/base_mapper.cc @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/mapping/detail/base_mapper.h" diff --git a/src/core/mapping/detail/base_mapper.h b/src/core/mapping/detail/base_mapper.h index 369275ce0d..005ed6dc02 100644 --- a/src/core/mapping/detail/base_mapper.h +++ b/src/core/mapping/detail/base_mapper.h @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/mapping/detail/core_mapper.cc b/src/core/mapping/detail/core_mapper.cc index ebb2a176a6..e51569be26 100644 --- a/src/core/mapping/detail/core_mapper.cc +++ b/src/core/mapping/detail/core_mapper.cc @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "env_defaults.h" diff --git a/src/core/mapping/detail/core_mapper.h b/src/core/mapping/detail/core_mapper.h index a7268c0697..06e3430361 100644 --- a/src/core/mapping/detail/core_mapper.h +++ b/src/core/mapping/detail/core_mapper.h @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/mapping/detail/default_mapper.cc b/src/core/mapping/detail/default_mapper.cc index b29d1ded6e..e5de73f571 100644 --- a/src/core/mapping/detail/default_mapper.cc +++ b/src/core/mapping/detail/default_mapper.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/mapping/detail/default_mapper.h" diff --git a/src/core/mapping/detail/default_mapper.h b/src/core/mapping/detail/default_mapper.h index 80d0f5d45b..d179e41690 100644 --- a/src/core/mapping/detail/default_mapper.h +++ b/src/core/mapping/detail/default_mapper.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/mapping/mapping.h" diff --git a/src/core/mapping/detail/instance_manager.cc b/src/core/mapping/detail/instance_manager.cc index 5269183722..2dfb4fa9c8 100644 --- a/src/core/mapping/detail/instance_manager.cc +++ b/src/core/mapping/detail/instance_manager.cc @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/mapping/detail/instance_manager.h" diff --git a/src/core/mapping/detail/instance_manager.h b/src/core/mapping/detail/instance_manager.h index f1755594ac..4f2b8a2e7b 100644 --- a/src/core/mapping/detail/instance_manager.h +++ b/src/core/mapping/detail/instance_manager.h @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/mapping/detail/machine.cc b/src/core/mapping/detail/machine.cc index db186f38ba..e99f8b74e1 100644 --- a/src/core/mapping/detail/machine.cc +++ b/src/core/mapping/detail/machine.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/mapping/detail/machine.h" diff --git a/src/core/mapping/detail/machine.h b/src/core/mapping/detail/machine.h index 1665cdd0a5..b949f2dd6f 100644 --- a/src/core/mapping/detail/machine.h +++ b/src/core/mapping/detail/machine.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/mapping/detail/mapping.cc b/src/core/mapping/detail/mapping.cc index 7942f2c080..f00e232f66 100644 --- a/src/core/mapping/detail/mapping.cc +++ b/src/core/mapping/detail/mapping.cc @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/mapping/detail/mapping.h" diff --git a/src/core/mapping/detail/mapping.h b/src/core/mapping/detail/mapping.h index e8f19fd221..40eb0d586d 100644 --- a/src/core/mapping/detail/mapping.h +++ b/src/core/mapping/detail/mapping.h @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/mapping/detail/operation.cc b/src/core/mapping/detail/operation.cc index 7a06149a4a..80c64380bc 100644 --- a/src/core/mapping/detail/operation.cc +++ b/src/core/mapping/detail/operation.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/mapping/detail/operation.h" diff --git a/src/core/mapping/detail/operation.h b/src/core/mapping/detail/operation.h index b185fed29c..9a812aa80a 100644 --- a/src/core/mapping/detail/operation.h +++ b/src/core/mapping/detail/operation.h @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/mapping/detail/operation.inl b/src/core/mapping/detail/operation.inl index 22e4bf9408..964dcfbd7b 100644 --- a/src/core/mapping/detail/operation.inl +++ b/src/core/mapping/detail/operation.inl @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/mapping/detail/store.cc b/src/core/mapping/detail/store.cc index 81592703bf..f14db69470 100644 --- a/src/core/mapping/detail/store.cc +++ b/src/core/mapping/detail/store.cc @@ -1,17 +1,13 @@ -/* Copyright 2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/mapping/detail/store.h" diff --git a/src/core/mapping/detail/store.h b/src/core/mapping/detail/store.h index e6725fe368..9478dd019d 100644 --- a/src/core/mapping/detail/store.h +++ b/src/core/mapping/detail/store.h @@ -1,17 +1,13 @@ -/* Copyright 2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/mapping/machine.cc b/src/core/mapping/machine.cc index 83fc6ebe33..3145044600 100644 --- a/src/core/mapping/machine.cc +++ b/src/core/mapping/machine.cc @@ -1,17 +1,13 @@ -/* Copyright 2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/mapping/machine.h" diff --git a/src/core/mapping/machine.h b/src/core/mapping/machine.h index 9a85518513..b50e6ca364 100644 --- a/src/core/mapping/machine.h +++ b/src/core/mapping/machine.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/mapping/mapping.cc b/src/core/mapping/mapping.cc index e4efeef977..fbf09bd604 100644 --- a/src/core/mapping/mapping.cc +++ b/src/core/mapping/mapping.cc @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/mapping/mapping.h" diff --git a/src/core/mapping/mapping.h b/src/core/mapping/mapping.h index 4e51b54505..1de024ade0 100644 --- a/src/core/mapping/mapping.h +++ b/src/core/mapping/mapping.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/mapping/operation.cc b/src/core/mapping/operation.cc index af2b5cc76f..ddb23aff4a 100644 --- a/src/core/mapping/operation.cc +++ b/src/core/mapping/operation.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/mapping/operation.h" diff --git a/src/core/mapping/operation.h b/src/core/mapping/operation.h index 71b325d32e..7f1084361f 100644 --- a/src/core/mapping/operation.h +++ b/src/core/mapping/operation.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/mapping/store.cc b/src/core/mapping/store.cc index ce39ac28c9..307de709d1 100644 --- a/src/core/mapping/store.cc +++ b/src/core/mapping/store.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/mapping/store.h" diff --git a/src/core/mapping/store.h b/src/core/mapping/store.h index 31c211eb00..6a1be37baf 100644 --- a/src/core/mapping/store.h +++ b/src/core/mapping/store.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/operation/detail/copy.cc b/src/core/operation/detail/copy.cc index 585f7140c3..88c032a551 100644 --- a/src/core/operation/detail/copy.cc +++ b/src/core/operation/detail/copy.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/operation/detail/copy.h" diff --git a/src/core/operation/detail/copy.h b/src/core/operation/detail/copy.h index e04086cb2d..936c9e79f4 100644 --- a/src/core/operation/detail/copy.h +++ b/src/core/operation/detail/copy.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/operation/detail/copy_launcher.cc b/src/core/operation/detail/copy_launcher.cc index c9747b89be..b4d53697a1 100644 --- a/src/core/operation/detail/copy_launcher.cc +++ b/src/core/operation/detail/copy_launcher.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/operation/detail/copy_launcher.h" diff --git a/src/core/operation/detail/copy_launcher.h b/src/core/operation/detail/copy_launcher.h index c4a298a76b..610b3907e3 100644 --- a/src/core/operation/detail/copy_launcher.h +++ b/src/core/operation/detail/copy_launcher.h @@ -1,17 +1,13 @@ -/* Copyright 2021 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/operation/detail/fill.cc b/src/core/operation/detail/fill.cc index 0a14326c7e..a8e9ae3bc1 100644 --- a/src/core/operation/detail/fill.cc +++ b/src/core/operation/detail/fill.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/operation/detail/fill.h" diff --git a/src/core/operation/detail/fill.h b/src/core/operation/detail/fill.h index 17956ec794..f330a0ed52 100644 --- a/src/core/operation/detail/fill.h +++ b/src/core/operation/detail/fill.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/operation/detail/fill_launcher.cc b/src/core/operation/detail/fill_launcher.cc index e154138c6e..e88f3877ca 100644 --- a/src/core/operation/detail/fill_launcher.cc +++ b/src/core/operation/detail/fill_launcher.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/operation/detail/fill_launcher.h" diff --git a/src/core/operation/detail/fill_launcher.h b/src/core/operation/detail/fill_launcher.h index c77118970e..b28a5f245d 100644 --- a/src/core/operation/detail/fill_launcher.h +++ b/src/core/operation/detail/fill_launcher.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/operation/detail/gather.cc b/src/core/operation/detail/gather.cc index 8b658859a2..40334fd2ea 100644 --- a/src/core/operation/detail/gather.cc +++ b/src/core/operation/detail/gather.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/operation/detail/gather.h" diff --git a/src/core/operation/detail/gather.h b/src/core/operation/detail/gather.h index c584c92b8f..a9244c7683 100644 --- a/src/core/operation/detail/gather.h +++ b/src/core/operation/detail/gather.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/operation/detail/launcher_arg.cc b/src/core/operation/detail/launcher_arg.cc index ad992a2ba6..c63b415bf9 100644 --- a/src/core/operation/detail/launcher_arg.cc +++ b/src/core/operation/detail/launcher_arg.cc @@ -1,17 +1,13 @@ -/* Copyright 2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/operation/detail/launcher_arg.h" diff --git a/src/core/operation/detail/launcher_arg.h b/src/core/operation/detail/launcher_arg.h index 2ec7121540..be4faccf45 100644 --- a/src/core/operation/detail/launcher_arg.h +++ b/src/core/operation/detail/launcher_arg.h @@ -1,17 +1,13 @@ -/* Copyright 2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/operation/detail/operation.cc b/src/core/operation/detail/operation.cc index 8abc7fcf6f..5cb0acb09b 100644 --- a/src/core/operation/detail/operation.cc +++ b/src/core/operation/detail/operation.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/operation/detail/operation.h" diff --git a/src/core/operation/detail/operation.h b/src/core/operation/detail/operation.h index 6257a56261..60b8f8c905 100644 --- a/src/core/operation/detail/operation.h +++ b/src/core/operation/detail/operation.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/operation/detail/projection.cc b/src/core/operation/detail/projection.cc index 1d00b12613..57844a3d58 100644 --- a/src/core/operation/detail/projection.cc +++ b/src/core/operation/detail/projection.cc @@ -1,17 +1,13 @@ -/* Copyright 2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/operation/detail/projection.h" diff --git a/src/core/operation/detail/projection.h b/src/core/operation/detail/projection.h index c4dab4828b..5b41146b41 100644 --- a/src/core/operation/detail/projection.h +++ b/src/core/operation/detail/projection.h @@ -1,17 +1,13 @@ -/* Copyright 2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/operation/detail/req_analyzer.cc b/src/core/operation/detail/req_analyzer.cc index 006f567ed9..f08245a7b5 100644 --- a/src/core/operation/detail/req_analyzer.cc +++ b/src/core/operation/detail/req_analyzer.cc @@ -1,17 +1,13 @@ -/* Copyright 2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/operation/detail/req_analyzer.h" diff --git a/src/core/operation/detail/req_analyzer.h b/src/core/operation/detail/req_analyzer.h index e2109d512e..110e7df158 100644 --- a/src/core/operation/detail/req_analyzer.h +++ b/src/core/operation/detail/req_analyzer.h @@ -1,17 +1,13 @@ -/* Copyright 2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/operation/detail/scatter.cc b/src/core/operation/detail/scatter.cc index e2df1c2dc7..1a39e34d4d 100644 --- a/src/core/operation/detail/scatter.cc +++ b/src/core/operation/detail/scatter.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/operation/detail/scatter.h" diff --git a/src/core/operation/detail/scatter.h b/src/core/operation/detail/scatter.h index 1880a6fdeb..15a625f81b 100644 --- a/src/core/operation/detail/scatter.h +++ b/src/core/operation/detail/scatter.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/operation/detail/scatter_gather.cc b/src/core/operation/detail/scatter_gather.cc index 41cb1cc203..00238d0cbb 100644 --- a/src/core/operation/detail/scatter_gather.cc +++ b/src/core/operation/detail/scatter_gather.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/operation/detail/scatter_gather.h" diff --git a/src/core/operation/detail/scatter_gather.h b/src/core/operation/detail/scatter_gather.h index b3677573c8..ef9c9e5982 100644 --- a/src/core/operation/detail/scatter_gather.h +++ b/src/core/operation/detail/scatter_gather.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/operation/detail/task.cc b/src/core/operation/detail/task.cc index 36e898f972..71dbf204f3 100644 --- a/src/core/operation/detail/task.cc +++ b/src/core/operation/detail/task.cc @@ -1,17 +1,13 @@ -/* Copyright 2021 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/operation/detail/task.h" diff --git a/src/core/operation/detail/task.h b/src/core/operation/detail/task.h index a10c457a94..d118d249c9 100644 --- a/src/core/operation/detail/task.h +++ b/src/core/operation/detail/task.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/operation/detail/task_launcher.cc b/src/core/operation/detail/task_launcher.cc index f4970f3d28..af3de34ff1 100644 --- a/src/core/operation/detail/task_launcher.cc +++ b/src/core/operation/detail/task_launcher.cc @@ -1,17 +1,13 @@ -/* Copyright 2021 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/operation/detail/task_launcher.h" diff --git a/src/core/operation/detail/task_launcher.h b/src/core/operation/detail/task_launcher.h index 9bbff0a159..3872ade358 100644 --- a/src/core/operation/detail/task_launcher.h +++ b/src/core/operation/detail/task_launcher.h @@ -1,17 +1,13 @@ -/* Copyright 2021 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/operation/task.cc b/src/core/operation/task.cc index 56c028be2f..91d4c69541 100644 --- a/src/core/operation/task.cc +++ b/src/core/operation/task.cc @@ -1,17 +1,13 @@ -/* Copyright 2021 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/operation/task.h" diff --git a/src/core/operation/task.h b/src/core/operation/task.h index d5527aca35..9feb64b078 100644 --- a/src/core/operation/task.h +++ b/src/core/operation/task.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/partitioning/constraint.cc b/src/core/partitioning/constraint.cc index 2fc94ff6c2..5486b9ec9c 100644 --- a/src/core/partitioning/constraint.cc +++ b/src/core/partitioning/constraint.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/partitioning/constraint.h" diff --git a/src/core/partitioning/constraint.h b/src/core/partitioning/constraint.h index 6b6000c900..fd520e69d5 100644 --- a/src/core/partitioning/constraint.h +++ b/src/core/partitioning/constraint.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/partitioning/detail/constraint.cc b/src/core/partitioning/detail/constraint.cc index e24f3a0287..1c0c1efa76 100644 --- a/src/core/partitioning/detail/constraint.cc +++ b/src/core/partitioning/detail/constraint.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/partitioning/detail/constraint.h" diff --git a/src/core/partitioning/detail/constraint.h b/src/core/partitioning/detail/constraint.h index 2c35a86dde..405cc99414 100644 --- a/src/core/partitioning/detail/constraint.h +++ b/src/core/partitioning/detail/constraint.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/partitioning/detail/constraint_solver.cc b/src/core/partitioning/detail/constraint_solver.cc index 149c55db0a..7fcd7930e2 100644 --- a/src/core/partitioning/detail/constraint_solver.cc +++ b/src/core/partitioning/detail/constraint_solver.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/partitioning/detail/constraint_solver.h" diff --git a/src/core/partitioning/detail/constraint_solver.h b/src/core/partitioning/detail/constraint_solver.h index 67109fdd5e..8338d21170 100644 --- a/src/core/partitioning/detail/constraint_solver.h +++ b/src/core/partitioning/detail/constraint_solver.h @@ -1,17 +1,13 @@ -/* Copyright 2021 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/partitioning/detail/partitioner.cc b/src/core/partitioning/detail/partitioner.cc index ff746b6b27..a903cd3350 100644 --- a/src/core/partitioning/detail/partitioner.cc +++ b/src/core/partitioning/detail/partitioner.cc @@ -1,17 +1,13 @@ -/* Copyright 2021 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/partitioning/detail/partitioner.h" diff --git a/src/core/partitioning/detail/partitioner.h b/src/core/partitioning/detail/partitioner.h index 9576b11283..f332d7a535 100644 --- a/src/core/partitioning/detail/partitioner.h +++ b/src/core/partitioning/detail/partitioner.h @@ -1,17 +1,13 @@ -/* Copyright 2021 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index cfd3cada26..374a2e3d2c 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -1,17 +1,13 @@ -/* Copyright 2021 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/src/core/partitioning/partition.h b/src/core/partitioning/partition.h index fed9ed4712..e19ae94793 100644 --- a/src/core/partitioning/partition.h +++ b/src/core/partitioning/partition.h @@ -1,17 +1,13 @@ -/* Copyright 2021 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/partitioning/restriction.cc b/src/core/partitioning/restriction.cc index 568c3fe34b..f53922fae0 100644 --- a/src/core/partitioning/restriction.cc +++ b/src/core/partitioning/restriction.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/partitioning/restriction.h" diff --git a/src/core/partitioning/restriction.h b/src/core/partitioning/restriction.h index 143c398428..d5ce8f855f 100644 --- a/src/core/partitioning/restriction.h +++ b/src/core/partitioning/restriction.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/runtime/detail/communicator_manager.cc b/src/core/runtime/detail/communicator_manager.cc index 62b8cbeb16..ff0263e5f9 100644 --- a/src/core/runtime/detail/communicator_manager.cc +++ b/src/core/runtime/detail/communicator_manager.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/runtime/detail/communicator_manager.h" diff --git a/src/core/runtime/detail/communicator_manager.h b/src/core/runtime/detail/communicator_manager.h index 77258e5a46..96b75c0390 100644 --- a/src/core/runtime/detail/communicator_manager.h +++ b/src/core/runtime/detail/communicator_manager.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/runtime/detail/field_manager.cc b/src/core/runtime/detail/field_manager.cc index c167e5b418..d28c025d2f 100644 --- a/src/core/runtime/detail/field_manager.cc +++ b/src/core/runtime/detail/field_manager.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/runtime/detail/field_manager.h" diff --git a/src/core/runtime/detail/field_manager.h b/src/core/runtime/detail/field_manager.h index 3e5384ff73..d1abd1b23d 100644 --- a/src/core/runtime/detail/field_manager.h +++ b/src/core/runtime/detail/field_manager.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/runtime/detail/library.cc b/src/core/runtime/detail/library.cc index fbfdf07fe0..9a6b467fb3 100644 --- a/src/core/runtime/detail/library.cc +++ b/src/core/runtime/detail/library.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/runtime/detail/library.h" diff --git a/src/core/runtime/detail/library.h b/src/core/runtime/detail/library.h index c89818721d..6750576861 100644 --- a/src/core/runtime/detail/library.h +++ b/src/core/runtime/detail/library.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/runtime/detail/machine_manager.cc b/src/core/runtime/detail/machine_manager.cc index e41f7529be..6a1363f9cf 100644 --- a/src/core/runtime/detail/machine_manager.cc +++ b/src/core/runtime/detail/machine_manager.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/runtime/detail/machine_manager.h" diff --git a/src/core/runtime/detail/machine_manager.h b/src/core/runtime/detail/machine_manager.h index b7ccad662d..20d0285609 100644 --- a/src/core/runtime/detail/machine_manager.h +++ b/src/core/runtime/detail/machine_manager.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/runtime/detail/partition_manager.cc b/src/core/runtime/detail/partition_manager.cc index a511f5474c..9bc6077d94 100644 --- a/src/core/runtime/detail/partition_manager.cc +++ b/src/core/runtime/detail/partition_manager.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/runtime/detail/partition_manager.h" diff --git a/src/core/runtime/detail/partition_manager.h b/src/core/runtime/detail/partition_manager.h index 94e6872d58..12b9355103 100644 --- a/src/core/runtime/detail/partition_manager.h +++ b/src/core/runtime/detail/partition_manager.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/runtime/detail/projection.cc b/src/core/runtime/detail/projection.cc index 17440f113e..4b8d56eaeb 100644 --- a/src/core/runtime/detail/projection.cc +++ b/src/core/runtime/detail/projection.cc @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/src/core/runtime/detail/projection.h b/src/core/runtime/detail/projection.h index 389b3403d1..a1040213ba 100644 --- a/src/core/runtime/detail/projection.h +++ b/src/core/runtime/detail/projection.h @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/runtime/detail/provenance_manager.cc b/src/core/runtime/detail/provenance_manager.cc index 5603ed9f35..721a1da750 100644 --- a/src/core/runtime/detail/provenance_manager.cc +++ b/src/core/runtime/detail/provenance_manager.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/runtime/detail/provenance_manager.h" diff --git a/src/core/runtime/detail/provenance_manager.h b/src/core/runtime/detail/provenance_manager.h index d478365626..6ae5ed4bf6 100644 --- a/src/core/runtime/detail/provenance_manager.h +++ b/src/core/runtime/detail/provenance_manager.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/runtime/detail/region_manager.cc b/src/core/runtime/detail/region_manager.cc index ed982f49b2..def21a20c1 100644 --- a/src/core/runtime/detail/region_manager.cc +++ b/src/core/runtime/detail/region_manager.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/runtime/detail/region_manager.h" diff --git a/src/core/runtime/detail/region_manager.h b/src/core/runtime/detail/region_manager.h index bcbe024e87..42aa9979d5 100644 --- a/src/core/runtime/detail/region_manager.h +++ b/src/core/runtime/detail/region_manager.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index a0e532db36..cafaedfe73 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/runtime/detail/runtime.h" diff --git a/src/core/runtime/detail/runtime.h b/src/core/runtime/detail/runtime.h index 37fce3437a..d638364c67 100644 --- a/src/core/runtime/detail/runtime.h +++ b/src/core/runtime/detail/runtime.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/runtime/detail/shard.cc b/src/core/runtime/detail/shard.cc index 0f1094c7e8..1cd543c0d8 100644 --- a/src/core/runtime/detail/shard.cc +++ b/src/core/runtime/detail/shard.cc @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/runtime/detail/shard.h" diff --git a/src/core/runtime/detail/shard.h b/src/core/runtime/detail/shard.h index 2db94047d9..3cac7edc2c 100644 --- a/src/core/runtime/detail/shard.h +++ b/src/core/runtime/detail/shard.h @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/runtime/library.cc b/src/core/runtime/library.cc index 6f28e8088d..398da76196 100644 --- a/src/core/runtime/library.cc +++ b/src/core/runtime/library.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/runtime/library.h" diff --git a/src/core/runtime/library.h b/src/core/runtime/library.h index eb78e83206..8033d11df2 100644 --- a/src/core/runtime/library.h +++ b/src/core/runtime/library.h @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/runtime/library.inl b/src/core/runtime/library.inl index 91b8214d7d..f248d37b03 100644 --- a/src/core/runtime/library.inl +++ b/src/core/runtime/library.inl @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/runtime/resource.h b/src/core/runtime/resource.h index b546d45c31..9a331831a2 100644 --- a/src/core/runtime/resource.h +++ b/src/core/runtime/resource.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 9a87c62e85..7582a2f1ec 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -1,17 +1,13 @@ -/* Copyright 2021-2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/runtime/runtime.h" diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 47b10631d3..f61ff7d2df 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -1,17 +1,13 @@ -/* Copyright 2021-2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/runtime/runtime.inl b/src/core/runtime/runtime.inl index f10a7e8746..3d895c48f7 100644 --- a/src/core/runtime/runtime.inl +++ b/src/core/runtime/runtime.inl @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/runtime/tracker.cc b/src/core/runtime/tracker.cc index 3a08cd3eed..74e796f7ef 100644 --- a/src/core/runtime/tracker.cc +++ b/src/core/runtime/tracker.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/runtime/tracker.h" diff --git a/src/core/runtime/tracker.h b/src/core/runtime/tracker.h index a4b30338bb..d3be930658 100644 --- a/src/core/runtime/tracker.h +++ b/src/core/runtime/tracker.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/src/core/task/detail/return.cc b/src/core/task/detail/return.cc index 2d48589ce5..463c60dc5a 100644 --- a/src/core/task/detail/return.cc +++ b/src/core/task/detail/return.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/task/detail/return.h" diff --git a/src/core/task/detail/return.h b/src/core/task/detail/return.h index 11ce7958e0..eab7b76553 100644 --- a/src/core/task/detail/return.h +++ b/src/core/task/detail/return.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/task/detail/task_context.cc b/src/core/task/detail/task_context.cc index d66a9e2eb4..3df8d6eb48 100644 --- a/src/core/task/detail/task_context.cc +++ b/src/core/task/detail/task_context.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/task/detail/task_context.h" diff --git a/src/core/task/detail/task_context.h b/src/core/task/detail/task_context.h index f3116de124..0464d01099 100644 --- a/src/core/task/detail/task_context.h +++ b/src/core/task/detail/task_context.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/task/exception.h b/src/core/task/exception.h index 7d756cd3d3..06fabfb9f8 100644 --- a/src/core/task/exception.h +++ b/src/core/task/exception.h @@ -1,17 +1,13 @@ -/* Copyright 2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/task/registrar.cc b/src/core/task/registrar.cc index 193381fc26..dce60ab8dd 100644 --- a/src/core/task/registrar.cc +++ b/src/core/task/registrar.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/task/registrar.h" diff --git a/src/core/task/registrar.h b/src/core/task/registrar.h index 7cb04ab875..0f5b2c9ce4 100644 --- a/src/core/task/registrar.h +++ b/src/core/task/registrar.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/task/task.cc b/src/core/task/task.cc index cfbd61046a..66c629ea0c 100644 --- a/src/core/task/task.cc +++ b/src/core/task/task.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/task/task.h" diff --git a/src/core/task/task.h b/src/core/task/task.h index 39561e3ee6..b63dbcb49b 100644 --- a/src/core/task/task.h +++ b/src/core/task/task.h @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/task/task.inl b/src/core/task/task.inl index 02295a7a7a..eee9cb788e 100644 --- a/src/core/task/task.inl +++ b/src/core/task/task.inl @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/task/task_context.cc b/src/core/task/task_context.cc index 2fb5405a9e..85180a4d5f 100644 --- a/src/core/task/task_context.cc +++ b/src/core/task/task_context.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/task/task_context.h" diff --git a/src/core/task/task_context.h b/src/core/task/task_context.h index b18c4557f9..cc06d00256 100644 --- a/src/core/task/task_context.h +++ b/src/core/task/task_context.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/task/task_info.cc b/src/core/task/task_info.cc index 5f9eb9fa5f..e1b8c97390 100644 --- a/src/core/task/task_info.cc +++ b/src/core/task/task_info.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/task/task_info.h" diff --git a/src/core/task/task_info.h b/src/core/task/task_info.h index 96170e4ce5..607c2a006f 100644 --- a/src/core/task/task_info.h +++ b/src/core/task/task_info.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/task/variant_helper.h b/src/core/task/variant_helper.h index 9b16649aca..96001e22ec 100644 --- a/src/core/task/variant_helper.h +++ b/src/core/task/variant_helper.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/task/variant_options.cc b/src/core/task/variant_options.cc index fea99fe318..aa4904c02e 100644 --- a/src/core/task/variant_options.cc +++ b/src/core/task/variant_options.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/task/variant_options.h" diff --git a/src/core/task/variant_options.h b/src/core/task/variant_options.h index cf56f701aa..188b1cee39 100644 --- a/src/core/task/variant_options.h +++ b/src/core/task/variant_options.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/type/detail/type_info.cc b/src/core/type/detail/type_info.cc index 3b90d9d1b9..d3bdd0be52 100644 --- a/src/core/type/detail/type_info.cc +++ b/src/core/type/detail/type_info.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/src/core/type/detail/type_info.h b/src/core/type/detail/type_info.h index ad92349849..ec3a41810b 100644 --- a/src/core/type/detail/type_info.h +++ b/src/core/type/detail/type_info.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/type/type_info.cc b/src/core/type/type_info.cc index 667679ed7d..42082c8a48 100644 --- a/src/core/type/type_info.cc +++ b/src/core/type/type_info.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/type/type_info.h" diff --git a/src/core/type/type_info.h b/src/core/type/type_info.h index 4dcfa7b3d8..18f8e4cdf2 100644 --- a/src/core/type/type_info.h +++ b/src/core/type/type_info.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/type/type_traits.h b/src/core/type/type_traits.h index f6ebba8b3e..3b2f820351 100644 --- a/src/core/type/type_traits.h +++ b/src/core/type/type_traits.h @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/utilities/debug.cc b/src/core/utilities/debug.cc index 1f7bd3358c..37c1f69155 100644 --- a/src/core/utilities/debug.cc +++ b/src/core/utilities/debug.cc @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/utilities/debug.h" diff --git a/src/core/utilities/debug.h b/src/core/utilities/debug.h index d5e86cac46..3d72c7bd39 100644 --- a/src/core/utilities/debug.h +++ b/src/core/utilities/debug.h @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/utilities/deserializer.cc b/src/core/utilities/deserializer.cc index 52fbcd5310..224f46ec96 100644 --- a/src/core/utilities/deserializer.cc +++ b/src/core/utilities/deserializer.cc @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/utilities/deserializer.h" diff --git a/src/core/utilities/deserializer.h b/src/core/utilities/deserializer.h index 4ba6724a58..4f4f285625 100644 --- a/src/core/utilities/deserializer.h +++ b/src/core/utilities/deserializer.h @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/utilities/deserializer.inl b/src/core/utilities/deserializer.inl index 6a73119496..c6b7117933 100644 --- a/src/core/utilities/deserializer.inl +++ b/src/core/utilities/deserializer.inl @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/utilities/detail/buffer_builder.cc b/src/core/utilities/detail/buffer_builder.cc index b1ec4c8dc4..4b441e4143 100644 --- a/src/core/utilities/detail/buffer_builder.cc +++ b/src/core/utilities/detail/buffer_builder.cc @@ -1,17 +1,13 @@ -/* Copyright 2021-2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/utilities/detail/buffer_builder.h" diff --git a/src/core/utilities/detail/buffer_builder.h b/src/core/utilities/detail/buffer_builder.h index a16b63bfd6..1754c1399b 100644 --- a/src/core/utilities/detail/buffer_builder.h +++ b/src/core/utilities/detail/buffer_builder.h @@ -1,17 +1,13 @@ -/* Copyright 2021-2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/utilities/detail/buffer_builder.inl b/src/core/utilities/detail/buffer_builder.inl index fc779d5591..e243beb99b 100644 --- a/src/core/utilities/detail/buffer_builder.inl +++ b/src/core/utilities/detail/buffer_builder.inl @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/utilities/dispatch.h b/src/core/utilities/dispatch.h index 3b058a94db..178472a6ae 100644 --- a/src/core/utilities/dispatch.h +++ b/src/core/utilities/dispatch.h @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/utilities/linearize.cc b/src/core/utilities/linearize.cc index 02ed39ac2b..4e1965e058 100644 --- a/src/core/utilities/linearize.cc +++ b/src/core/utilities/linearize.cc @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/utilities/linearize.h" diff --git a/src/core/utilities/linearize.h b/src/core/utilities/linearize.h index a8d1720acb..f222f71ba8 100644 --- a/src/core/utilities/linearize.h +++ b/src/core/utilities/linearize.h @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/utilities/machine.cc b/src/core/utilities/machine.cc index ca79637e97..42e803a1d9 100644 --- a/src/core/utilities/machine.cc +++ b/src/core/utilities/machine.cc @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/utilities/machine.h" diff --git a/src/core/utilities/machine.h b/src/core/utilities/machine.h index 5e0cd9d60e..90250a2d72 100644 --- a/src/core/utilities/machine.h +++ b/src/core/utilities/machine.h @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/utilities/multi_set.h b/src/core/utilities/multi_set.h index 4811a3884c..6eaac88bb7 100644 --- a/src/core/utilities/multi_set.h +++ b/src/core/utilities/multi_set.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/utilities/multi_set.inl b/src/core/utilities/multi_set.inl index f14b0e21d0..5e0864be56 100644 --- a/src/core/utilities/multi_set.inl +++ b/src/core/utilities/multi_set.inl @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/utilities/nvtx_help.h b/src/core/utilities/nvtx_help.h index 5d69b7fd3e..e550c0670d 100644 --- a/src/core/utilities/nvtx_help.h +++ b/src/core/utilities/nvtx_help.h @@ -1,17 +1,13 @@ -/* Copyright 2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/utilities/ordered_set.h b/src/core/utilities/ordered_set.h index 525e34583c..06515f8de8 100644 --- a/src/core/utilities/ordered_set.h +++ b/src/core/utilities/ordered_set.h @@ -1,17 +1,13 @@ -/* Copyright 2021 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/utilities/span.h b/src/core/utilities/span.h index 22a012288d..88f47eecdf 100644 --- a/src/core/utilities/span.h +++ b/src/core/utilities/span.h @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/utilities/tuple.h b/src/core/utilities/tuple.h index 2f4d837e22..84ecc36cf9 100644 --- a/src/core/utilities/tuple.h +++ b/src/core/utilities/tuple.h @@ -1,17 +1,13 @@ -/* Copyright 2021 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/utilities/tuple.inl b/src/core/utilities/tuple.inl index 6535f74cdd..0d178a822e 100644 --- a/src/core/utilities/tuple.inl +++ b/src/core/utilities/tuple.inl @@ -1,17 +1,13 @@ -/* Copyright 2021 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/core/utilities/typedefs.h b/src/core/utilities/typedefs.h index 8b249b1be7..9cd53f1f66 100644 --- a/src/core/utilities/typedefs.h +++ b/src/core/utilities/typedefs.h @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/env_defaults.h b/src/env_defaults.h index 9d238ea088..8da014dc94 100644 --- a/src/env_defaults.h +++ b/src/env_defaults.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ // These values are copied manually in legate.settings and there is a Python diff --git a/src/legate.h b/src/legate.h index fdaee963a0..d897e0023b 100644 --- a/src/legate.h +++ b/src/legate.h @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/legate.mk b/src/legate.mk index e226693ed6..9f643fa22b 100644 --- a/src/legate.mk +++ b/src/legate.mk @@ -1,17 +1,12 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. ifndef LIBNAME $(error LIBNAME must be given, aborting build) diff --git a/src/legate_defines.h b/src/legate_defines.h index 7c08bc3b1f..3fd346a09c 100644 --- a/src/legate_defines.h +++ b/src/legate_defines.h @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/src/legate_preamble.h b/src/legate_preamble.h index 4e4269ac35..354d374170 100644 --- a/src/legate_preamble.h +++ b/src/legate_preamble.h @@ -1,17 +1,13 @@ -/* Copyright 2021-2022 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #ifndef __LEGATE_PREAMBLE_H__ diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt index e1139d3d16..36b4df7193 100644 --- a/tests/cpp/CMakeLists.txt +++ b/tests/cpp/CMakeLists.txt @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= cmake_minimum_required(VERSION 3.22.1 FATAL_ERROR) diff --git a/tests/cpp/cmake/thirdparty/get_nccl.cmake b/tests/cpp/cmake/thirdparty/get_nccl.cmake index 1aee52b6f5..5269e68d3c 100644 --- a/tests/cpp/cmake/thirdparty/get_nccl.cmake +++ b/tests/cpp/cmake/thirdparty/get_nccl.cmake @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2022 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= function(find_or_configure_nccl) diff --git a/tests/cpp/integration/alignment_constraints.cc b/tests/cpp/integration/alignment_constraints.cc index e0d2ab8d6c..9007e2a501 100644 --- a/tests/cpp/integration/alignment_constraints.cc +++ b/tests/cpp/integration/alignment_constraints.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/integration/broadcast_constraints.cc b/tests/cpp/integration/broadcast_constraints.cc index ac217cfafd..209a0ffaa2 100644 --- a/tests/cpp/integration/broadcast_constraints.cc +++ b/tests/cpp/integration/broadcast_constraints.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/integration/copy_failure.cc b/tests/cpp/integration/copy_failure.cc index f4d0913cba..0bdc6392a9 100644 --- a/tests/cpp/integration/copy_failure.cc +++ b/tests/cpp/integration/copy_failure.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/integration/copy_gather.cc b/tests/cpp/integration/copy_gather.cc index 83602e0fe7..5e009f83f4 100644 --- a/tests/cpp/integration/copy_gather.cc +++ b/tests/cpp/integration/copy_gather.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/integration/copy_gather_scatter.cc b/tests/cpp/integration/copy_gather_scatter.cc index 47b553184e..227df98439 100644 --- a/tests/cpp/integration/copy_gather_scatter.cc +++ b/tests/cpp/integration/copy_gather_scatter.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/integration/copy_normal.cc b/tests/cpp/integration/copy_normal.cc index e563e379e3..60de59ed1e 100644 --- a/tests/cpp/integration/copy_normal.cc +++ b/tests/cpp/integration/copy_normal.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/integration/copy_scatter.cc b/tests/cpp/integration/copy_scatter.cc index adb6612e31..0843fda228 100644 --- a/tests/cpp/integration/copy_scatter.cc +++ b/tests/cpp/integration/copy_scatter.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/integration/copy_util.inl b/tests/cpp/integration/copy_util.inl index 4786f352bc..db70e24f8a 100644 --- a/tests/cpp/integration/copy_util.inl +++ b/tests/cpp/integration/copy_util.inl @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/data/detail/logical_store.h" diff --git a/tests/cpp/integration/cpu_communicator.cc b/tests/cpp/integration/cpu_communicator.cc index 4f395b6919..5f17b6f36b 100644 --- a/tests/cpp/integration/cpu_communicator.cc +++ b/tests/cpp/integration/cpu_communicator.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/integration/exception.cc b/tests/cpp/integration/exception.cc index 0c35acd689..692d9fd4e6 100644 --- a/tests/cpp/integration/exception.cc +++ b/tests/cpp/integration/exception.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/integration/fill.cc b/tests/cpp/integration/fill.cc index 435d920a06..e409143215 100644 --- a/tests/cpp/integration/fill.cc +++ b/tests/cpp/integration/fill.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/integration/image_constraints.cc b/tests/cpp/integration/image_constraints.cc index 84e4c89743..0c3d0858fd 100644 --- a/tests/cpp/integration/image_constraints.cc +++ b/tests/cpp/integration/image_constraints.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/integration/inline_map.cc b/tests/cpp/integration/inline_map.cc index 7154a28d8f..f22170ff69 100644 --- a/tests/cpp/integration/inline_map.cc +++ b/tests/cpp/integration/inline_map.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/integration/inout.cc b/tests/cpp/integration/inout.cc index 73001785cf..b9a6783230 100644 --- a/tests/cpp/integration/inout.cc +++ b/tests/cpp/integration/inout.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/integration/machine_scope.cc b/tests/cpp/integration/machine_scope.cc index d4488dfa22..6e77c86071 100644 --- a/tests/cpp/integration/machine_scope.cc +++ b/tests/cpp/integration/machine_scope.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/integration/manual_simple.cc b/tests/cpp/integration/manual_simple.cc index eabbab155d..44fdc8f11e 100644 --- a/tests/cpp/integration/manual_simple.cc +++ b/tests/cpp/integration/manual_simple.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/integration/multi_scalar_out.cc b/tests/cpp/integration/multi_scalar_out.cc index 01cb9f6fcc..957dd52184 100644 --- a/tests/cpp/integration/multi_scalar_out.cc +++ b/tests/cpp/integration/multi_scalar_out.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/integration/nccl.cu b/tests/cpp/integration/nccl.cu index e9a3dd7241..9bb4444b98 100644 --- a/tests/cpp/integration/nccl.cu +++ b/tests/cpp/integration/nccl.cu @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/integration/provenance.cc b/tests/cpp/integration/provenance.cc index a9f5815c29..73160cd55a 100644 --- a/tests/cpp/integration/provenance.cc +++ b/tests/cpp/integration/provenance.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/integration/region_manager.cc b/tests/cpp/integration/region_manager.cc index 34c216f429..3f7f75076f 100644 --- a/tests/cpp/integration/region_manager.cc +++ b/tests/cpp/integration/region_manager.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/integration/tasks/task_region_manager.cc b/tests/cpp/integration/tasks/task_region_manager.cc index 4cc1db44de..9c59024d7a 100644 --- a/tests/cpp/integration/tasks/task_region_manager.cc +++ b/tests/cpp/integration/tasks/task_region_manager.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "task_region_manager.h" diff --git a/tests/cpp/integration/tasks/task_region_manager.h b/tests/cpp/integration/tasks/task_region_manager.h index d186d06417..53f77279e6 100644 --- a/tests/cpp/integration/tasks/task_region_manager.h +++ b/tests/cpp/integration/tasks/task_region_manager.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/tests/cpp/integration/tasks/task_simple.cc b/tests/cpp/integration/tasks/task_simple.cc index 41a955670f..cb147acd4f 100644 --- a/tests/cpp/integration/tasks/task_simple.cc +++ b/tests/cpp/integration/tasks/task_simple.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "task_simple.h" diff --git a/tests/cpp/integration/tasks/task_simple.h b/tests/cpp/integration/tasks/task_simple.h index be17ec11a4..7a0cb26b37 100644 --- a/tests/cpp/integration/tasks/task_simple.h +++ b/tests/cpp/integration/tasks/task_simple.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/tests/cpp/integration/weighted.cc b/tests/cpp/integration/weighted.cc index eff377d50c..c6acdaf3a8 100644 --- a/tests/cpp/integration/weighted.cc +++ b/tests/cpp/integration/weighted.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/main.cc b/tests/cpp/main.cc index b5d5945ba8..c3d9e46a2e 100644 --- a/tests/cpp/main.cc +++ b/tests/cpp/main.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/run.py b/tests/cpp/run.py index d3736cdf88..52ce071164 100755 --- a/tests/cpp/run.py +++ b/tests/cpp/run.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import argparse import os diff --git a/tests/cpp/unit/library.cc b/tests/cpp/unit/library.cc index 22094c3626..2c055cc853 100644 --- a/tests/cpp/unit/library.cc +++ b/tests/cpp/unit/library.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/unit/machine.cc b/tests/cpp/unit/machine.cc index 4903f68ffc..7118296e8a 100644 --- a/tests/cpp/unit/machine.cc +++ b/tests/cpp/unit/machine.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/unit/registration.cc b/tests/cpp/unit/registration.cc index 3946b8c293..653f311bce 100644 --- a/tests/cpp/unit/registration.cc +++ b/tests/cpp/unit/registration.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/unit/scalar.cc b/tests/cpp/unit/scalar.cc index 8386c753f6..d74c7e4f11 100644 --- a/tests/cpp/unit/scalar.cc +++ b/tests/cpp/unit/scalar.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/unit/span.cc b/tests/cpp/unit/span.cc index f60d708e58..d4057a8faf 100644 --- a/tests/cpp/unit/span.cc +++ b/tests/cpp/unit/span.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/unit/store.cc b/tests/cpp/unit/store.cc index 095b6e3180..1d89e70645 100644 --- a/tests/cpp/unit/store.cc +++ b/tests/cpp/unit/store.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index 142e5a9e5a..93336842c0 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= # We abuse find package for testing purposes here to diff --git a/tests/integration/collective/CMakeLists.txt b/tests/integration/collective/CMakeLists.txt index de4ed3f0c5..f495cd8bd6 100644 --- a/tests/integration/collective/CMakeLists.txt +++ b/tests/integration/collective/CMakeLists.txt @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= # We abuse find package for testing purposes here to diff --git a/tests/integration/collective/collective/__init__.py b/tests/integration/collective/collective/__init__.py index 6e493e39f7..9bd711ada6 100644 --- a/tests/integration/collective/collective/__init__.py +++ b/tests/integration/collective/collective/__init__.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from .lib import user_lib, user_context # type: ignore from .collective import ( diff --git a/tests/integration/collective/collective/collective.py b/tests/integration/collective/collective/collective.py index 4e831085a7..d4f9688c25 100644 --- a/tests/integration/collective/collective/collective.py +++ b/tests/integration/collective/collective/collective.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from enum import IntEnum from typing import Any, Tuple diff --git a/tests/integration/collective/collective/lib.py b/tests/integration/collective/collective/lib.py index e3d7a69d32..1d6d17b55c 100644 --- a/tests/integration/collective/collective/lib.py +++ b/tests/integration/collective/collective/lib.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import os from typing import Any diff --git a/tests/integration/collective/editable-install.sh b/tests/integration/collective/editable-install.sh index 5ba1a212f9..1cd98e43c3 100755 --- a/tests/integration/collective/editable-install.sh +++ b/tests/integration/collective/editable-install.sh @@ -1,18 +1,14 @@ #!/bin/bash -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. legate_root=`python -c 'import legate.install_info as i; from pathlib import Path; print(Path(i.libpath).parent.resolve())'` echo "Using Legate at $legate_root" diff --git a/tests/integration/collective/examples/test_collective.py b/tests/integration/collective/examples/test_collective.py index c686dd19cb..4e3f7a4b2c 100644 --- a/tests/integration/collective/examples/test_collective.py +++ b/tests/integration/collective/examples/test_collective.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import pytest from collective import ( collective_test, diff --git a/tests/integration/collective/install.sh b/tests/integration/collective/install.sh index 9e9ff542b0..2ad82c5835 100644 --- a/tests/integration/collective/install.sh +++ b/tests/integration/collective/install.sh @@ -1,17 +1,13 @@ #!/bin/bash -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. python -m pip install . diff --git a/tests/integration/collective/setup.py b/tests/integration/collective/setup.py index a5d103de54..9de5d3d3d4 100644 --- a/tests/integration/collective/setup.py +++ b/tests/integration/collective/setup.py @@ -1,19 +1,15 @@ #!/usr/bin/env python3 -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import os from pathlib import Path @@ -39,12 +35,12 @@ version="0.1", description="Collective instances test", author="NVIDIA Corporation", - license="Apache 2.0", + license="Proprietary", classifiers=[ "Intended Audience :: Developers", "Topic :: Database", "Topic :: Scientific/Engineering", - "License :: OSI Approved :: Apache Software License", + "License :: Proprietary :: Nvidia Proprietary", "Programming Language :: Python", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", diff --git a/tests/integration/collective/src/CMakeLists.txt b/tests/integration/collective/src/CMakeLists.txt index c48b94ee94..59007f93e9 100644 --- a/tests/integration/collective/src/CMakeLists.txt +++ b/tests/integration/collective/src/CMakeLists.txt @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= add_library( diff --git a/tests/integration/collective/src/collective_cffi.h b/tests/integration/collective/src/collective_cffi.h index 2c5c52165c..1ae6148569 100644 --- a/tests/integration/collective/src/collective_cffi.h +++ b/tests/integration/collective/src/collective_cffi.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #ifndef __REGISTER_C__ diff --git a/tests/integration/collective/src/collective_test.h b/tests/integration/collective/src/collective_test.h index 7f9a1bc956..00fba7be62 100644 --- a/tests/integration/collective/src/collective_test.h +++ b/tests/integration/collective/src/collective_test.h @@ -1,20 +1,16 @@ - -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ + #pragma once #include "collective_cffi.h" diff --git a/tests/integration/collective/src/library.cc b/tests/integration/collective/src/library.cc index c18f02e839..629118109b 100644 --- a/tests/integration/collective/src/library.cc +++ b/tests/integration/collective/src/library.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "library.h" diff --git a/tests/integration/collective/src/library.h b/tests/integration/collective/src/library.h index d230ce7ff3..d13789a60e 100644 --- a/tests/integration/collective/src/library.h +++ b/tests/integration/collective/src/library.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/tests/integration/region_manager/CMakeLists.txt b/tests/integration/region_manager/CMakeLists.txt index da2a18cc10..817a877f82 100644 --- a/tests/integration/region_manager/CMakeLists.txt +++ b/tests/integration/region_manager/CMakeLists.txt @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= cmake_minimum_required(VERSION 3.22.1 FATAL_ERROR) diff --git a/tests/integration/region_manager/examples/test_region_manager.py b/tests/integration/region_manager/examples/test_region_manager.py index 98de2aa065..d91c19e8bb 100644 --- a/tests/integration/region_manager/examples/test_region_manager.py +++ b/tests/integration/region_manager/examples/test_region_manager.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import pytest from region_manager import user_context, user_lib diff --git a/tests/integration/region_manager/region_manager/__init__.py b/tests/integration/region_manager/region_manager/__init__.py index c391d1e867..7447d1d2d3 100644 --- a/tests/integration/region_manager/region_manager/__init__.py +++ b/tests/integration/region_manager/region_manager/__init__.py @@ -1,15 +1,11 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from .lib import user_lib, user_context diff --git a/tests/integration/region_manager/region_manager/lib.py b/tests/integration/region_manager/region_manager/lib.py index 8ca1d231fe..c63d171427 100644 --- a/tests/integration/region_manager/region_manager/lib.py +++ b/tests/integration/region_manager/region_manager/lib.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import os from typing import Any diff --git a/tests/integration/region_manager/setup.py b/tests/integration/region_manager/setup.py index 5becbcfaff..dc6134680f 100644 --- a/tests/integration/region_manager/setup.py +++ b/tests/integration/region_manager/setup.py @@ -1,19 +1,15 @@ #!/usr/bin/env python3 -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import os from pathlib import Path @@ -39,12 +35,12 @@ version="0.1", description="Region manager test", author="NVIDIA Corporation", - license="Apache 2.0", + license="Proprietary", classifiers=[ "Intended Audience :: Developers", "Topic :: Database", "Topic :: Scientific/Engineering", - "License :: OSI Approved :: Apache Software License", + "License :: Proprietary :: Nvidia Proprietary", "Programming Language :: Python", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", diff --git a/tests/integration/region_manager/src/CMakeLists.txt b/tests/integration/region_manager/src/CMakeLists.txt index d2259cd04d..25fd1df7fd 100644 --- a/tests/integration/region_manager/src/CMakeLists.txt +++ b/tests/integration/region_manager/src/CMakeLists.txt @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= add_library( diff --git a/tests/integration/region_manager/src/library.cc b/tests/integration/region_manager/src/library.cc index 1023fc70c2..f32e775ad1 100644 --- a/tests/integration/region_manager/src/library.cc +++ b/tests/integration/region_manager/src/library.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "library.h" diff --git a/tests/integration/region_manager/src/library.h b/tests/integration/region_manager/src/library.h index 4881078b32..75dde22b30 100644 --- a/tests/integration/region_manager/src/library.h +++ b/tests/integration/region_manager/src/library.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/tests/integration/region_manager/src/region_manager_cffi.h b/tests/integration/region_manager/src/region_manager_cffi.h index 7e1fa422fa..5272b7a670 100644 --- a/tests/integration/region_manager/src/region_manager_cffi.h +++ b/tests/integration/region_manager/src/region_manager_cffi.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #ifndef __REGION_MANAGER_C__ diff --git a/tests/integration/region_manager/src/tester.cc b/tests/integration/region_manager/src/tester.cc index 0dcf3074de..a9d193883a 100644 --- a/tests/integration/region_manager/src/tester.cc +++ b/tests/integration/region_manager/src/tester.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "tester.h" diff --git a/tests/integration/region_manager/src/tester.h b/tests/integration/region_manager/src/tester.h index f6a7e9da2f..cb67f16571 100644 --- a/tests/integration/region_manager/src/tester.h +++ b/tests/integration/region_manager/src/tester.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/tests/integration/registry/CMakeLists.txt b/tests/integration/registry/CMakeLists.txt index 8e43c7241a..9370184477 100644 --- a/tests/integration/registry/CMakeLists.txt +++ b/tests/integration/registry/CMakeLists.txt @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= cmake_minimum_required(VERSION 3.22.1 FATAL_ERROR) diff --git a/tests/integration/registry/examples/test_registry.py b/tests/integration/registry/examples/test_registry.py index 9358df35e6..5081d3d03f 100644 --- a/tests/integration/registry/examples/test_registry.py +++ b/tests/integration/registry/examples/test_registry.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import pytest from registry import user_context, user_lib diff --git a/tests/integration/registry/registry/__init__.py b/tests/integration/registry/registry/__init__.py index c391d1e867..7447d1d2d3 100644 --- a/tests/integration/registry/registry/__init__.py +++ b/tests/integration/registry/registry/__init__.py @@ -1,15 +1,11 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from .lib import user_lib, user_context diff --git a/tests/integration/registry/registry/lib.py b/tests/integration/registry/registry/lib.py index 7704ebf6de..323ac8202f 100644 --- a/tests/integration/registry/registry/lib.py +++ b/tests/integration/registry/registry/lib.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import os from typing import Any diff --git a/tests/integration/registry/setup.py b/tests/integration/registry/setup.py index 9a2b5f7c71..4a398cfd31 100644 --- a/tests/integration/registry/setup.py +++ b/tests/integration/registry/setup.py @@ -1,19 +1,15 @@ #!/usr/bin/env python3 -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import os from pathlib import Path @@ -39,12 +35,12 @@ version="0.1", description="Task registration test", author="NVIDIA Corporation", - license="Apache 2.0", + license="Proprietary", classifiers=[ "Intended Audience :: Developers", "Topic :: Database", "Topic :: Scientific/Engineering", - "License :: OSI Approved :: Apache Software License", + "License :: Proprietary :: Nvidia Proprietary", "Programming Language :: Python", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", diff --git a/tests/integration/registry/src/CMakeLists.txt b/tests/integration/registry/src/CMakeLists.txt index 256ba12a3c..c9078b7971 100644 --- a/tests/integration/registry/src/CMakeLists.txt +++ b/tests/integration/registry/src/CMakeLists.txt @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= add_library( diff --git a/tests/integration/registry/src/hello.cc b/tests/integration/registry/src/hello.cc index 2f1224ae56..e82507ef8f 100644 --- a/tests/integration/registry/src/hello.cc +++ b/tests/integration/registry/src/hello.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "library.h" diff --git a/tests/integration/registry/src/library.cc b/tests/integration/registry/src/library.cc index 2199d55e16..43a55fab43 100644 --- a/tests/integration/registry/src/library.cc +++ b/tests/integration/registry/src/library.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "library.h" diff --git a/tests/integration/registry/src/library.h b/tests/integration/registry/src/library.h index 1d54d8487e..9788a9d5c4 100644 --- a/tests/integration/registry/src/library.h +++ b/tests/integration/registry/src/library.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/tests/integration/registry/src/no_variant.cc b/tests/integration/registry/src/no_variant.cc index 886f4edf09..a9d132cc2a 100644 --- a/tests/integration/registry/src/no_variant.cc +++ b/tests/integration/registry/src/no_variant.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "library.h" diff --git a/tests/integration/registry/src/registry_cffi.h b/tests/integration/registry/src/registry_cffi.h index 5516c6d88d..08dbf4a72a 100644 --- a/tests/integration/registry/src/registry_cffi.h +++ b/tests/integration/registry/src/registry_cffi.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #ifndef __REGISTER_C__ diff --git a/tests/integration/registry/src/world.cc b/tests/integration/registry/src/world.cc index 0b56158f0a..9ab6e8fb45 100644 --- a/tests/integration/registry/src/world.cc +++ b/tests/integration/registry/src/world.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "world.h" diff --git a/tests/integration/registry/src/world.h b/tests/integration/registry/src/world.h index cfb8453c45..f616a40793 100644 --- a/tests/integration/registry/src/world.h +++ b/tests/integration/registry/src/world.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/tests/integration/scoping/CMakeLists.txt b/tests/integration/scoping/CMakeLists.txt index 09a20e66b1..eec6224a13 100644 --- a/tests/integration/scoping/CMakeLists.txt +++ b/tests/integration/scoping/CMakeLists.txt @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= cmake_minimum_required(VERSION 3.22.1 FATAL_ERROR) diff --git a/tests/integration/scoping/examples/test_scoping.py b/tests/integration/scoping/examples/test_scoping.py index 466eb7dcaf..a6c501b737 100644 --- a/tests/integration/scoping/examples/test_scoping.py +++ b/tests/integration/scoping/examples/test_scoping.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import pytest from scoping import user_context, user_lib diff --git a/tests/integration/scoping/scoping/__init__.py b/tests/integration/scoping/scoping/__init__.py index c391d1e867..7447d1d2d3 100644 --- a/tests/integration/scoping/scoping/__init__.py +++ b/tests/integration/scoping/scoping/__init__.py @@ -1,15 +1,11 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from .lib import user_lib, user_context diff --git a/tests/integration/scoping/scoping/lib.py b/tests/integration/scoping/scoping/lib.py index 040c2815c2..837b243550 100644 --- a/tests/integration/scoping/scoping/lib.py +++ b/tests/integration/scoping/scoping/lib.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import os from typing import Any diff --git a/tests/integration/scoping/setup.py b/tests/integration/scoping/setup.py index 1a79df2021..c7c9971e67 100644 --- a/tests/integration/scoping/setup.py +++ b/tests/integration/scoping/setup.py @@ -1,19 +1,15 @@ #!/usr/bin/env python3 -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import os from pathlib import Path @@ -39,12 +35,12 @@ version="0.1", description="Resource scoping test", author="NVIDIA Corporation", - license="Apache 2.0", + license="Proprietary", classifiers=[ "Intended Audience :: Developers", "Topic :: Database", "Topic :: Scientific/Engineering", - "License :: OSI Approved :: Apache Software License", + "License :: Proprietary :: Nvidia Proprietary", "Programming Language :: Python", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", diff --git a/tests/integration/scoping/src/CMakeLists.txt b/tests/integration/scoping/src/CMakeLists.txt index fdd4bd68bd..9bc0bc1e0e 100644 --- a/tests/integration/scoping/src/CMakeLists.txt +++ b/tests/integration/scoping/src/CMakeLists.txt @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= add_library( diff --git a/tests/integration/scoping/src/library.cc b/tests/integration/scoping/src/library.cc index db4f775963..602a0fb0e2 100644 --- a/tests/integration/scoping/src/library.cc +++ b/tests/integration/scoping/src/library.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "core/mapping/mapping.h" diff --git a/tests/integration/scoping/src/scoping_cffi.h b/tests/integration/scoping/src/scoping_cffi.h index ab9d7dfc3f..0e739e179e 100644 --- a/tests/integration/scoping/src/scoping_cffi.h +++ b/tests/integration/scoping/src/scoping_cffi.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #ifndef __REGISTER_C__ diff --git a/tests/integration/tree_reduce/CMakeLists.txt b/tests/integration/tree_reduce/CMakeLists.txt index 8245f9b2dc..e9a400279a 100644 --- a/tests/integration/tree_reduce/CMakeLists.txt +++ b/tests/integration/tree_reduce/CMakeLists.txt @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= cmake_minimum_required(VERSION 3.22.1 FATAL_ERROR) diff --git a/tests/integration/tree_reduce/examples/test_tree_reduce.py b/tests/integration/tree_reduce/examples/test_tree_reduce.py index eee8b7cf5c..1cb38ffd79 100644 --- a/tests/integration/tree_reduce/examples/test_tree_reduce.py +++ b/tests/integration/tree_reduce/examples/test_tree_reduce.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import pytest from tree_reduce import user_context, user_lib diff --git a/tests/integration/tree_reduce/setup.py b/tests/integration/tree_reduce/setup.py index 3261acafed..ba63c2a5b2 100644 --- a/tests/integration/tree_reduce/setup.py +++ b/tests/integration/tree_reduce/setup.py @@ -1,19 +1,15 @@ #!/usr/bin/env python3 -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import os from pathlib import Path @@ -39,12 +35,12 @@ version="0.1", description="Tree reduction test", author="NVIDIA Corporation", - license="Apache 2.0", + license="Proprietary", classifiers=[ "Intended Audience :: Developers", "Topic :: Database", "Topic :: Scientific/Engineering", - "License :: OSI Approved :: Apache Software License", + "License :: Proprietary :: Nvidia Proprietary", "Programming Language :: Python", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", diff --git a/tests/integration/tree_reduce/src/CMakeLists.txt b/tests/integration/tree_reduce/src/CMakeLists.txt index 1620c68731..bf57c65eeb 100644 --- a/tests/integration/tree_reduce/src/CMakeLists.txt +++ b/tests/integration/tree_reduce/src/CMakeLists.txt @@ -1,17 +1,13 @@ #============================================================================= -# Copyright 2023 NVIDIA Corporation +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. #============================================================================= add_library( diff --git a/tests/integration/tree_reduce/src/library.cc b/tests/integration/tree_reduce/src/library.cc index 500382b775..f285ea9c8a 100644 --- a/tests/integration/tree_reduce/src/library.cc +++ b/tests/integration/tree_reduce/src/library.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "library.h" diff --git a/tests/integration/tree_reduce/src/library.h b/tests/integration/tree_reduce/src/library.h index cf1fb48cb0..d500837022 100644 --- a/tests/integration/tree_reduce/src/library.h +++ b/tests/integration/tree_reduce/src/library.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/tests/integration/tree_reduce/src/produce_normal.cc b/tests/integration/tree_reduce/src/produce_normal.cc index 4143873bd7..c6c5c8a6bf 100644 --- a/tests/integration/tree_reduce/src/produce_normal.cc +++ b/tests/integration/tree_reduce/src/produce_normal.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "produce_normal.h" diff --git a/tests/integration/tree_reduce/src/produce_normal.h b/tests/integration/tree_reduce/src/produce_normal.h index 151e9e1eac..6f4822dbbb 100644 --- a/tests/integration/tree_reduce/src/produce_normal.h +++ b/tests/integration/tree_reduce/src/produce_normal.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/tests/integration/tree_reduce/src/produce_unbound.cc b/tests/integration/tree_reduce/src/produce_unbound.cc index 52f5e474f0..ffc6343e2b 100644 --- a/tests/integration/tree_reduce/src/produce_unbound.cc +++ b/tests/integration/tree_reduce/src/produce_unbound.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "produce_unbound.h" diff --git a/tests/integration/tree_reduce/src/produce_unbound.h b/tests/integration/tree_reduce/src/produce_unbound.h index 68c6b5ddd6..6217259128 100644 --- a/tests/integration/tree_reduce/src/produce_unbound.h +++ b/tests/integration/tree_reduce/src/produce_unbound.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/tests/integration/tree_reduce/src/reduce_normal.cc b/tests/integration/tree_reduce/src/reduce_normal.cc index 0c8081e068..1a3aa50d78 100644 --- a/tests/integration/tree_reduce/src/reduce_normal.cc +++ b/tests/integration/tree_reduce/src/reduce_normal.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "reduce_normal.h" diff --git a/tests/integration/tree_reduce/src/reduce_normal.h b/tests/integration/tree_reduce/src/reduce_normal.h index d3107cc0cf..2b02dd1cdd 100644 --- a/tests/integration/tree_reduce/src/reduce_normal.h +++ b/tests/integration/tree_reduce/src/reduce_normal.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/tests/integration/tree_reduce/src/reduce_unbound.cc b/tests/integration/tree_reduce/src/reduce_unbound.cc index bf7ca72449..c2b16b31fd 100644 --- a/tests/integration/tree_reduce/src/reduce_unbound.cc +++ b/tests/integration/tree_reduce/src/reduce_unbound.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include "reduce_unbound.h" diff --git a/tests/integration/tree_reduce/src/reduce_unbound.h b/tests/integration/tree_reduce/src/reduce_unbound.h index a69e996947..82454b1b6c 100644 --- a/tests/integration/tree_reduce/src/reduce_unbound.h +++ b/tests/integration/tree_reduce/src/reduce_unbound.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #pragma once diff --git a/tests/integration/tree_reduce/src/tree_reduce_cffi.h b/tests/integration/tree_reduce/src/tree_reduce_cffi.h index 92a13dbe98..9ef5006e67 100644 --- a/tests/integration/tree_reduce/src/tree_reduce_cffi.h +++ b/tests/integration/tree_reduce/src/tree_reduce_cffi.h @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #ifndef __REGISTER_C__ diff --git a/tests/integration/tree_reduce/tree_reduce/__init__.py b/tests/integration/tree_reduce/tree_reduce/__init__.py index c391d1e867..7447d1d2d3 100644 --- a/tests/integration/tree_reduce/tree_reduce/__init__.py +++ b/tests/integration/tree_reduce/tree_reduce/__init__.py @@ -1,15 +1,11 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from .lib import user_lib, user_context diff --git a/tests/integration/tree_reduce/tree_reduce/lib.py b/tests/integration/tree_reduce/tree_reduce/lib.py index 26f1a76b7f..9b18fad026 100644 --- a/tests/integration/tree_reduce/tree_reduce/lib.py +++ b/tests/integration/tree_reduce/tree_reduce/lib.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import os from typing import Any diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index f0b271624d..19271d856c 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -1,15 +1,11 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations diff --git a/tests/unit/legate/__init__.py b/tests/unit/legate/__init__.py index f0b271624d..19271d856c 100644 --- a/tests/unit/legate/__init__.py +++ b/tests/unit/legate/__init__.py @@ -1,15 +1,11 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations diff --git a/tests/unit/legate/core/test_invalid_scalar_arg.py b/tests/unit/legate/core/test_invalid_scalar_arg.py index dc1a6b79e4..5aa26ad356 100644 --- a/tests/unit/legate/core/test_invalid_scalar_arg.py +++ b/tests/unit/legate/core/test_invalid_scalar_arg.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import pytest diff --git a/tests/unit/legate/core/test_machine.py b/tests/unit/legate/core/test_machine.py index bf216626fc..b38ecb26aa 100644 --- a/tests/unit/legate/core/test_machine.py +++ b/tests/unit/legate/core/test_machine.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import struct diff --git a/tests/unit/legate/core/test_store.py b/tests/unit/legate/core/test_store.py index fec6f5664c..d34fa68c1b 100644 --- a/tests/unit/legate/core/test_store.py +++ b/tests/unit/legate/core/test_store.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import pytest diff --git a/tests/unit/legate/core/test_type_uid.py b/tests/unit/legate/core/test_type_uid.py index 18b57cc50c..0fa86240be 100644 --- a/tests/unit/legate/core/test_type_uid.py +++ b/tests/unit/legate/core/test_type_uid.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import pytest diff --git a/tests/unit/legate/driver/__init__.py b/tests/unit/legate/driver/__init__.py index f0b271624d..19271d856c 100644 --- a/tests/unit/legate/driver/__init__.py +++ b/tests/unit/legate/driver/__init__.py @@ -1,15 +1,11 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations diff --git a/tests/unit/legate/driver/conftest.py b/tests/unit/legate/driver/conftest.py index 09c8c7d180..a31e865925 100644 --- a/tests/unit/legate/driver/conftest.py +++ b/tests/unit/legate/driver/conftest.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from importlib import reload diff --git a/tests/unit/legate/driver/test_args.py b/tests/unit/legate/driver/test_args.py index e78a01c300..ca73d89de2 100644 --- a/tests/unit/legate/driver/test_args.py +++ b/tests/unit/legate/driver/test_args.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from argparse import SUPPRESS diff --git a/tests/unit/legate/driver/test_command.py b/tests/unit/legate/driver/test_command.py index a6b62844ed..9c245b8388 100644 --- a/tests/unit/legate/driver/test_command.py +++ b/tests/unit/legate/driver/test_command.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import argparse diff --git a/tests/unit/legate/driver/test_config.py b/tests/unit/legate/driver/test_config.py index 73c99db665..1741672190 100644 --- a/tests/unit/legate/driver/test_config.py +++ b/tests/unit/legate/driver/test_config.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import os diff --git a/tests/unit/legate/driver/test_defaults.py b/tests/unit/legate/driver/test_defaults.py index b67f4202cc..a5420b16cc 100644 --- a/tests/unit/legate/driver/test_defaults.py +++ b/tests/unit/legate/driver/test_defaults.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from importlib import reload diff --git a/tests/unit/legate/driver/test_driver.py b/tests/unit/legate/driver/test_driver.py index 47a3cfcfc8..e54432a7a3 100644 --- a/tests/unit/legate/driver/test_driver.py +++ b/tests/unit/legate/driver/test_driver.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import re diff --git a/tests/unit/legate/driver/test_launcher.py b/tests/unit/legate/driver/test_launcher.py index 23ca3f1362..22748263f8 100644 --- a/tests/unit/legate/driver/test_launcher.py +++ b/tests/unit/legate/driver/test_launcher.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import os diff --git a/tests/unit/legate/driver/test_logs.py b/tests/unit/legate/driver/test_logs.py index 0cc6bb79da..9670b6cc25 100644 --- a/tests/unit/legate/driver/test_logs.py +++ b/tests/unit/legate/driver/test_logs.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import pytest diff --git a/tests/unit/legate/driver/test_main.py b/tests/unit/legate/driver/test_main.py index 77d29953c5..67ff8d1328 100644 --- a/tests/unit/legate/driver/test_main.py +++ b/tests/unit/legate/driver/test_main.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import sys diff --git a/tests/unit/legate/driver/util.py b/tests/unit/legate/driver/util.py index fad7a9f76e..ed38018299 100644 --- a/tests/unit/legate/driver/util.py +++ b/tests/unit/legate/driver/util.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from typing import Any diff --git a/tests/unit/legate/jupyter/__init__.py b/tests/unit/legate/jupyter/__init__.py index f0b271624d..19271d856c 100644 --- a/tests/unit/legate/jupyter/__init__.py +++ b/tests/unit/legate/jupyter/__init__.py @@ -1,15 +1,11 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations diff --git a/tests/unit/legate/jupyter/test_args.py b/tests/unit/legate/jupyter/test_args.py index c0904927a5..fe624a750e 100644 --- a/tests/unit/legate/jupyter/test_args.py +++ b/tests/unit/legate/jupyter/test_args.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import legate.driver.args as m diff --git a/tests/unit/legate/jupyter/test_config.py b/tests/unit/legate/jupyter/test_config.py index dbf01d3d33..c1c1b633c5 100644 --- a/tests/unit/legate/jupyter/test_config.py +++ b/tests/unit/legate/jupyter/test_config.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from pathlib import Path diff --git a/tests/unit/legate/jupyter/test_kernel.py b/tests/unit/legate/jupyter/test_kernel.py index 105451b93e..828becfd6f 100644 --- a/tests/unit/legate/jupyter/test_kernel.py +++ b/tests/unit/legate/jupyter/test_kernel.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import json diff --git a/tests/unit/legate/jupyter/test_main.py b/tests/unit/legate/jupyter/test_main.py index c25c2c5b30..f74dace850 100644 --- a/tests/unit/legate/jupyter/test_main.py +++ b/tests/unit/legate/jupyter/test_main.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import sys diff --git a/tests/unit/legate/test_cycle_check.py b/tests/unit/legate/test_cycle_check.py index 1494090d19..454b3fe6c3 100644 --- a/tests/unit/legate/test_cycle_check.py +++ b/tests/unit/legate/test_cycle_check.py @@ -1,17 +1,13 @@ -# Copyright 2022-2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import os import subprocess diff --git a/tests/unit/legate/test_settings.py b/tests/unit/legate/test_settings.py index 0e3e5b850a..88d9be4bff 100644 --- a/tests/unit/legate/test_settings.py +++ b/tests/unit/legate/test_settings.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from pathlib import Path diff --git a/tests/unit/legate/tester/__init__.py b/tests/unit/legate/tester/__init__.py index f0b271624d..19271d856c 100644 --- a/tests/unit/legate/tester/__init__.py +++ b/tests/unit/legate/tester/__init__.py @@ -1,15 +1,11 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations diff --git a/tests/unit/legate/tester/stages/__init__.py b/tests/unit/legate/tester/stages/__init__.py index a955e39e08..9ed8bcbb84 100644 --- a/tests/unit/legate/tester/stages/__init__.py +++ b/tests/unit/legate/tester/stages/__init__.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from typing import Any diff --git a/tests/unit/legate/tester/stages/_linux/__init__.py b/tests/unit/legate/tester/stages/_linux/__init__.py index 3459839199..33c5cf5980 100644 --- a/tests/unit/legate/tester/stages/_linux/__init__.py +++ b/tests/unit/legate/tester/stages/_linux/__init__.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import sys diff --git a/tests/unit/legate/tester/stages/_linux/test_cpu.py b/tests/unit/legate/tester/stages/_linux/test_cpu.py index 57aa66c47e..66f4d2ba21 100644 --- a/tests/unit/legate/tester/stages/_linux/test_cpu.py +++ b/tests/unit/legate/tester/stages/_linux/test_cpu.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """Consolidate test configuration from command-line and environment. """ diff --git a/tests/unit/legate/tester/stages/_linux/test_eager.py b/tests/unit/legate/tester/stages/_linux/test_eager.py index 8a9e6077a4..3cf0bce844 100644 --- a/tests/unit/legate/tester/stages/_linux/test_eager.py +++ b/tests/unit/legate/tester/stages/_linux/test_eager.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """Consolidate test configuration from command-line and environment. """ diff --git a/tests/unit/legate/tester/stages/_linux/test_gpu.py b/tests/unit/legate/tester/stages/_linux/test_gpu.py index 69996b0acf..76877bb506 100644 --- a/tests/unit/legate/tester/stages/_linux/test_gpu.py +++ b/tests/unit/legate/tester/stages/_linux/test_gpu.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """Consolidate test configuration from command-line and environment. """ diff --git a/tests/unit/legate/tester/stages/_linux/test_omp.py b/tests/unit/legate/tester/stages/_linux/test_omp.py index 1cb2ec1956..8b39a31672 100644 --- a/tests/unit/legate/tester/stages/_linux/test_omp.py +++ b/tests/unit/legate/tester/stages/_linux/test_omp.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """Consolidate test configuration from command-line and environment. """ diff --git a/tests/unit/legate/tester/stages/test_test_stage.py b/tests/unit/legate/tester/stages/test_test_stage.py index 51b8d32fc8..866f4a1e06 100644 --- a/tests/unit/legate/tester/stages/test_test_stage.py +++ b/tests/unit/legate/tester/stages/test_test_stage.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """Consolidate test configuration from command-line and environment. """ diff --git a/tests/unit/legate/tester/stages/test_util.py b/tests/unit/legate/tester/stages/test_util.py index 40eb0e47b7..709746e1fb 100644 --- a/tests/unit/legate/tester/stages/test_util.py +++ b/tests/unit/legate/tester/stages/test_util.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """Consolidate test configuration from command-line and environment. """ diff --git a/tests/unit/legate/tester/test___init__.py b/tests/unit/legate/tester/test___init__.py index ec22fa24ba..2e12a6e2ba 100644 --- a/tests/unit/legate/tester/test___init__.py +++ b/tests/unit/legate/tester/test___init__.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """Consolidate test configuration from command-line and environment. """ diff --git a/tests/unit/legate/tester/test_args.py b/tests/unit/legate/tester/test_args.py index 2dca7a7316..67735e5310 100644 --- a/tests/unit/legate/tester/test_args.py +++ b/tests/unit/legate/tester/test_args.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """Consolidate test configuration from command-line and environment. """ diff --git a/tests/unit/legate/tester/test_config.py b/tests/unit/legate/tester/test_config.py index 8d7d1bab98..ea32c4d33d 100644 --- a/tests/unit/legate/tester/test_config.py +++ b/tests/unit/legate/tester/test_config.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """Consolidate test configuration from command-line and environment. """ diff --git a/tests/unit/legate/tester/test_logger.py b/tests/unit/legate/tester/test_logger.py index 40228c2f43..3a4cb20bb4 100644 --- a/tests/unit/legate/tester/test_logger.py +++ b/tests/unit/legate/tester/test_logger.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """Consolidate test configuration from command-line and environment. """ diff --git a/tests/unit/legate/tester/test_test_system.py b/tests/unit/legate/tester/test_test_system.py index 55b6b7166f..c72aebcc07 100644 --- a/tests/unit/legate/tester/test_test_system.py +++ b/tests/unit/legate/tester/test_test_system.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """Consolidate test configuration from command-line and environment. """ diff --git a/tests/unit/legate/util/__init__.py b/tests/unit/legate/util/__init__.py index f0b271624d..19271d856c 100644 --- a/tests/unit/legate/util/__init__.py +++ b/tests/unit/legate/util/__init__.py @@ -1,15 +1,11 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations diff --git a/tests/unit/legate/util/test_args.py b/tests/unit/legate/util/test_args.py index 0bf417df10..1d52dbbf7d 100644 --- a/tests/unit/legate/util/test_args.py +++ b/tests/unit/legate/util/test_args.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + import sys from argparse import ArgumentParser diff --git a/tests/unit/legate/util/test_colors.py b/tests/unit/legate/util/test_colors.py index 60dce0ec37..f230e2532c 100644 --- a/tests/unit/legate/util/test_colors.py +++ b/tests/unit/legate/util/test_colors.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from typing import Any diff --git a/tests/unit/legate/util/test_fs.py b/tests/unit/legate/util/test_fs.py index 32cd452b37..5408758050 100644 --- a/tests/unit/legate/util/test_fs.py +++ b/tests/unit/legate/util/test_fs.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from pathlib import Path diff --git a/tests/unit/legate/util/test_settings.py b/tests/unit/legate/util/test_settings.py index ab6fa311f3..1e3983d688 100644 --- a/tests/unit/legate/util/test_settings.py +++ b/tests/unit/legate/util/test_settings.py @@ -1,17 +1,13 @@ -# Copyright 2023 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import os diff --git a/tests/unit/legate/util/test_system.py b/tests/unit/legate/util/test_system.py index 44e99a9c0a..815adb6dcb 100644 --- a/tests/unit/legate/util/test_system.py +++ b/tests/unit/legate/util/test_system.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations import os diff --git a/tests/unit/legate/util/test_types.py b/tests/unit/legate/util/test_types.py index 070bc458f7..9be6f5771c 100644 --- a/tests/unit/legate/util/test_types.py +++ b/tests/unit/legate/util/test_types.py @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + """Consolidate test configuration from command-line and environment. """ diff --git a/tests/unit/legate/util/test_ui.py b/tests/unit/legate/util/test_ui.py index 1ae30703ed..a193a39552 100644 --- a/tests/unit/legate/util/test_ui.py +++ b/tests/unit/legate/util/test_ui.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from datetime import timedelta diff --git a/tests/unit/util.py b/tests/unit/util.py index b6ce793c03..768eb712ea 100644 --- a/tests/unit/util.py +++ b/tests/unit/util.py @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from itertools import chain, combinations diff --git a/typings/IPython/__init__.pyi b/typings/IPython/__init__.pyi index 13b35e47e2..8d45cad035 100644 --- a/typings/IPython/__init__.pyi +++ b/typings/IPython/__init__.pyi @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from .core.magic import Magics diff --git a/typings/IPython/core/magic.pyi b/typings/IPython/core/magic.pyi index 354c7ce2cb..a5135f3ff7 100644 --- a/typings/IPython/core/magic.pyi +++ b/typings/IPython/core/magic.pyi @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from typing import Any, Callable, TypeVar diff --git a/typings/jupyter_client/kernelspec.pyi b/typings/jupyter_client/kernelspec.pyi index d69b0e3b75..7c09a7b8dc 100644 --- a/typings/jupyter_client/kernelspec.pyi +++ b/typings/jupyter_client/kernelspec.pyi @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from typing import Any diff --git a/typings/legion_cffi/__init__.pyi b/typings/legion_cffi/__init__.pyi index 50d84df5b2..14c84f4265 100644 --- a/typings/legion_cffi/__init__.pyi +++ b/typings/legion_cffi/__init__.pyi @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from __future__ import annotations from typing import Any, Callable diff --git a/typings/legion_cffi/lib.pyi b/typings/legion_cffi/lib.pyi index e5399760cb..e9d1739eda 100644 --- a/typings/legion_cffi/lib.pyi +++ b/typings/legion_cffi/lib.pyi @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from typing import Any diff --git a/typings/legion_top/__init__.pyi b/typings/legion_top/__init__.pyi index 2ee59f46f4..0d1e3a9712 100644 --- a/typings/legion_top/__init__.pyi +++ b/typings/legion_top/__init__.pyi @@ -1,17 +1,13 @@ -# Copyright 2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from typing import Any, Callable, Tuple diff --git a/typings/pyarrow/__init__.pyi b/typings/pyarrow/__init__.pyi index cc2ac93aa9..d9626385ab 100644 --- a/typings/pyarrow/__init__.pyi +++ b/typings/pyarrow/__init__.pyi @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from typing import Any, Union diff --git a/typings/pyarrow/lib.pyi b/typings/pyarrow/lib.pyi index 398361089b..4bd782ed19 100644 --- a/typings/pyarrow/lib.pyi +++ b/typings/pyarrow/lib.pyi @@ -1,17 +1,13 @@ -# Copyright 2021-2022 NVIDIA Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary # +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + from typing import Any From 3fbc3778282a139e83aa52d61bf4a2269005594f Mon Sep 17 00:00:00 2001 From: Parag Kulkarni Date: Tue, 25 Jul 2023 23:52:05 -0700 Subject: [PATCH 0172/1425] Add unit test to Legate core CI * - Add Test stage and unit test to Legate core. See merge request legate/legate.core.internal!91 --- .gitlab-ci.yml | 52 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 78d114f41b..2c87693cb3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,6 +8,7 @@ workflow: stages: - build + - test Build:LegateCore: stage: build @@ -22,15 +23,26 @@ Build:LegateCore: CMAKE_BUILD_PARALLEL_LEVEL: 1 PYTHONDONTWRITEBYTECODE: 1 DEFAULT_CONDA_ENV: legate + GIT_DEPTH: 0 + GIT_STRATEGY: clone + USE_CUDA: 'OFF' # Add these if we want to speed up builds with sccache - # SCCACHE_REGION: us-east-2 - # SCCACHE_BUCKET: rapids-sccache-devs - # SCCACHE_S3_KEY_PREFIX: legate-cunumeric-dev - # VAULT_HOST: ${{ secrets.PERSONAL_ACCESS_TOKEN && 'https://vault.ops.k8s.rapids.ai' || '' }} - # GH_TOKEN: "${{ secrets.PERSONAL_ACCESS_TOKEN }}" - # GITHUB_TOKEN: "${{ secrets.PERSONAL_ACCESS_TOKEN }}" + SCCACHE_REGION: us-east-2 + SCCACHE_BUCKET: rapids-sccache-devs + SCCACHE_S3_KEY_PREFIX: legate-core-int-dev + GH_TOKEN: '' + GITHUB_TOKEN: '' + VAULT_HOST: '' + VAULT_S3_TTL: "28800s" # 8 hours before_script: + # Set variables for AWS authentication + - | + if [ $PERSONAL_ACCESS_TOKEN ]; then + VAULT_HOST='https://vault.ops.k8s.rapids.ai'; + GH_TOKEN=$PERSONAL_ACCESS_TOKEN; + GITHUB_TOKEN=$PERSONAL_ACCESS_TOKEN; + fi - cp -R $CI_PROJECT_DIR ~coder/legate - mkdir -p ~coder/.local - cp -R $CI_PROJECT_DIR/continuous_integration/home/coder/.local/bin ~coder/.local/. @@ -59,6 +71,7 @@ Build:LegateCore: # Copy the artifacts - mkdir -p $CI_PROJECT_DIR/artifacts - cp /tmp/out/* $CI_PROJECT_DIR/artifacts/ + - cp -ar /tmp/conda-build $CI_PROJECT_DIR/artifacts/. artifacts: name: "$CI_COMMIT_REF_NAME" @@ -66,3 +79,30 @@ Build:LegateCore: - artifacts exclude: - artifacts/env*.yaml + +Test:LegateCoreUnitTest: + stage: test + + tags: + - gpu/enabled + + variables: + DEFAULT_CONDA_ENV: legate + PYTHONDONTWRITEBYTECODE: 1 + + before_script: + - cp -R $CI_PROJECT_DIR /home/coder/legate + - cp -ar $CI_PROJECT_DIR/continuous_integration/home/coder/.gitconfig /home/coder/ + - cp -ar $CI_PROJECT_DIR/continuous_integration/home/coder/.local /home/coder/ + - chmod a+x /home/coder/.local/bin/* + - chown -R coder:coder /home/coder/ + # Create the legate conda environment + - su coder -c 'cd ~/; exec entrypoint mamba create -n "${DEFAULT_CONDA_ENV:-legate}"' + # Install legate-core and cunumeric in the legate conda environment + - su coder -c 'cd ~/; exec entrypoint mamba install -y -n "${DEFAULT_CONDA_ENV:-legate}" -c nvidia -c conda-forge -c $CI_PROJECT_DIR/artifacts/conda-build/legate_core legate-core' + - su coder -c 'cd ~/; exec entrypoint mamba install -y -n "${DEFAULT_CONDA_ENV:-legate}" -c conda-forge pytest pytest-mock ipython jupyter_client' + + script: + # Run legate.core unit tests + - su coder -c 'cd ~/legate/tests/unit; pytest' + From e6bf0d85fc40d9c94704404358ba0b8c761065d8 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 26 Jul 2023 00:45:09 -0700 Subject: [PATCH 0173/1425] Two fixes for copies * Use normal instances for reduction copy targets * Read-write privilege doesn't work for reduction copy targets See merge request legate/legate.core.internal!95 --- src/core/mapping/detail/base_mapper.cc | 2 ++ src/core/operation/detail/copy_launcher.cc | 8 ++------ src/core/operation/detail/copy_launcher.h | 4 +--- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/core/mapping/detail/base_mapper.cc b/src/core/mapping/detail/base_mapper.cc index 80d2f8efc4..7bea9d3a80 100644 --- a/src/core/mapping/detail/base_mapper.cc +++ b/src/core/mapping/detail/base_mapper.cc @@ -563,6 +563,8 @@ bool BaseMapper::map_legate_store(const Legion::Mapping::MapperContext ctx, } } #endif + // Targets of reduction copies should be mapped to normal instances + if (mappable.get_mappable_type() == Legion::Mappable::COPY_MAPPABLE) redop = 0; // Generate layout constraints from the store mapping Legion::LayoutConstraintSet layout_constraints; diff --git a/src/core/operation/detail/copy_launcher.cc b/src/core/operation/detail/copy_launcher.cc index b4d53697a1..a079d38923 100644 --- a/src/core/operation/detail/copy_launcher.cc +++ b/src/core/operation/detail/copy_launcher.cc @@ -122,13 +122,9 @@ void CopyLauncher::add_inout(detail::LogicalStore* store, std::unique_ptr proj_info, - bool read_write) + std::unique_ptr proj_info) { - if (read_write) - add_store(outputs_, store, std::move(proj_info), LEGION_READ_WRITE); - else - add_store(outputs_, store, std::move(proj_info), LEGION_REDUCE); + add_store(outputs_, store, std::move(proj_info), LEGION_REDUCE); } void CopyLauncher::add_source_indirect(detail::LogicalStore* store, std::unique_ptr proj_info) diff --git a/src/core/operation/detail/copy_launcher.h b/src/core/operation/detail/copy_launcher.h index 610b3907e3..26d1989825 100644 --- a/src/core/operation/detail/copy_launcher.h +++ b/src/core/operation/detail/copy_launcher.h @@ -39,9 +39,7 @@ class CopyLauncher { void add_input(detail::LogicalStore* store, std::unique_ptr proj_info); void add_output(detail::LogicalStore* store, std::unique_ptr proj_info); void add_inout(detail::LogicalStore* store, std::unique_ptr proj_info); - void add_reduction(detail::LogicalStore* store, - std::unique_ptr proj_info, - bool read_write); + void add_reduction(detail::LogicalStore* store, std::unique_ptr proj_info); void add_source_indirect(detail::LogicalStore* store, std::unique_ptr proj_info); void add_target_indirect(detail::LogicalStore* store, std::unique_ptr proj_info); From 44dd82c8b6b634e28323e13140cf70c20ae74358 Mon Sep 17 00:00:00 2001 From: Joy Shen Date: Wed, 26 Jul 2023 21:26:41 -0700 Subject: [PATCH 0174/1425] Rename LogicalStore unit test file name. * Rename LogicalStore unit test file name. * Rename LogicalStore unit test file name. See merge request legate/legate.core.internal!94 --- tests/cpp/unit/{store.cc => logical_store.cc} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/cpp/unit/{store.cc => logical_store.cc} (100%) diff --git a/tests/cpp/unit/store.cc b/tests/cpp/unit/logical_store.cc similarity index 100% rename from tests/cpp/unit/store.cc rename to tests/cpp/unit/logical_store.cc From 75df0be8affa7b3e705c606c0d149cd384513f0f Mon Sep 17 00:00:00 2001 From: Eric A Niebler Date: Thu, 27 Jul 2023 10:34:49 -0700 Subject: [PATCH 0175/1425] add missing #include * add missing #include See merge request legate/legate.core.internal!97 --- src/core/data/allocator.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/data/allocator.cc b/src/core/data/allocator.cc index 42c5388cb4..6b759102b0 100644 --- a/src/core/data/allocator.cc +++ b/src/core/data/allocator.cc @@ -14,6 +14,8 @@ #include "core/data/buffer.h" #include "core/utilities/typedefs.h" +#include + namespace legate { class ScopedAllocator::Impl { From 0bbfecfe0b908e674c7c3c680fa3d26f34bfe7db Mon Sep 17 00:00:00 2001 From: Eric A Niebler Date: Thu, 27 Jul 2023 12:12:42 -0700 Subject: [PATCH 0176/1425] add a utility for using unique_ptr with incomplete types * install new memory utilities headers * actually use the new deleter utility with unique_ptr * add a utility for using unique_ptr with incomplete types See merge request legate/legate.core.internal!96 --- legate_core_cpp.cmake | 2 ++ src/core/partitioning/constraint.cc | 24 +------------- src/core/partitioning/constraint.h | 26 +++------------ src/core/partitioning/detail/constraint.cc | 6 ++++ src/core/utilities/memory.h | 38 ++++++++++++++++++++++ src/core/utilities/memory.inl | 26 +++++++++++++++ 6 files changed, 78 insertions(+), 44 deletions(-) create mode 100644 src/core/utilities/memory.h create mode 100644 src/core/utilities/memory.inl diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index 41803a2c38..30790c6b21 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -513,6 +513,8 @@ install( FILES src/core/utilities/debug.h src/core/utilities/dispatch.h src/core/utilities/machine.h + src/core/utilities/memory.h + src/core/utilities/memory.inl src/core/utilities/nvtx_help.h src/core/utilities/span.h src/core/utilities/tuple.h diff --git a/src/core/partitioning/constraint.cc b/src/core/partitioning/constraint.cc index 5486b9ec9c..7607d9dcef 100644 --- a/src/core/partitioning/constraint.cc +++ b/src/core/partitioning/constraint.cc @@ -19,33 +19,11 @@ std::string Variable::to_string() const { return impl_->to_string(); } Variable::Variable(const detail::Variable* impl) : impl_(impl) {} -Variable::~Variable() {} - -Variable::Variable(const Variable&) = default; - -Variable& Variable::operator=(const Variable&) = default; - std::string Constraint::to_string() const { return impl_->to_string(); } Constraint::Constraint(detail::Constraint* impl) : impl_(impl) {} -Constraint::~Constraint() { delete impl_; } - -Constraint::Constraint(Constraint&& other) : impl_(other.impl_) { other.impl_ = nullptr; } - -Constraint& Constraint::operator=(Constraint&& other) -{ - impl_ = other.impl_; - other.impl_ = nullptr; - return *this; -} - -detail::Constraint* Constraint::release() -{ - auto result = impl_; - impl_ = nullptr; - return result; -} +detail::Constraint* Constraint::release() { return impl_.release(); } Constraint align(Variable lhs, Variable rhs) { diff --git a/src/core/partitioning/constraint.h b/src/core/partitioning/constraint.h index fd520e69d5..633b5065c9 100644 --- a/src/core/partitioning/constraint.h +++ b/src/core/partitioning/constraint.h @@ -12,8 +12,7 @@ #pragma once -#include - +#include "core/utilities/memory.h" #include "core/utilities/tuple.h" /** @defgroup partitioning Partitioning @@ -31,6 +30,7 @@ namespace detail { class Constraint; class Variable; } // namespace detail +extern template class default_delete; /** * @ingroup partitioning @@ -42,15 +42,7 @@ class Variable { public: Variable(const detail::Variable* impl); - ~Variable(); - - public: - Variable(const Variable&); - Variable& operator=(const Variable&); - - private: - Variable(Variable&&) = delete; - Variable& operator=(Variable&&) = delete; + ~Variable() = default; public: const detail::Variable* impl() const { return impl_; } @@ -69,22 +61,14 @@ class Constraint { public: Constraint(detail::Constraint* impl); - ~Constraint(); - - public: - Constraint(Constraint&&); - Constraint& operator=(Constraint&&); - - private: - Constraint(const Constraint&) = delete; - Constraint& operator=(const Constraint&) = delete; + ~Constraint() = default; private: friend class AutoTask; detail::Constraint* release(); private: - detail::Constraint* impl_{nullptr}; + std::unique_ptr> impl_; }; /** diff --git a/src/core/partitioning/detail/constraint.cc b/src/core/partitioning/detail/constraint.cc index 1c0c1efa76..d2cd75b4b8 100644 --- a/src/core/partitioning/detail/constraint.cc +++ b/src/core/partitioning/detail/constraint.cc @@ -18,6 +18,7 @@ #include "core/operation/detail/operation.h" #include "core/partitioning/detail/partitioner.h" #include "core/partitioning/partition.h" +#include "core/utilities/memory.h" namespace legate::detail { @@ -156,3 +157,8 @@ std::unique_ptr image(const Variable* var_function, const Varia } } // namespace legate::detail + +// explicitly instantiate the deleter for std::unique_ptr +namespace legate { +template class default_delete; +} diff --git a/src/core/utilities/memory.h b/src/core/utilities/memory.h new file mode 100644 index 0000000000..86ab4f2ac6 --- /dev/null +++ b/src/core/utilities/memory.h @@ -0,0 +1,38 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#pragma once + +#include + +namespace legate { + +/// @brief deleter for using unique_ptr with incomplete types +/// @tparam T the type to delete +/// @code{.cpp} +/// // in header file: +/// struct Foo; +/// extern template class legate::default_delete; // Suppress instantiation +/// std::unique_ptr> foo; // OK +/// +/// // in source file: +/// struct Foo { int x; }; +/// template class legate::default_delete; // Explicit instantiation +/// @endcode +template +struct default_delete { + void operator()(T*) const; +}; + +} // namespace legate + +#include "core/utilities/memory.inl" diff --git a/src/core/utilities/memory.inl b/src/core/utilities/memory.inl new file mode 100644 index 0000000000..ed4547546a --- /dev/null +++ b/src/core/utilities/memory.inl @@ -0,0 +1,26 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#pragma once + +#include "core/utilities/memory.h" + +namespace legate { + +template +void default_delete::operator()(T* ptr) const +{ + static_assert(sizeof(T) > 0, "default_delete cannot be instantiated for incomplete type"); + delete ptr; +} + +} // namespace legate From df4f2c806b6022c5c37e797d76dfc86f2907a813 Mon Sep 17 00:00:00 2001 From: Jeremiah Wilke Date: Thu, 27 Jul 2023 16:06:01 -0700 Subject: [PATCH 0177/1425] Remove impl header from legate.h * remove impl header from legate.h See merge request legate/legate.core.internal!98 --- src/legate.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/legate.h b/src/legate.h index d897e0023b..c14fbf2163 100644 --- a/src/legate.h +++ b/src/legate.h @@ -29,7 +29,6 @@ #include "core/mapping/operation.h" #include "core/operation/task.h" #include "core/partitioning/constraint.h" -#include "core/partitioning/partition.h" #include "core/runtime/library.h" #include "core/runtime/runtime.h" #include "core/runtime/tracker.h" From e32c591b13db1c5a484edb4a6c4a95aa598151de Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Thu, 27 Jul 2023 18:14:54 -0700 Subject: [PATCH 0178/1425] A bag of minor fixes * Missing shebang * Update the line numbers to compare in the test to make it pass again * Make the test pass on a single processor * Remove an obsolete file * Fix a typo See merge request legate/legate.core.internal!99 --- src/core/data/logical_store.h | 2 +- src/core/mapping/detail/operation.inl | 39 ---------------------- tests/cpp/integration/image_constraints.cc | 2 +- tests/cpp/integration/provenance.cc | 4 +-- tests/cpp/run.py | 2 ++ 5 files changed, 6 insertions(+), 43 deletions(-) delete mode 100644 src/core/mapping/detail/operation.inl diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index 65ac7fd8e7..3022988ba0 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -243,7 +243,7 @@ class LogicalStore { * [4, 8]]] * @endcode * - * @code(.unparsed} + * @code{.unparsed} * [[[1, 5], * [3, 7]], * diff --git a/src/core/mapping/detail/operation.inl b/src/core/mapping/detail/operation.inl deleted file mode 100644 index 964dcfbd7b..0000000000 --- a/src/core/mapping/detail/operation.inl +++ /dev/null @@ -1,39 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: LicenseRef-NvidiaProprietary - * - * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual - * property and proprietary rights in and to this material, related - * documentation and any modifications thereto. Any use, reproduction, - * disclosure or distribution of this material and related documentation - * without an express license agreement from NVIDIA CORPORATION or - * its affiliates is strictly prohibited. - */ - -#pragma once - -// Useful for IDEs -#include "core/mapping/operation.h" - -namespace legate::mapping { - -template -Legion::Rect RegionField::shape(Legion::Mapping::MapperRuntime* runtime, - const Legion::Mapping::MapperContext context) const -{ - return Legion::Rect(domain(runtime, context)); -} - -template -Legion::Rect FutureWrapper::shape() const -{ - return Legion::Rect(domain()); -} - -template -Legion::Rect Store::shape() const -{ - return Legion::Rect(domain()); -} - -} // namespace legate::mapping diff --git a/tests/cpp/integration/image_constraints.cc b/tests/cpp/integration/image_constraints.cc index 0c3d0858fd..29c2fe6543 100644 --- a/tests/cpp/integration/image_constraints.cc +++ b/tests/cpp/integration/image_constraints.cc @@ -144,7 +144,7 @@ struct ImageTester : public legate::LegateTask> { auto& func = context.inputs().at(0); auto range = context.inputs().at(1).domain(); - EXPECT_FALSE(range.get_volume() > 1 && range.dense()); + EXPECT_FALSE(!context.is_single_task() && range.get_volume() > 1 && range.dense()); if constexpr (RECT) { const auto& rect_type = func.type().as_struct_type(); diff --git a/tests/cpp/integration/provenance.cc b/tests/cpp/integration/provenance.cc index 73160cd55a..0f436d006b 100644 --- a/tests/cpp/integration/provenance.cc +++ b/tests/cpp/integration/provenance.cc @@ -105,7 +105,7 @@ void test_provenance_tracker(legate::Library library) auto runtime = legate::Runtime::get_runtime(); // auto task auto task = runtime->create_task(library, PROVENANCE); - std::string provenance = "provenance.cc:108"; + std::string provenance = "provenance.cc:104"; task.add_scalar_arg(legate::Scalar(provenance)); runtime->submit(std::move(task)); } @@ -117,7 +117,7 @@ void test_nested_provenance_tracker(legate::Library library) // The provenance string used by test_provenance_tracker should be popped out at this point auto runtime = legate::Runtime::get_runtime(); auto task = runtime->create_task(library, PROVENANCE); - std::string provenance = "provenance.cc:119"; + std::string provenance = "provenance.cc:115"; task.add_scalar_arg(legate::Scalar(provenance)); runtime->submit(std::move(task)); } diff --git a/tests/cpp/run.py b/tests/cpp/run.py index 52ce071164..9b030fc52a 100755 --- a/tests/cpp/run.py +++ b/tests/cpp/run.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + # SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # From aa373a29a0dc8241f872a64811ab49c326de28f4 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 28 Jul 2023 09:59:44 -0700 Subject: [PATCH 0179/1425] Reformat license headers in Python files * Reformat license headers in Python files See merge request legate/legate.core.internal!100 --- docs/legate/core/source/conf.py | 3 ++- examples/hello/examples/cunumeric-variance.py | 3 ++- examples/hello/examples/hello-world.py | 3 ++- examples/hello/examples/variance.py | 4 +++- examples/hello/hello/hello.py | 3 ++- examples/hello/setup.py | 3 ++- examples/io/examples/even_tiles.py | 3 ++- examples/io/examples/single_file.py | 3 ++- examples/io/examples/uneven_tiles.py | 3 ++- examples/io/legateio/legateio.py | 3 ++- examples/io/setup.py | 3 ++- examples/reduction/examples/bincount.py | 3 ++- examples/reduction/examples/histogram.py | 3 ++- examples/reduction/examples/matmul.py | 3 ++- examples/reduction/examples/sum_over_axis.py | 3 ++- examples/reduction/examples/unique.py | 3 ++- examples/reduction/reduction/reduction.py | 3 ++- examples/reduction/setup.py | 3 ++- install.py | 3 ++- legate/_sphinxext/__init__.py | 1 - legate/_sphinxext/settings.py | 3 ++- legate/core/_legion/env.py | 3 ++- legate/core/_legion/field.py | 3 ++- legate/core/_legion/future.py | 3 ++- legate/core/_legion/geometry.py | 3 ++- legate/core/_legion/operation.py | 3 ++- legate/core/_legion/partition.py | 3 ++- legate/core/_legion/partition_functor.py | 3 ++- legate/core/_legion/pending.py | 3 ++- legate/core/_legion/region.py | 3 ++- legate/core/_legion/space.py | 3 ++- legate/core/_legion/task.py | 3 ++- legate/core/_legion/transform.py | 3 ++- legate/core/_legion/util.py | 3 ++- legate/core/allocation.py | 3 ++- legate/core/communicator.py | 3 ++- legate/core/constraints.py | 3 ++- legate/core/context.py | 3 ++- legate/core/corelib.py | 3 ++- legate/core/cycle_detector.py | 3 ++- legate/core/exception.py | 3 ++- legate/core/io.py | 3 ++- legate/core/launcher.py | 3 ++- legate/core/legate.py | 3 ++- legate/core/machine.py | 3 ++- legate/core/operation.py | 3 ++- legate/core/partition.py | 3 ++- legate/core/projection.py | 3 ++- legate/core/restriction.py | 3 ++- legate/core/runtime.py | 3 ++- legate/core/shape.py | 3 ++- legate/core/solver.py | 3 ++- legate/core/store.py | 3 ++- legate/core/transform.py | 3 ++- legate/core/types.py | 3 ++- legate/core/types.pyi | 1 - legate/core/utils.py | 3 ++- legate/driver/args.py | 3 ++- legate/driver/command.py | 3 ++- legate/driver/config.py | 3 ++- legate/driver/defaults.py | 3 ++- legate/driver/driver.py | 3 ++- legate/driver/launcher.py | 3 ++- legate/driver/logs.py | 3 ++- legate/driver/main.py | 3 ++- legate/jupyter/_legion_kernel.py | 3 ++- legate/jupyter/args.py | 3 ++- legate/jupyter/config.py | 3 ++- legate/jupyter/kernel.py | 3 ++- legate/jupyter/magic.py | 3 ++- legate/jupyter/main.py | 3 ++- legate/settings.py | 3 ++- legate/tester/args.py | 3 ++- legate/tester/config.py | 3 ++- legate/tester/logger.py | 3 ++- legate/tester/stages/_linux/cpu.py | 3 ++- legate/tester/stages/_linux/eager.py | 3 ++- legate/tester/stages/_linux/gpu.py | 3 ++- legate/tester/stages/_linux/omp.py | 3 ++- legate/tester/stages/_osx/cpu.py | 3 ++- legate/tester/stages/_osx/eager.py | 3 ++- legate/tester/stages/_osx/gpu.py | 3 ++- legate/tester/stages/_osx/omp.py | 3 ++- legate/tester/stages/test_stage.py | 3 ++- legate/tester/stages/util.py | 3 ++- legate/tester/test_plan.py | 3 ++- legate/tester/test_system.py | 3 ++- legate/timing/timing.py | 3 ++- legate/util/args.py | 3 ++- legate/util/colors.py | 3 ++- legate/util/fs.py | 3 ++- legate/util/info.py | 3 ++- legate/util/settings.py | 3 ++- legate/util/shared_args.py | 3 ++- legate/util/system.py | 3 ++- legate/util/types.py | 3 ++- legate/util/ui.py | 3 ++- setup.py | 3 ++- tests/cpp/run.py | 3 ++- tests/integration/collective/collective/collective.py | 3 ++- tests/integration/collective/collective/lib.py | 3 ++- tests/integration/collective/examples/test_collective.py | 3 ++- tests/integration/collective/setup.py | 3 ++- tests/integration/collective/src/collective_test.h | 1 - .../region_manager/examples/test_region_manager.py | 3 ++- tests/integration/region_manager/region_manager/lib.py | 3 ++- tests/integration/region_manager/setup.py | 3 ++- tests/integration/registry/examples/test_registry.py | 3 ++- tests/integration/registry/registry/lib.py | 3 ++- tests/integration/registry/setup.py | 3 ++- tests/integration/scoping/examples/test_scoping.py | 3 ++- tests/integration/scoping/scoping/lib.py | 3 ++- tests/integration/scoping/setup.py | 3 ++- tests/integration/tree_reduce/examples/test_tree_reduce.py | 3 ++- tests/integration/tree_reduce/setup.py | 3 ++- tests/integration/tree_reduce/tree_reduce/lib.py | 3 ++- tests/unit/legate/core/test_invalid_scalar_arg.py | 3 ++- tests/unit/legate/core/test_machine.py | 3 ++- tests/unit/legate/core/test_store.py | 3 ++- tests/unit/legate/core/test_type_uid.py | 3 ++- tests/unit/legate/driver/conftest.py | 3 ++- tests/unit/legate/driver/test_args.py | 3 ++- tests/unit/legate/driver/test_command.py | 3 ++- tests/unit/legate/driver/test_config.py | 3 ++- tests/unit/legate/driver/test_defaults.py | 3 ++- tests/unit/legate/driver/test_driver.py | 3 ++- tests/unit/legate/driver/test_launcher.py | 3 ++- tests/unit/legate/driver/test_logs.py | 3 ++- tests/unit/legate/driver/test_main.py | 3 ++- tests/unit/legate/driver/util.py | 3 ++- tests/unit/legate/jupyter/test_args.py | 3 ++- tests/unit/legate/jupyter/test_config.py | 3 ++- tests/unit/legate/jupyter/test_kernel.py | 3 ++- tests/unit/legate/jupyter/test_main.py | 3 ++- tests/unit/legate/test_cycle_check.py | 3 ++- tests/unit/legate/test_settings.py | 3 ++- tests/unit/legate/tester/stages/_linux/test_cpu.py | 3 ++- tests/unit/legate/tester/stages/_linux/test_eager.py | 3 ++- tests/unit/legate/tester/stages/_linux/test_gpu.py | 3 ++- tests/unit/legate/tester/stages/_linux/test_omp.py | 3 ++- tests/unit/legate/tester/stages/test_test_stage.py | 3 ++- tests/unit/legate/tester/stages/test_util.py | 3 ++- tests/unit/legate/tester/test___init__.py | 3 ++- tests/unit/legate/tester/test_args.py | 3 ++- tests/unit/legate/tester/test_config.py | 3 ++- tests/unit/legate/tester/test_logger.py | 3 ++- tests/unit/legate/tester/test_test_system.py | 3 ++- tests/unit/legate/util/test_args.py | 3 ++- tests/unit/legate/util/test_colors.py | 3 ++- tests/unit/legate/util/test_fs.py | 3 ++- tests/unit/legate/util/test_settings.py | 3 ++- tests/unit/legate/util/test_system.py | 3 ++- tests/unit/legate/util/test_types.py | 3 ++- tests/unit/legate/util/test_ui.py | 3 ++- tests/unit/util.py | 3 ++- typings/legion_cffi/lib.pyi | 1 - typings/legion_top/__init__.pyi | 1 - typings/pyarrow/__init__.pyi | 1 - typings/pyarrow/lib.pyi | 1 - 159 files changed, 305 insertions(+), 159 deletions(-) diff --git a/docs/legate/core/source/conf.py b/docs/legate/core/source/conf.py index 4a4498ec27..34aac6c84e 100644 --- a/docs/legate/core/source/conf.py +++ b/docs/legate/core/source/conf.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/examples/hello/examples/cunumeric-variance.py b/examples/hello/examples/cunumeric-variance.py index 0182e80641..a906f0993e 100644 --- a/examples/hello/examples/cunumeric-variance.py +++ b/examples/hello/examples/cunumeric-variance.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/examples/hello/examples/hello-world.py b/examples/hello/examples/hello-world.py index 12c1429a5e..d280a7cf16 100644 --- a/examples/hello/examples/hello-world.py +++ b/examples/hello/examples/hello-world.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/examples/hello/examples/variance.py b/examples/hello/examples/variance.py index 7675fa707e..f262f7b0c6 100644 --- a/examples/hello/examples/variance.py +++ b/examples/hello/examples/variance.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual @@ -11,6 +12,7 @@ # its affiliates is strictly prohibited. from typing import Any + from hello import iota, square, sum, to_scalar from legate.core import Store diff --git a/examples/hello/hello/hello.py b/examples/hello/hello/hello.py index 239eeb4829..4d07d335b9 100644 --- a/examples/hello/hello/hello.py +++ b/examples/hello/hello/hello.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/examples/hello/setup.py b/examples/hello/setup.py index 12cc4d3f94..8b1c104b1c 100644 --- a/examples/hello/setup.py +++ b/examples/hello/setup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/examples/io/examples/even_tiles.py b/examples/io/examples/even_tiles.py index c5af9905d1..e05f87dd1f 100644 --- a/examples/io/examples/even_tiles.py +++ b/examples/io/examples/even_tiles.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/examples/io/examples/single_file.py b/examples/io/examples/single_file.py index 8d8eecdffe..e7cb6c3e1d 100644 --- a/examples/io/examples/single_file.py +++ b/examples/io/examples/single_file.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/examples/io/examples/uneven_tiles.py b/examples/io/examples/uneven_tiles.py index 567187dfb0..855cc0bce6 100644 --- a/examples/io/examples/uneven_tiles.py +++ b/examples/io/examples/uneven_tiles.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/examples/io/legateio/legateio.py b/examples/io/legateio/legateio.py index 82c58ace11..f69e728da2 100644 --- a/examples/io/legateio/legateio.py +++ b/examples/io/legateio/legateio.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/examples/io/setup.py b/examples/io/setup.py index a62872d42d..bc5d754d73 100644 --- a/examples/io/setup.py +++ b/examples/io/setup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/examples/reduction/examples/bincount.py b/examples/reduction/examples/bincount.py index 260f529ca5..f74fa5a634 100644 --- a/examples/reduction/examples/bincount.py +++ b/examples/reduction/examples/bincount.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/examples/reduction/examples/histogram.py b/examples/reduction/examples/histogram.py index ae5e077b2c..417f81c648 100644 --- a/examples/reduction/examples/histogram.py +++ b/examples/reduction/examples/histogram.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/examples/reduction/examples/matmul.py b/examples/reduction/examples/matmul.py index 598200b97c..9524481836 100644 --- a/examples/reduction/examples/matmul.py +++ b/examples/reduction/examples/matmul.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/examples/reduction/examples/sum_over_axis.py b/examples/reduction/examples/sum_over_axis.py index 5892beeb42..7066ab298c 100644 --- a/examples/reduction/examples/sum_over_axis.py +++ b/examples/reduction/examples/sum_over_axis.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/examples/reduction/examples/unique.py b/examples/reduction/examples/unique.py index df8db764e8..ec82fe9cf2 100644 --- a/examples/reduction/examples/unique.py +++ b/examples/reduction/examples/unique.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/examples/reduction/reduction/reduction.py b/examples/reduction/reduction/reduction.py index 30ebd928b1..3b7c133ce7 100644 --- a/examples/reduction/reduction/reduction.py +++ b/examples/reduction/reduction/reduction.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/examples/reduction/setup.py b/examples/reduction/setup.py index 0561ca423d..b0c920cf9f 100644 --- a/examples/reduction/setup.py +++ b/examples/reduction/setup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/install.py b/install.py index 6ca1dbc02b..03cbb2e633 100755 --- a/install.py +++ b/install.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/_sphinxext/__init__.py b/legate/_sphinxext/__init__.py index 167966a07f..0e3c635062 100644 --- a/legate/_sphinxext/__init__.py +++ b/legate/_sphinxext/__init__.py @@ -7,4 +7,3 @@ # disclosure or distribution of this material and related documentation # without an express license agreement from NVIDIA CORPORATION or # its affiliates is strictly prohibited. - diff --git a/legate/_sphinxext/settings.py b/legate/_sphinxext/settings.py index 8c9652a0a7..fe125db7bd 100644 --- a/legate/_sphinxext/settings.py +++ b/legate/_sphinxext/settings.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/_legion/env.py b/legate/core/_legion/env.py index 7d054db818..ba36753267 100644 --- a/legate/core/_legion/env.py +++ b/legate/core/_legion/env.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/_legion/field.py b/legate/core/_legion/field.py index 06c407ec22..1898ad2d0c 100644 --- a/legate/core/_legion/field.py +++ b/legate/core/_legion/field.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/_legion/future.py b/legate/core/_legion/future.py index a57e5ecd6f..11b9447372 100644 --- a/legate/core/_legion/future.py +++ b/legate/core/_legion/future.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/_legion/geometry.py b/legate/core/_legion/geometry.py index 41f2303338..f472f7d9c6 100644 --- a/legate/core/_legion/geometry.py +++ b/legate/core/_legion/geometry.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/_legion/operation.py b/legate/core/_legion/operation.py index 0e6e323983..3caf5f3eac 100644 --- a/legate/core/_legion/operation.py +++ b/legate/core/_legion/operation.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/_legion/partition.py b/legate/core/_legion/partition.py index c86ccfe584..ca7c0c5e95 100644 --- a/legate/core/_legion/partition.py +++ b/legate/core/_legion/partition.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/_legion/partition_functor.py b/legate/core/_legion/partition_functor.py index 37d10bbc36..026a85a873 100644 --- a/legate/core/_legion/partition_functor.py +++ b/legate/core/_legion/partition_functor.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/_legion/pending.py b/legate/core/_legion/pending.py index b841a69ee1..7b2b601017 100644 --- a/legate/core/_legion/pending.py +++ b/legate/core/_legion/pending.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/_legion/region.py b/legate/core/_legion/region.py index 1d8a616feb..7121602986 100644 --- a/legate/core/_legion/region.py +++ b/legate/core/_legion/region.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/_legion/space.py b/legate/core/_legion/space.py index b7cab91ee0..f859f9c1e6 100644 --- a/legate/core/_legion/space.py +++ b/legate/core/_legion/space.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/_legion/task.py b/legate/core/_legion/task.py index f5cd980b8e..a2cb2c52f0 100644 --- a/legate/core/_legion/task.py +++ b/legate/core/_legion/task.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/_legion/transform.py b/legate/core/_legion/transform.py index a035813516..7c14865838 100644 --- a/legate/core/_legion/transform.py +++ b/legate/core/_legion/transform.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/_legion/util.py b/legate/core/_legion/util.py index e74469d5be..0e4cebc323 100644 --- a/legate/core/_legion/util.py +++ b/legate/core/_legion/util.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/allocation.py b/legate/core/allocation.py index 44c862edbf..0687286220 100644 --- a/legate/core/allocation.py +++ b/legate/core/allocation.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/communicator.py b/legate/core/communicator.py index 01615adca2..0a44bebb26 100644 --- a/legate/core/communicator.py +++ b/legate/core/communicator.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/constraints.py b/legate/core/constraints.py index 98c773c307..a7fc9fa6fe 100644 --- a/legate/core/constraints.py +++ b/legate/core/constraints.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/context.py b/legate/core/context.py index 57245abdb7..08f40ad6e3 100644 --- a/legate/core/context.py +++ b/legate/core/context.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/corelib.py b/legate/core/corelib.py index 4d30274ee0..813dcc4823 100644 --- a/legate/core/corelib.py +++ b/legate/core/corelib.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/cycle_detector.py b/legate/core/cycle_detector.py index 3f75e24bce..42b56ff45b 100644 --- a/legate/core/cycle_detector.py +++ b/legate/core/cycle_detector.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/exception.py b/legate/core/exception.py index 483adf2bc5..bf8d9f397c 100644 --- a/legate/core/exception.py +++ b/legate/core/exception.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/io.py b/legate/core/io.py index 8080116b47..7bf1c3f16b 100644 --- a/legate/core/io.py +++ b/legate/core/io.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/launcher.py b/legate/core/launcher.py index 53db9555b5..d95350c7a2 100644 --- a/legate/core/launcher.py +++ b/legate/core/launcher.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/legate.py b/legate/core/legate.py index 561ce77c53..d5dba9bf8e 100644 --- a/legate/core/legate.py +++ b/legate/core/legate.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/machine.py b/legate/core/machine.py index 87c1ced9fe..66928ea89a 100644 --- a/legate/core/machine.py +++ b/legate/core/machine.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/operation.py b/legate/core/operation.py index 91baa2d5eb..3d6a351e71 100644 --- a/legate/core/operation.py +++ b/legate/core/operation.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/partition.py b/legate/core/partition.py index ee8d16c327..bc6998da04 100644 --- a/legate/core/partition.py +++ b/legate/core/partition.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/projection.py b/legate/core/projection.py index 2e97d5eaac..d8b79dabc2 100644 --- a/legate/core/projection.py +++ b/legate/core/projection.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/restriction.py b/legate/core/restriction.py index e244ded10c..bd7e5a759a 100644 --- a/legate/core/restriction.py +++ b/legate/core/restriction.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/runtime.py b/legate/core/runtime.py index 70344d76d7..343494e33d 100644 --- a/legate/core/runtime.py +++ b/legate/core/runtime.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/shape.py b/legate/core/shape.py index fa70e97605..f8c6a56c32 100644 --- a/legate/core/shape.py +++ b/legate/core/shape.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/solver.py b/legate/core/solver.py index 5fc5f4c936..1d1434b4a4 100644 --- a/legate/core/solver.py +++ b/legate/core/solver.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/store.py b/legate/core/store.py index 6e5d7d2f31..3d3410c4e7 100644 --- a/legate/core/store.py +++ b/legate/core/store.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/transform.py b/legate/core/transform.py index ad5fe06fbe..342c3f135d 100644 --- a/legate/core/transform.py +++ b/legate/core/transform.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/types.py b/legate/core/types.py index 820cd9df7a..36a21ced50 100644 --- a/legate/core/types.py +++ b/legate/core/types.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/core/types.pyi b/legate/core/types.pyi index 15db64bb57..c9afe507ae 100644 --- a/legate/core/types.pyi +++ b/legate/core/types.pyi @@ -8,7 +8,6 @@ # without an express license agreement from NVIDIA CORPORATION or # its affiliates is strictly prohibited. - from typing import Any import numpy as np diff --git a/legate/core/utils.py b/legate/core/utils.py index 7150692257..c035498a43 100644 --- a/legate/core/utils.py +++ b/legate/core/utils.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/driver/args.py b/legate/driver/args.py index 8b715f87bd..1731777846 100644 --- a/legate/driver/args.py +++ b/legate/driver/args.py @@ -1,6 +1,7 @@ #!/usr/bin/env python -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/driver/command.py b/legate/driver/command.py index 6e92b0b5e1..1d48f92786 100644 --- a/legate/driver/command.py +++ b/legate/driver/command.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/driver/config.py b/legate/driver/config.py index 5efe3ccb73..4fed9c8ecb 100644 --- a/legate/driver/config.py +++ b/legate/driver/config.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/driver/defaults.py b/legate/driver/defaults.py index 698b3df6ae..d332cc6f4e 100644 --- a/legate/driver/defaults.py +++ b/legate/driver/defaults.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/driver/driver.py b/legate/driver/driver.py index f605a2614c..8e200252e3 100644 --- a/legate/driver/driver.py +++ b/legate/driver/driver.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/driver/launcher.py b/legate/driver/launcher.py index 8f72c65a9e..7d059bf4c5 100644 --- a/legate/driver/launcher.py +++ b/legate/driver/launcher.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/driver/logs.py b/legate/driver/logs.py index 7a0d6d052b..2c4077bbaa 100644 --- a/legate/driver/logs.py +++ b/legate/driver/logs.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/driver/main.py b/legate/driver/main.py index e41b755548..aa96b0683b 100644 --- a/legate/driver/main.py +++ b/legate/driver/main.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/jupyter/_legion_kernel.py b/legate/jupyter/_legion_kernel.py index 60ad831841..bcacd8796c 100644 --- a/legate/jupyter/_legion_kernel.py +++ b/legate/jupyter/_legion_kernel.py @@ -1,6 +1,7 @@ #!/usr/bin/env python -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/jupyter/args.py b/legate/jupyter/args.py index e1c95ffc3c..fb7eb28cc6 100644 --- a/legate/jupyter/args.py +++ b/legate/jupyter/args.py @@ -1,6 +1,7 @@ #!/usr/bin/env python -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/jupyter/config.py b/legate/jupyter/config.py index 7f20d06d5a..ba43561932 100644 --- a/legate/jupyter/config.py +++ b/legate/jupyter/config.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/jupyter/kernel.py b/legate/jupyter/kernel.py index e95b429585..52aee9994d 100644 --- a/legate/jupyter/kernel.py +++ b/legate/jupyter/kernel.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/jupyter/magic.py b/legate/jupyter/magic.py index 54b5c9a1fa..b1f9c89781 100644 --- a/legate/jupyter/magic.py +++ b/legate/jupyter/magic.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/jupyter/main.py b/legate/jupyter/main.py index 0423487554..e15b0638c3 100644 --- a/legate/jupyter/main.py +++ b/legate/jupyter/main.py @@ -1,6 +1,7 @@ #!/usr/bin/env python -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/settings.py b/legate/settings.py index 25a6cd6042..4768612b29 100644 --- a/legate/settings.py +++ b/legate/settings.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/tester/args.py b/legate/tester/args.py index 3a878acfe5..762d82dfad 100644 --- a/legate/tester/args.py +++ b/legate/tester/args.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/tester/config.py b/legate/tester/config.py index ef6a8b59a3..75e96bf1ec 100644 --- a/legate/tester/config.py +++ b/legate/tester/config.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/tester/logger.py b/legate/tester/logger.py index 3430dab7ba..b4a6ce655e 100644 --- a/legate/tester/logger.py +++ b/legate/tester/logger.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/tester/stages/_linux/cpu.py b/legate/tester/stages/_linux/cpu.py index 6f26ebcabe..8ab2d1cea5 100644 --- a/legate/tester/stages/_linux/cpu.py +++ b/legate/tester/stages/_linux/cpu.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/tester/stages/_linux/eager.py b/legate/tester/stages/_linux/eager.py index cd067389f5..52b070c775 100644 --- a/legate/tester/stages/_linux/eager.py +++ b/legate/tester/stages/_linux/eager.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/tester/stages/_linux/gpu.py b/legate/tester/stages/_linux/gpu.py index 584f3fc417..2abf1a06d7 100644 --- a/legate/tester/stages/_linux/gpu.py +++ b/legate/tester/stages/_linux/gpu.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/tester/stages/_linux/omp.py b/legate/tester/stages/_linux/omp.py index 1b77d5d4b4..0fd0085bbc 100644 --- a/legate/tester/stages/_linux/omp.py +++ b/legate/tester/stages/_linux/omp.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/tester/stages/_osx/cpu.py b/legate/tester/stages/_osx/cpu.py index f9a65a8820..c075d99372 100644 --- a/legate/tester/stages/_osx/cpu.py +++ b/legate/tester/stages/_osx/cpu.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/tester/stages/_osx/eager.py b/legate/tester/stages/_osx/eager.py index f2a302bd3c..135dc784b7 100644 --- a/legate/tester/stages/_osx/eager.py +++ b/legate/tester/stages/_osx/eager.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/tester/stages/_osx/gpu.py b/legate/tester/stages/_osx/gpu.py index cd004a0fb6..d9da558c86 100644 --- a/legate/tester/stages/_osx/gpu.py +++ b/legate/tester/stages/_osx/gpu.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/tester/stages/_osx/omp.py b/legate/tester/stages/_osx/omp.py index 6835c6c097..a4f5a3422b 100644 --- a/legate/tester/stages/_osx/omp.py +++ b/legate/tester/stages/_osx/omp.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/tester/stages/test_stage.py b/legate/tester/stages/test_stage.py index 523e0ff32b..a518513aee 100644 --- a/legate/tester/stages/test_stage.py +++ b/legate/tester/stages/test_stage.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/tester/stages/util.py b/legate/tester/stages/util.py index ce6506cbbc..794e15de06 100644 --- a/legate/tester/stages/util.py +++ b/legate/tester/stages/util.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/tester/test_plan.py b/legate/tester/test_plan.py index 561fb9ad7f..034be761b6 100644 --- a/legate/tester/test_plan.py +++ b/legate/tester/test_plan.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/tester/test_system.py b/legate/tester/test_system.py index 2d0a271b02..4e9365e6a0 100644 --- a/legate/tester/test_system.py +++ b/legate/tester/test_system.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/timing/timing.py b/legate/timing/timing.py index 1ac9ee67ad..73078e5bc7 100644 --- a/legate/timing/timing.py +++ b/legate/timing/timing.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/util/args.py b/legate/util/args.py index 3a2431fe5e..908befc977 100644 --- a/legate/util/args.py +++ b/legate/util/args.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/util/colors.py b/legate/util/colors.py index 81f1a17819..10a5fe894c 100644 --- a/legate/util/colors.py +++ b/legate/util/colors.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/util/fs.py b/legate/util/fs.py index 2e1de8906e..4ceb6dc288 100644 --- a/legate/util/fs.py +++ b/legate/util/fs.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/util/info.py b/legate/util/info.py index 81703063a5..283cb43cf9 100644 --- a/legate/util/info.py +++ b/legate/util/info.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/util/settings.py b/legate/util/settings.py index 8c25fea1b9..8d6a400da1 100644 --- a/legate/util/settings.py +++ b/legate/util/settings.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/util/shared_args.py b/legate/util/shared_args.py index f9181cf011..dffaa57276 100644 --- a/legate/util/shared_args.py +++ b/legate/util/shared_args.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/util/system.py b/legate/util/system.py index a6b6307a7a..f117946af9 100644 --- a/legate/util/system.py +++ b/legate/util/system.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/util/types.py b/legate/util/types.py index 5524e53f5f..9178fc106f 100644 --- a/legate/util/types.py +++ b/legate/util/types.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/legate/util/ui.py b/legate/util/ui.py index 7210ba072b..aefb81ea85 100644 --- a/legate/util/ui.py +++ b/legate/util/ui.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/setup.py b/setup.py index 9c0bf9c8a3..3b885b8c0c 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/cpp/run.py b/tests/cpp/run.py index 9b030fc52a..5cb8489585 100755 --- a/tests/cpp/run.py +++ b/tests/cpp/run.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/integration/collective/collective/collective.py b/tests/integration/collective/collective/collective.py index d4f9688c25..42091d7491 100644 --- a/tests/integration/collective/collective/collective.py +++ b/tests/integration/collective/collective/collective.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/integration/collective/collective/lib.py b/tests/integration/collective/collective/lib.py index 1d6d17b55c..7849bb99f9 100644 --- a/tests/integration/collective/collective/lib.py +++ b/tests/integration/collective/collective/lib.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/integration/collective/examples/test_collective.py b/tests/integration/collective/examples/test_collective.py index 4e3f7a4b2c..6b2d2c2e0a 100644 --- a/tests/integration/collective/examples/test_collective.py +++ b/tests/integration/collective/examples/test_collective.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/integration/collective/setup.py b/tests/integration/collective/setup.py index 9de5d3d3d4..d1c6e2f776 100644 --- a/tests/integration/collective/setup.py +++ b/tests/integration/collective/setup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/integration/collective/src/collective_test.h b/tests/integration/collective/src/collective_test.h index 00fba7be62..37ab9a5fdc 100644 --- a/tests/integration/collective/src/collective_test.h +++ b/tests/integration/collective/src/collective_test.h @@ -10,7 +10,6 @@ * its affiliates is strictly prohibited. */ - #pragma once #include "collective_cffi.h" diff --git a/tests/integration/region_manager/examples/test_region_manager.py b/tests/integration/region_manager/examples/test_region_manager.py index d91c19e8bb..84d2d06668 100644 --- a/tests/integration/region_manager/examples/test_region_manager.py +++ b/tests/integration/region_manager/examples/test_region_manager.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/integration/region_manager/region_manager/lib.py b/tests/integration/region_manager/region_manager/lib.py index c63d171427..115e2157e9 100644 --- a/tests/integration/region_manager/region_manager/lib.py +++ b/tests/integration/region_manager/region_manager/lib.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/integration/region_manager/setup.py b/tests/integration/region_manager/setup.py index dc6134680f..7723affea1 100644 --- a/tests/integration/region_manager/setup.py +++ b/tests/integration/region_manager/setup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/integration/registry/examples/test_registry.py b/tests/integration/registry/examples/test_registry.py index 5081d3d03f..a8ec2176af 100644 --- a/tests/integration/registry/examples/test_registry.py +++ b/tests/integration/registry/examples/test_registry.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/integration/registry/registry/lib.py b/tests/integration/registry/registry/lib.py index 323ac8202f..0fce696aaa 100644 --- a/tests/integration/registry/registry/lib.py +++ b/tests/integration/registry/registry/lib.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/integration/registry/setup.py b/tests/integration/registry/setup.py index 4a398cfd31..1d67311124 100644 --- a/tests/integration/registry/setup.py +++ b/tests/integration/registry/setup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/integration/scoping/examples/test_scoping.py b/tests/integration/scoping/examples/test_scoping.py index a6c501b737..f70614a3b4 100644 --- a/tests/integration/scoping/examples/test_scoping.py +++ b/tests/integration/scoping/examples/test_scoping.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/integration/scoping/scoping/lib.py b/tests/integration/scoping/scoping/lib.py index 837b243550..a5cf2642ca 100644 --- a/tests/integration/scoping/scoping/lib.py +++ b/tests/integration/scoping/scoping/lib.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/integration/scoping/setup.py b/tests/integration/scoping/setup.py index c7c9971e67..9baa21d5f4 100644 --- a/tests/integration/scoping/setup.py +++ b/tests/integration/scoping/setup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/integration/tree_reduce/examples/test_tree_reduce.py b/tests/integration/tree_reduce/examples/test_tree_reduce.py index 1cb38ffd79..82f27722a2 100644 --- a/tests/integration/tree_reduce/examples/test_tree_reduce.py +++ b/tests/integration/tree_reduce/examples/test_tree_reduce.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/integration/tree_reduce/setup.py b/tests/integration/tree_reduce/setup.py index ba63c2a5b2..3f45b34842 100644 --- a/tests/integration/tree_reduce/setup.py +++ b/tests/integration/tree_reduce/setup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/integration/tree_reduce/tree_reduce/lib.py b/tests/integration/tree_reduce/tree_reduce/lib.py index 9b18fad026..6e0052c4a5 100644 --- a/tests/integration/tree_reduce/tree_reduce/lib.py +++ b/tests/integration/tree_reduce/tree_reduce/lib.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/core/test_invalid_scalar_arg.py b/tests/unit/legate/core/test_invalid_scalar_arg.py index 5aa26ad356..2cd4eae16a 100644 --- a/tests/unit/legate/core/test_invalid_scalar_arg.py +++ b/tests/unit/legate/core/test_invalid_scalar_arg.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/core/test_machine.py b/tests/unit/legate/core/test_machine.py index b38ecb26aa..7c20274aa1 100644 --- a/tests/unit/legate/core/test_machine.py +++ b/tests/unit/legate/core/test_machine.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/core/test_store.py b/tests/unit/legate/core/test_store.py index d34fa68c1b..c4a1c10210 100644 --- a/tests/unit/legate/core/test_store.py +++ b/tests/unit/legate/core/test_store.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/core/test_type_uid.py b/tests/unit/legate/core/test_type_uid.py index 0fa86240be..4c11e7b96c 100644 --- a/tests/unit/legate/core/test_type_uid.py +++ b/tests/unit/legate/core/test_type_uid.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/driver/conftest.py b/tests/unit/legate/driver/conftest.py index a31e865925..4cda9ab774 100644 --- a/tests/unit/legate/driver/conftest.py +++ b/tests/unit/legate/driver/conftest.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/driver/test_args.py b/tests/unit/legate/driver/test_args.py index ca73d89de2..3292bc9b4f 100644 --- a/tests/unit/legate/driver/test_args.py +++ b/tests/unit/legate/driver/test_args.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/driver/test_command.py b/tests/unit/legate/driver/test_command.py index 9c245b8388..a163a3cdc3 100644 --- a/tests/unit/legate/driver/test_command.py +++ b/tests/unit/legate/driver/test_command.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/driver/test_config.py b/tests/unit/legate/driver/test_config.py index 1741672190..24c3793422 100644 --- a/tests/unit/legate/driver/test_config.py +++ b/tests/unit/legate/driver/test_config.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/driver/test_defaults.py b/tests/unit/legate/driver/test_defaults.py index a5420b16cc..85a1649448 100644 --- a/tests/unit/legate/driver/test_defaults.py +++ b/tests/unit/legate/driver/test_defaults.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/driver/test_driver.py b/tests/unit/legate/driver/test_driver.py index e54432a7a3..c36d2f74c0 100644 --- a/tests/unit/legate/driver/test_driver.py +++ b/tests/unit/legate/driver/test_driver.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/driver/test_launcher.py b/tests/unit/legate/driver/test_launcher.py index 22748263f8..cf05734e48 100644 --- a/tests/unit/legate/driver/test_launcher.py +++ b/tests/unit/legate/driver/test_launcher.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/driver/test_logs.py b/tests/unit/legate/driver/test_logs.py index 9670b6cc25..7f3be7740b 100644 --- a/tests/unit/legate/driver/test_logs.py +++ b/tests/unit/legate/driver/test_logs.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/driver/test_main.py b/tests/unit/legate/driver/test_main.py index 67ff8d1328..a2e1294fa6 100644 --- a/tests/unit/legate/driver/test_main.py +++ b/tests/unit/legate/driver/test_main.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/driver/util.py b/tests/unit/legate/driver/util.py index ed38018299..a0e7b76c67 100644 --- a/tests/unit/legate/driver/util.py +++ b/tests/unit/legate/driver/util.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/jupyter/test_args.py b/tests/unit/legate/jupyter/test_args.py index fe624a750e..1efd20bc16 100644 --- a/tests/unit/legate/jupyter/test_args.py +++ b/tests/unit/legate/jupyter/test_args.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/jupyter/test_config.py b/tests/unit/legate/jupyter/test_config.py index c1c1b633c5..1361496411 100644 --- a/tests/unit/legate/jupyter/test_config.py +++ b/tests/unit/legate/jupyter/test_config.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/jupyter/test_kernel.py b/tests/unit/legate/jupyter/test_kernel.py index 828becfd6f..6f2aabac34 100644 --- a/tests/unit/legate/jupyter/test_kernel.py +++ b/tests/unit/legate/jupyter/test_kernel.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/jupyter/test_main.py b/tests/unit/legate/jupyter/test_main.py index f74dace850..627ba9fe0c 100644 --- a/tests/unit/legate/jupyter/test_main.py +++ b/tests/unit/legate/jupyter/test_main.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/test_cycle_check.py b/tests/unit/legate/test_cycle_check.py index 454b3fe6c3..d7869d5e63 100644 --- a/tests/unit/legate/test_cycle_check.py +++ b/tests/unit/legate/test_cycle_check.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/test_settings.py b/tests/unit/legate/test_settings.py index 88d9be4bff..c14f941e7a 100644 --- a/tests/unit/legate/test_settings.py +++ b/tests/unit/legate/test_settings.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/tester/stages/_linux/test_cpu.py b/tests/unit/legate/tester/stages/_linux/test_cpu.py index 66f4d2ba21..c4a4546a26 100644 --- a/tests/unit/legate/tester/stages/_linux/test_cpu.py +++ b/tests/unit/legate/tester/stages/_linux/test_cpu.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/tester/stages/_linux/test_eager.py b/tests/unit/legate/tester/stages/_linux/test_eager.py index 3cf0bce844..9ca55fadd8 100644 --- a/tests/unit/legate/tester/stages/_linux/test_eager.py +++ b/tests/unit/legate/tester/stages/_linux/test_eager.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/tester/stages/_linux/test_gpu.py b/tests/unit/legate/tester/stages/_linux/test_gpu.py index 76877bb506..2c83cbb7ed 100644 --- a/tests/unit/legate/tester/stages/_linux/test_gpu.py +++ b/tests/unit/legate/tester/stages/_linux/test_gpu.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/tester/stages/_linux/test_omp.py b/tests/unit/legate/tester/stages/_linux/test_omp.py index 8b39a31672..2bc16107c2 100644 --- a/tests/unit/legate/tester/stages/_linux/test_omp.py +++ b/tests/unit/legate/tester/stages/_linux/test_omp.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/tester/stages/test_test_stage.py b/tests/unit/legate/tester/stages/test_test_stage.py index 866f4a1e06..7aa18ed97f 100644 --- a/tests/unit/legate/tester/stages/test_test_stage.py +++ b/tests/unit/legate/tester/stages/test_test_stage.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/tester/stages/test_util.py b/tests/unit/legate/tester/stages/test_util.py index 709746e1fb..43533211b4 100644 --- a/tests/unit/legate/tester/stages/test_util.py +++ b/tests/unit/legate/tester/stages/test_util.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/tester/test___init__.py b/tests/unit/legate/tester/test___init__.py index 2e12a6e2ba..9844405091 100644 --- a/tests/unit/legate/tester/test___init__.py +++ b/tests/unit/legate/tester/test___init__.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/tester/test_args.py b/tests/unit/legate/tester/test_args.py index 67735e5310..226a1265e5 100644 --- a/tests/unit/legate/tester/test_args.py +++ b/tests/unit/legate/tester/test_args.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/tester/test_config.py b/tests/unit/legate/tester/test_config.py index ea32c4d33d..8136e335cc 100644 --- a/tests/unit/legate/tester/test_config.py +++ b/tests/unit/legate/tester/test_config.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/tester/test_logger.py b/tests/unit/legate/tester/test_logger.py index 3a4cb20bb4..8783a2b831 100644 --- a/tests/unit/legate/tester/test_logger.py +++ b/tests/unit/legate/tester/test_logger.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/tester/test_test_system.py b/tests/unit/legate/tester/test_test_system.py index c72aebcc07..108cb242cb 100644 --- a/tests/unit/legate/tester/test_test_system.py +++ b/tests/unit/legate/tester/test_test_system.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/util/test_args.py b/tests/unit/legate/util/test_args.py index 1d52dbbf7d..6b6ae72446 100644 --- a/tests/unit/legate/util/test_args.py +++ b/tests/unit/legate/util/test_args.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/util/test_colors.py b/tests/unit/legate/util/test_colors.py index f230e2532c..3f4fb95ea4 100644 --- a/tests/unit/legate/util/test_colors.py +++ b/tests/unit/legate/util/test_colors.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/util/test_fs.py b/tests/unit/legate/util/test_fs.py index 5408758050..93aa1e7ede 100644 --- a/tests/unit/legate/util/test_fs.py +++ b/tests/unit/legate/util/test_fs.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/util/test_settings.py b/tests/unit/legate/util/test_settings.py index 1e3983d688..a41d39f916 100644 --- a/tests/unit/legate/util/test_settings.py +++ b/tests/unit/legate/util/test_settings.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/util/test_system.py b/tests/unit/legate/util/test_system.py index 815adb6dcb..c616dff118 100644 --- a/tests/unit/legate/util/test_system.py +++ b/tests/unit/legate/util/test_system.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/util/test_types.py b/tests/unit/legate/util/test_types.py index 9be6f5771c..bffa080f6e 100644 --- a/tests/unit/legate/util/test_types.py +++ b/tests/unit/legate/util/test_types.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/legate/util/test_ui.py b/tests/unit/legate/util/test_ui.py index a193a39552..798b89e19f 100644 --- a/tests/unit/legate/util/test_ui.py +++ b/tests/unit/legate/util/test_ui.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/unit/util.py b/tests/unit/util.py index 768eb712ea..51fc7c52fd 100644 --- a/tests/unit/util.py +++ b/tests/unit/util.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/typings/legion_cffi/lib.pyi b/typings/legion_cffi/lib.pyi index e9d1739eda..cd08e07c58 100644 --- a/typings/legion_cffi/lib.pyi +++ b/typings/legion_cffi/lib.pyi @@ -8,7 +8,6 @@ # without an express license agreement from NVIDIA CORPORATION or # its affiliates is strictly prohibited. - from typing import Any class legion_runtime_t: ... diff --git a/typings/legion_top/__init__.pyi b/typings/legion_top/__init__.pyi index 0d1e3a9712..4102cbcd29 100644 --- a/typings/legion_top/__init__.pyi +++ b/typings/legion_top/__init__.pyi @@ -8,7 +8,6 @@ # without an express license agreement from NVIDIA CORPORATION or # its affiliates is strictly prohibited. - from typing import Any, Callable, Tuple from legion_cffi.lib import legion_context_t, legion_runtime_t diff --git a/typings/pyarrow/__init__.pyi b/typings/pyarrow/__init__.pyi index d9626385ab..956ff7aa16 100644 --- a/typings/pyarrow/__init__.pyi +++ b/typings/pyarrow/__init__.pyi @@ -8,7 +8,6 @@ # without an express license agreement from NVIDIA CORPORATION or # its affiliates is strictly prohibited. - from typing import Any, Union from .lib import ( diff --git a/typings/pyarrow/lib.pyi b/typings/pyarrow/lib.pyi index 4bd782ed19..f62d8abe9d 100644 --- a/typings/pyarrow/lib.pyi +++ b/typings/pyarrow/lib.pyi @@ -8,7 +8,6 @@ # without an express license agreement from NVIDIA CORPORATION or # its affiliates is strictly prohibited. - from typing import Any class DataType: From ef3b3f5f02a590671482537c26d4d284f8b8b5d8 Mon Sep 17 00:00:00 2001 From: Bryan Van de Ven Date: Fri, 28 Jul 2023 10:29:54 -0700 Subject: [PATCH 0180/1425] Reduction copies * docs * Basic test for ADD reduction * updates for copies PR * chekpoint test * don't pass read_write * improve expr with ternary * pass missing redop * default only one of each pair * make new arg optional * Add support for reduction copies Co-authored-by: Wonchan Lee See merge request legate/legate.core.internal!88 --- src/core/operation/detail/copy.cc | 17 ++- src/core/operation/detail/copy.h | 5 +- src/core/operation/detail/gather.cc | 17 ++- src/core/operation/detail/gather.h | 5 +- src/core/operation/detail/scatter.cc | 17 ++- src/core/operation/detail/scatter.h | 5 +- src/core/operation/detail/scatter_gather.cc | 17 ++- src/core/operation/detail/scatter_gather.h | 5 +- src/core/runtime/detail/runtime.cc | 24 ++-- src/core/runtime/detail/runtime.h | 13 +- src/core/runtime/runtime.cc | 60 +++++++-- src/core/runtime/runtime.h | 101 ++++++++++++++- tests/cpp/integration/copy_normal.cc | 131 +++++++++++++++++--- 13 files changed, 360 insertions(+), 57 deletions(-) diff --git a/src/core/operation/detail/copy.cc b/src/core/operation/detail/copy.cc index 88c032a551..df62ee81c3 100644 --- a/src/core/operation/detail/copy.cc +++ b/src/core/operation/detail/copy.cc @@ -17,17 +17,20 @@ #include "core/partitioning/detail/constraint_solver.h" #include "core/partitioning/detail/partitioner.h" #include "core/partitioning/partition.h" +#include "core/type/detail/type_info.h" namespace legate::detail { Copy::Copy(std::shared_ptr target, std::shared_ptr source, int64_t unique_id, - mapping::detail::Machine&& machine) + mapping::detail::Machine&& machine, + std::optional redop) : Operation(unique_id, std::move(machine)), target_{target.get(), declare_partition()}, source_{source.get(), declare_partition()}, - constraint_(align(target_.variable, source_.variable)) + constraint_(align(target_.variable, source_.variable)), + redop_{redop} { record_partition(target_.variable, std::move(target)); record_partition(source_.variable, std::move(source)); @@ -55,7 +58,15 @@ void Copy::launch(Strategy* p_strategy) auto launch_domain = strategy.launch_domain(this); launcher.add_input(source_.store, create_projection_info(strategy, launch_domain, source_)); - launcher.add_output(target_.store, create_projection_info(strategy, launch_domain, target_)); + + if (!redop_) { + launcher.add_output(target_.store, create_projection_info(strategy, launch_domain, target_)); + } else { + auto store_partition = target_.store->create_partition(strategy[target_.variable]); + auto proj = store_partition->create_projection_info(launch_domain); + proj->set_reduction_op(target_.store->type()->find_reduction_operator(redop_.value())); + launcher.add_reduction(target_.store, std::move(proj)); + } if (launch_domain != nullptr) { return launcher.execute(*launch_domain); diff --git a/src/core/operation/detail/copy.h b/src/core/operation/detail/copy.h index 936c9e79f4..d2ac322a55 100644 --- a/src/core/operation/detail/copy.h +++ b/src/core/operation/detail/copy.h @@ -13,6 +13,7 @@ #pragma once #include +#include #include "core/data/detail/logical_store.h" #include "core/operation/detail/operation.h" @@ -27,7 +28,8 @@ class Copy : public Operation { Copy(std::shared_ptr target, std::shared_ptr source, int64_t unique_id, - mapping::detail::Machine&& machine); + mapping::detail::Machine&& machine, + std::optional redop); public: void validate() override; @@ -43,6 +45,7 @@ class Copy : public Operation { StoreArg target_; StoreArg source_; std::unique_ptr constraint_; + std::optional redop_; }; } // namespace legate::detail diff --git a/src/core/operation/detail/gather.cc b/src/core/operation/detail/gather.cc index 40334fd2ea..bcd417372f 100644 --- a/src/core/operation/detail/gather.cc +++ b/src/core/operation/detail/gather.cc @@ -24,12 +24,14 @@ Gather::Gather(std::shared_ptr target, std::shared_ptr source, std::shared_ptr source_indirect, int64_t unique_id, - mapping::detail::Machine&& machine) + mapping::detail::Machine&& machine, + std::optional redop) : Operation(unique_id, std::move(machine)), target_{target.get(), declare_partition()}, source_{source.get(), declare_partition()}, source_indirect_{source_indirect.get(), declare_partition()}, - constraint_(align(target_.variable, source_indirect_.variable)) + constraint_(align(target_.variable, source_indirect_.variable)), + redop_{redop} { record_partition(target_.variable, std::move(target)); record_partition(source_.variable, std::move(source)); @@ -68,7 +70,16 @@ void Gather::launch(Strategy* p_strategy) launcher.add_input(source_.store, create_projection_info(strategy, launch_domain, source_)); launcher.add_source_indirect(source_indirect_.store, create_projection_info(strategy, launch_domain, source_indirect_)); - launcher.add_output(target_.store, create_projection_info(strategy, launch_domain, target_)); + + if (!redop_) { + launcher.add_output(target_.store, create_projection_info(strategy, launch_domain, target_)); + } else { + auto store_partition = target_.store->create_partition(strategy[target_.variable]); + auto proj = store_partition->create_projection_info(launch_domain); + proj->set_reduction_op(target_.store->type()->find_reduction_operator(redop_.value())); + launcher.add_reduction(target_.store, std::move(proj)); + } + launcher.set_source_indirect_out_of_range(out_of_range_); if (launch_domain != nullptr) { diff --git a/src/core/operation/detail/gather.h b/src/core/operation/detail/gather.h index a9244c7683..8265252be1 100644 --- a/src/core/operation/detail/gather.h +++ b/src/core/operation/detail/gather.h @@ -13,6 +13,7 @@ #pragma once #include +#include #include "core/data/detail/logical_store.h" #include "core/operation/detail/operation.h" @@ -28,7 +29,8 @@ class Gather : public Operation { std::shared_ptr source, std::shared_ptr source_indirect, int64_t unique_id, - mapping::detail::Machine&& machine); + mapping::detail::Machine&& machine, + std::optional redop); public: void set_indirect_out_of_range(bool flag) { out_of_range_ = flag; } @@ -49,6 +51,7 @@ class Gather : public Operation { StoreArg source_; StoreArg source_indirect_; std::unique_ptr constraint_; + std::optional redop_; }; } // namespace legate::detail diff --git a/src/core/operation/detail/scatter.cc b/src/core/operation/detail/scatter.cc index 1a39e34d4d..8144352369 100644 --- a/src/core/operation/detail/scatter.cc +++ b/src/core/operation/detail/scatter.cc @@ -24,12 +24,14 @@ Scatter::Scatter(std::shared_ptr target, std::shared_ptr target_indirect, std::shared_ptr source, int64_t unique_id, - mapping::detail::Machine&& machine) + mapping::detail::Machine&& machine, + std::optional redop) : Operation(unique_id, std::move(machine)), target_{target.get(), declare_partition()}, target_indirect_{target_indirect.get(), declare_partition()}, source_{source.get(), declare_partition()}, - constraint_(align(source_.variable, target_indirect_.variable)) + constraint_(align(source_.variable, target_indirect_.variable)), + redop_{redop} { record_partition(target_.variable, std::move(target)); record_partition(target_indirect_.variable, std::move(target_indirect)); @@ -66,7 +68,16 @@ void Scatter::launch(Strategy* p_strategy) auto launch_domain = strategy.launch_domain(this); launcher.add_input(source_.store, create_projection_info(strategy, launch_domain, source_)); - launcher.add_inout(target_.store, create_projection_info(strategy, launch_domain, target_)); + + if (!redop_) { + launcher.add_inout(target_.store, create_projection_info(strategy, launch_domain, target_)); + } else { + auto store_partition = target_.store->create_partition(strategy[target_.variable]); + auto proj = store_partition->create_projection_info(launch_domain); + proj->set_reduction_op(target_.store->type()->find_reduction_operator(redop_.value())); + launcher.add_reduction(target_.store, std::move(proj)); + } + launcher.add_target_indirect(target_indirect_.store, create_projection_info(strategy, launch_domain, target_indirect_)); launcher.set_target_indirect_out_of_range(out_of_range_); diff --git a/src/core/operation/detail/scatter.h b/src/core/operation/detail/scatter.h index 15a625f81b..0209ed0e67 100644 --- a/src/core/operation/detail/scatter.h +++ b/src/core/operation/detail/scatter.h @@ -13,6 +13,7 @@ #pragma once #include +#include #include "core/data/detail/logical_store.h" #include "core/operation/detail/operation.h" @@ -28,7 +29,8 @@ class Scatter : public Operation { std::shared_ptr target_indirect, std::shared_ptr source, int64_t unique_id, - mapping::detail::Machine&& machine); + mapping::detail::Machine&& machine, + std::optional redop); public: void set_indirect_out_of_range(bool flag) { out_of_range_ = flag; } @@ -49,6 +51,7 @@ class Scatter : public Operation { StoreArg target_indirect_; StoreArg source_; std::unique_ptr constraint_; + std::optional redop_; }; } // namespace legate::detail diff --git a/src/core/operation/detail/scatter_gather.cc b/src/core/operation/detail/scatter_gather.cc index 00238d0cbb..9fb47efefe 100644 --- a/src/core/operation/detail/scatter_gather.cc +++ b/src/core/operation/detail/scatter_gather.cc @@ -25,13 +25,15 @@ ScatterGather::ScatterGather(std::shared_ptr target, std::shared_ptr source, std::shared_ptr source_indirect, int64_t unique_id, - mapping::detail::Machine&& machine) + mapping::detail::Machine&& machine, + std::optional redop) : Operation(unique_id, std::move(machine)), target_{target.get(), declare_partition()}, target_indirect_{target_indirect.get(), declare_partition()}, source_{source.get(), declare_partition()}, source_indirect_{source_indirect.get(), declare_partition()}, - constraint_(align(source_indirect_.variable, target_indirect_.variable)) + constraint_(align(source_indirect_.variable, target_indirect_.variable)), + redop_{redop} { record_partition(target_.variable, std::move(target)); record_partition(target_indirect_.variable, std::move(target_indirect)); @@ -86,7 +88,16 @@ void ScatterGather::launch(Strategy* p_strategy) launcher.add_input(source_.store, create_projection_info(strategy, launch_domain, source_)); launcher.add_source_indirect(source_indirect_.store, create_projection_info(strategy, launch_domain, source_indirect_)); - launcher.add_inout(target_.store, create_projection_info(strategy, launch_domain, target_)); + + if (!redop_) { + launcher.add_inout(target_.store, create_projection_info(strategy, launch_domain, target_)); + } else { + auto store_partition = target_.store->create_partition(strategy[target_.variable]); + auto proj = store_partition->create_projection_info(launch_domain); + proj->set_reduction_op(target_.store->type()->find_reduction_operator(redop_.value())); + launcher.add_reduction(target_.store, std::move(proj)); + } + launcher.add_target_indirect(target_indirect_.store, create_projection_info(strategy, launch_domain, target_indirect_)); launcher.set_target_indirect_out_of_range(target_indirect_out_of_range_); diff --git a/src/core/operation/detail/scatter_gather.h b/src/core/operation/detail/scatter_gather.h index ef9c9e5982..0ba1e2ae9a 100644 --- a/src/core/operation/detail/scatter_gather.h +++ b/src/core/operation/detail/scatter_gather.h @@ -13,6 +13,7 @@ #pragma once #include +#include #include "core/data/detail/logical_store.h" #include "core/operation/detail/operation.h" @@ -29,7 +30,8 @@ class ScatterGather : public Operation { std::shared_ptr source, std::shared_ptr source_indirect, int64_t unique_id, - mapping::detail::Machine&& machine); + mapping::detail::Machine&& machine, + std::optional redop); public: void set_source_indirect_out_of_range(bool flag); @@ -53,6 +55,7 @@ class ScatterGather : public Operation { StoreArg source_; StoreArg source_indirect_; std::unique_ptr constraint_; + std::optional redop_; }; } // namespace legate::detail diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index cafaedfe73..cbcad4610b 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -195,41 +195,48 @@ std::unique_ptr Runtime::create_task(Library* library, return std::unique_ptr(task); } -void Runtime::issue_copy(std::shared_ptr target, std::shared_ptr source) +void Runtime::issue_copy(std::shared_ptr target, + std::shared_ptr source, + std::optional redop) { auto machine = machine_manager_->get_machine(); submit(std::make_unique( - std::move(target), std::move(source), next_unique_id_++, std::move(machine))); + std::move(target), std::move(source), next_unique_id_++, std::move(machine), redop)); } void Runtime::issue_gather(std::shared_ptr target, std::shared_ptr source, - std::shared_ptr source_indirect) + std::shared_ptr source_indirect, + std::optional redop) { auto machine = machine_manager_->get_machine(); submit(std::make_unique(std::move(target), std::move(source), std::move(source_indirect), next_unique_id_++, - std::move(machine))); + std::move(machine), + redop)); } void Runtime::issue_scatter(std::shared_ptr target, std::shared_ptr target_indirect, - std::shared_ptr source) + std::shared_ptr source, + std::optional redop) { auto machine = machine_manager_->get_machine(); submit(std::make_unique(std::move(target), std::move(target_indirect), std::move(source), next_unique_id_++, - std::move(machine))); + std::move(machine), + redop)); } void Runtime::issue_scatter_gather(std::shared_ptr target, std::shared_ptr target_indirect, std::shared_ptr source, - std::shared_ptr source_indirect) + std::shared_ptr source_indirect, + std::optional redop) { auto machine = machine_manager_->get_machine(); submit(std::make_unique(std::move(target), @@ -237,7 +244,8 @@ void Runtime::issue_scatter_gather(std::shared_ptr target, std::move(source), std::move(source_indirect), next_unique_id_++, - std::move(machine))); + std::move(machine), + redop)); } void Runtime::issue_fill(std::shared_ptr lhs, std::shared_ptr value) diff --git a/src/core/runtime/detail/runtime.h b/src/core/runtime/detail/runtime.h index d638364c67..4e6c3d6005 100644 --- a/src/core/runtime/detail/runtime.h +++ b/src/core/runtime/detail/runtime.h @@ -71,17 +71,22 @@ class Runtime { std::unique_ptr create_task(Library* library, int64_t task_id, const Shape& launch_shape); - void issue_copy(std::shared_ptr target, std::shared_ptr source); + void issue_copy(std::shared_ptr target, + std::shared_ptr source, + std::optional redop); void issue_gather(std::shared_ptr target, std::shared_ptr source, - std::shared_ptr source_indirect); + std::shared_ptr source_indirect, + std::optional redop); void issue_scatter(std::shared_ptr target, std::shared_ptr target_indirect, - std::shared_ptr source); + std::shared_ptr source, + std::optional redop); void issue_scatter_gather(std::shared_ptr target, std::shared_ptr target_indirect, std::shared_ptr source, - std::shared_ptr source_indirect); + std::shared_ptr source_indirect, + std::optional redop); void issue_fill(std::shared_ptr lhs, std::shared_ptr value); void flush_scheduling_window(); void submit(std::unique_ptr op); diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 7582a2f1ec..c53b45655b 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -153,28 +153,72 @@ ManualTask Runtime::create_task(Library library, int64_t task_id, const Shape& l return ManualTask(impl_->create_task(library.impl(), task_id, launch_shape)); } -void Runtime::issue_copy(LogicalStore target, LogicalStore source) +void Runtime::issue_copy(LogicalStore target, + LogicalStore source, + std::optional redop) { - impl_->issue_copy(target.impl(), source.impl()); + auto op = redop ? std::make_optional(static_cast(redop.value())) : std::nullopt; + impl_->issue_copy(target.impl(), source.impl(), op); } -void Runtime::issue_gather(LogicalStore target, LogicalStore source, LogicalStore source_indirect) +void Runtime::issue_copy(LogicalStore target, LogicalStore source, std::optional redop) { - impl_->issue_gather(target.impl(), source.impl(), source_indirect.impl()); + impl_->issue_copy(target.impl(), source.impl(), redop); } -void Runtime::issue_scatter(LogicalStore target, LogicalStore target_indirect, LogicalStore source) +void Runtime::issue_gather(LogicalStore target, + LogicalStore source, + LogicalStore source_indirect, + std::optional redop) { - impl_->issue_scatter(target.impl(), target_indirect.impl(), source.impl()); + auto op = redop ? std::make_optional(static_cast(redop.value())) : std::nullopt; + impl_->issue_gather(target.impl(), source.impl(), source_indirect.impl(), op); +} + +void Runtime::issue_gather(LogicalStore target, + LogicalStore source, + LogicalStore source_indirect, + std::optional redop) +{ + impl_->issue_gather(target.impl(), source.impl(), source_indirect.impl(), redop); +} + +void Runtime::issue_scatter(LogicalStore target, + LogicalStore target_indirect, + LogicalStore source, + std::optional redop) +{ + auto op = redop ? std::make_optional(static_cast(redop.value())) : std::nullopt; + impl_->issue_scatter(target.impl(), target_indirect.impl(), source.impl(), op); +} + +void Runtime::issue_scatter(LogicalStore target, + LogicalStore target_indirect, + LogicalStore source, + std::optional redop) +{ + impl_->issue_scatter(target.impl(), target_indirect.impl(), source.impl(), redop); +} + +void Runtime::issue_scatter_gather(LogicalStore target, + LogicalStore target_indirect, + LogicalStore source, + LogicalStore source_indirect, + std::optional redop) +{ + auto op = redop ? std::make_optional(static_cast(redop.value())) : std::nullopt; + impl_->issue_scatter_gather( + target.impl(), target_indirect.impl(), source.impl(), source_indirect.impl(), op); } void Runtime::issue_scatter_gather(LogicalStore target, LogicalStore target_indirect, LogicalStore source, - LogicalStore source_indirect) + LogicalStore source_indirect, + std::optional redop) { impl_->issue_scatter_gather( - target.impl(), target_indirect.impl(), source.impl(), source_indirect.impl()); + target.impl(), target_indirect.impl(), source.impl(), source_indirect.impl(), redop); } void Runtime::issue_fill(LogicalStore lhs, LogicalStore value) diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index f61ff7d2df..78d789bef7 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -23,6 +23,7 @@ #include "core/runtime/library.h" #include "core/runtime/resource.h" #include "core/task/exception.h" +#include "core/type/type_info.h" #include "core/utilities/typedefs.h" /** @defgroup runtime Runtime and library contexts @@ -182,8 +183,44 @@ class Runtime { * * @param target Copy target * @param source Copy source + * @param redop ID of the reduction operator to use (optional). The store's type must support the + * operator. + * + * @throw std::invalid_argument If the store's type doesn't support the reduction operator + */ + void issue_copy(LogicalStore target, + LogicalStore source, + std::optional redop = std::nullopt); + /** + * @brief Issues a copy between stores. + * + * The source and target stores must have the same shape. + * + * @param target Copy target + * @param source Copy source + * @param redop ID of the reduction operator to use (optional). The store's type must support the + * operator. + * + * @throw std::invalid_argument If the store's type doesn't support the reduction operator + */ + void issue_copy(LogicalStore target, LogicalStore source, std::optional redop); + /** + * @brief Issues a gather copy between stores. + * + * The indirection store and the target store must have the same shape. + * + * @param target Copy target + * @param source Copy source + * @param source_indirect Store for source indirection + * @param redop ID of the reduction operator to use (optional). The store's type must support the + * operator. + * + * @throw std::invalid_argument If the store's type doesn't support the reduction operator */ - void issue_copy(LogicalStore target, LogicalStore source); + void issue_gather(LogicalStore target, + LogicalStore source, + LogicalStore source_indirect, + std::optional redop = std::nullopt); /** * @brief Issues a gather copy between stores. * @@ -192,8 +229,32 @@ class Runtime { * @param target Copy target * @param source Copy source * @param source_indirect Store for source indirection + * @param redop ID of the reduction operator to use (optional). The store's type must support the + * operator. + * + * @throw std::invalid_argument If the store's type doesn't support the reduction operator + */ + void issue_gather(LogicalStore target, + LogicalStore source, + LogicalStore source_indirect, + std::optional redop); + /** + * @brief Issues a scatter copy between stores. + * + * The indirection store and the source store must have the same shape. + * + * @param target Copy target + * @param target_indirect Store for target indirection + * @param source Copy source + * @param redop ID of the reduction operator to use (optional). The store's type must support the + * operator. + * + * @throw std::invalid_argument If the store's type doesn't support the reduction operator */ - void issue_gather(LogicalStore target, LogicalStore source, LogicalStore source_indirect); + void issue_scatter(LogicalStore target, + LogicalStore target_indirect, + LogicalStore source, + std::optional redop = std::nullopt); /** * @brief Issues a scatter copy between stores. * @@ -202,8 +263,15 @@ class Runtime { * @param target Copy target * @param target_indirect Store for target indirection * @param source Copy source + * @param redop ID of the reduction operator to use (optional). The store's type must support the + * operator. + * + * @throw std::invalid_argument If the store's type doesn't support the reduction operator */ - void issue_scatter(LogicalStore target, LogicalStore target_indirect, LogicalStore source); + void issue_scatter(LogicalStore target, + LogicalStore target_indirect, + LogicalStore source, + std::optional redop); /** * @brief Issues a scatter-gather copy between stores. * @@ -213,12 +281,35 @@ class Runtime { * @param target_indirect Store for target indirection * @param source Copy source * @param source_indirect Store for source indirection + * @param redop ID of the reduction operator to use (optional). The store's type must support the + * operator. + * + * @throw std::invalid_argument If the store's type doesn't support the reduction operator */ void issue_scatter_gather(LogicalStore target, LogicalStore target_indirect, LogicalStore source, - LogicalStore source_indirect); - + LogicalStore source_indirect, + std::optional redop = std::nullopt); + /** + * @brief Issues a scatter-gather copy between stores. + * + * The indirection stores must have the same shape. + * + * @param target Copy target + * @param target_indirect Store for target indirection + * @param source Copy source + * @param source_indirect Store for source indirection + * @param redop ID of the reduction operator to use (optional). The store's type must support the + * operator. + * + * @throw std::invalid_argument If the store's type doesn't support the reduction operator + */ + void issue_scatter_gather(LogicalStore target, + LogicalStore target_indirect, + LogicalStore source, + LogicalStore source_indirect, + std::optional redop); /** * @brief Fills a given store with a constant * diff --git a/tests/cpp/integration/copy_normal.cc b/tests/cpp/integration/copy_normal.cc index 60de59ed1e..4ecdfe9cbe 100644 --- a/tests/cpp/integration/copy_normal.cc +++ b/tests/cpp/integration/copy_normal.cc @@ -12,6 +12,7 @@ #include +#include "core/type/type_info.h" #include "legate.h" #include "copy_util.inl" @@ -22,11 +23,11 @@ static const char* library_name = "test_copy_normal"; static constexpr int32_t TEST_MAX_DIM = 3; -constexpr int32_t CHECK_TASK = FILL_TASK + TEST_MAX_DIM; +constexpr int32_t CHECK_COPY_TASK = FILL_TASK + TEST_MAX_DIM; template -struct CheckTask : public legate::LegateTask> { - struct CheckTaskBody { +struct CheckCopyTask : public legate::LegateTask> { + struct CheckCopyTaskBody { template void operator()(legate::Store& source, legate::Store& target, legate::Rect& shape) { @@ -39,7 +40,7 @@ struct CheckTask : public legate::LegateTask> { } }; - static const int32_t TASK_ID = CHECK_TASK + DIM; + static const int32_t TASK_ID = CHECK_COPY_TASK + DIM; static void cpu_variant(legate::TaskContext& context) { auto& source = context.inputs().at(0); @@ -48,7 +49,51 @@ struct CheckTask : public legate::LegateTask> { if (shape.empty()) return; - type_dispatch(source.type().code(), CheckTaskBody{}, source, target, shape); + type_dispatch(source.type().code(), CheckCopyTaskBody{}, source, target, shape); + } +}; + +constexpr int32_t CHECK_COPY_REDUCTION_TASK = CHECK_COPY_TASK + TEST_MAX_DIM; + +template +struct CheckCopyReductionTask : public legate::LegateTask> { + struct CheckCopyReductionTaskBody { + template ::value, int> = 0> + void operator()(legate::Store& source, + legate::Store& target, + legate::Scalar& seed, + legate::Rect& shape) + { + using VAL = legate::legate_type_of; + auto src = source.read_accessor(shape); + auto tgt = target.read_accessor(shape); + legate::PointInRectIterator it(shape); + size_t i = 1; + for (legate::PointInRectIterator it(shape); it.valid(); ++it, ++i) { + EXPECT_EQ(src[*it] + i * seed.value(), tgt[*it]); + } + } + template ::value, int> = 0> + void operator()(legate::Store& source, + legate::Store& target, + legate::Scalar& seed, + legate::Rect& shape) + { + assert(false); + } + }; + + static const int32_t TASK_ID = CHECK_COPY_REDUCTION_TASK + DIM; + static void cpu_variant(legate::TaskContext& context) + { + auto& source = context.inputs().at(0); + auto& target = context.inputs().at(1); + auto& seed = context.scalars().at(0); + auto shape = target.shape(); + + if (shape.empty()) return; + + type_dispatch(target.type().code(), CheckCopyReductionTaskBody{}, source, target, seed, shape); } }; @@ -59,18 +104,21 @@ void register_tasks() FillTask<1>::register_variants(library); FillTask<2>::register_variants(library); FillTask<3>::register_variants(library); - CheckTask<1>::register_variants(library); - CheckTask<2>::register_variants(library); - CheckTask<3>::register_variants(library); + CheckCopyTask<1>::register_variants(library); + CheckCopyTask<2>::register_variants(library); + CheckCopyTask<3>::register_variants(library); + CheckCopyReductionTask<1>::register_variants(library); + CheckCopyReductionTask<2>::register_variants(library); + CheckCopyReductionTask<3>::register_variants(library); } -void check_output(legate::Library library, - const legate::LogicalStore& src, - const legate::LogicalStore& tgt) +void check_copy_output(legate::Library library, + const legate::LogicalStore& src, + const legate::LogicalStore& tgt) { auto runtime = legate::Runtime::get_runtime(); auto machine = runtime->get_machine(); - auto task = runtime->create_task(library, CHECK_TASK + tgt.dim()); + auto task = runtime->create_task(library, CHECK_COPY_TASK + tgt.dim()); auto src_part = task.declare_partition(); auto tgt_part = task.declare_partition(); @@ -82,12 +130,39 @@ void check_output(legate::Library library, runtime->submit(std::move(task)); } +void check_copy_reduction_output(legate::Library library, + const legate::LogicalStore& src, + const legate::LogicalStore& tgt, + const legate::Scalar& seed) +{ + auto runtime = legate::Runtime::get_runtime(); + auto machine = runtime->get_machine(); + auto task = runtime->create_task(library, CHECK_COPY_REDUCTION_TASK + tgt.dim()); + + auto src_part = task.declare_partition(); + auto tgt_part = task.declare_partition(); + + task.add_input(src, src_part); + task.add_input(tgt, tgt_part); + task.add_constraint(legate::align(src_part, tgt_part)); + task.add_scalar_arg(seed); + + runtime->submit(std::move(task)); +} + struct NormalCopySpec { std::vector shape; legate::Type type; legate::Scalar seed; }; +struct NormalCopyReductionSpec { + std::vector shape; + legate::Type type; + legate::Scalar seed; + legate::ReductionOpKind redop; +}; + void test_normal_copy(const NormalCopySpec& spec) { auto runtime = legate::Runtime::get_runtime(); @@ -102,17 +177,41 @@ void test_normal_copy(const NormalCopySpec& spec) runtime->issue_copy(output, input); // check the result of copy - check_output(library, input, output); + check_copy_output(library, input, output); +} + +void test_normal_copy_reduction(const NormalCopyReductionSpec& spec) +{ + auto runtime = legate::Runtime::get_runtime(); + auto library = runtime->find_library(library_name); + + auto& [shape, type, seed, redop] = spec; + + auto input = runtime->create_store(shape, type); + auto output = runtime->create_store(shape, type); + + fill_input(library, input, seed); + fill_input(library, output, seed); + runtime->issue_copy(output, input, redop); + + // check the result of copy reduction + check_copy_reduction_output(library, input, output, seed); } TEST(Copy, Single) { legate::Core::perform_registration(); - // For some reason, clang-format gets tripped over by singleton initialization lists, - // so factor out the definition here - std::vector shape1d{9}; test_normal_copy({{4, 7}, legate::int64(), legate::Scalar(int64_t(12))}); test_normal_copy({{1000, 100}, legate::uint32(), legate::Scalar(uint32_t(3))}); } +TEST(Copy, SingleReduction) +{ + legate::Core::perform_registration(); + test_normal_copy_reduction( + {{4, 7}, legate::int64(), legate::Scalar(int64_t(12)), legate::ReductionOpKind::ADD}); + test_normal_copy_reduction( + {{1000, 100}, legate::uint32(), legate::Scalar(uint32_t(3)), legate::ReductionOpKind::ADD}); +} + } // namespace copy_normal From b70ddd83446b7e8f419d4c4b43874281f6a560de Mon Sep 17 00:00:00 2001 From: Irina Demeshko Date: Fri, 28 Jul 2023 16:27:30 -0700 Subject: [PATCH 0181/1425] Tree reduce implementation * moving call to projection functor up to compute_projection * fixing CI * generalizing Projection Functor logic * changing RadixFunctor to get only ndims as an input * addressing comments * code clean-up * removing unnecessary change * adding tree_reduce unique test * added more tests * small bugfix * using local input and output in tree_reduce * adding logic for nested reductions * fixing partitioning logic for tree_reduce * fixing some run-time errors * fixing compile-time errors after the merge * formatting * fixing minor bugs * more work on tree reduce * more work on tree reduce * created basic Reduce logic Co-authored-by: Wonchan Lee See merge request legate/legate.core.internal!82 --- legate_core_cpp.cmake | 1 + src/core/data/detail/logical_store.cc | 17 +- src/core/data/detail/logical_store.h | 7 +- src/core/operation/detail/reduce.cc | 127 +++++++++++++++ src/core/operation/detail/reduce.h | 57 +++++++ src/core/runtime/detail/projection.cc | 17 +- src/core/runtime/detail/projection.h | 13 +- src/core/runtime/detail/runtime.cc | 17 ++ src/core/runtime/detail/runtime.h | 5 + src/core/runtime/runtime.cc | 10 ++ src/core/runtime/runtime.h | 8 + tests/cpp/integration/tree_reduce.cc | 171 ++++++++++++++++++++ tests/cpp/integration/tree_reduce_unique.cc | 140 ++++++++++++++++ 13 files changed, 582 insertions(+), 8 deletions(-) create mode 100644 src/core/operation/detail/reduce.cc create mode 100644 src/core/operation/detail/reduce.h create mode 100644 tests/cpp/integration/tree_reduce.cc create mode 100644 tests/cpp/integration/tree_reduce_unique.cc diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index 30790c6b21..21836b6558 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -229,6 +229,7 @@ list(APPEND legate_core_SOURCES src/core/operation/detail/launcher_arg.cc src/core/operation/detail/operation.cc src/core/operation/detail/projection.cc + src/core/operation/detail/reduce.cc src/core/operation/detail/req_analyzer.cc src/core/operation/detail/scatter.cc src/core/operation/detail/scatter_gather.cc diff --git a/src/core/data/detail/logical_store.cc b/src/core/data/detail/logical_store.cc index 0b3af76da3..f61efbd48e 100644 --- a/src/core/data/detail/logical_store.cc +++ b/src/core/data/detail/logical_store.cc @@ -548,8 +548,17 @@ Restrictions LogicalStore::compute_restrictions() const return transform_->convert(storage_->compute_restrictions()); } -Legion::ProjectionID LogicalStore::compute_projection(int32_t launch_ndim) const +Legion::ProjectionID LogicalStore::compute_projection( + int32_t launch_ndim, std::optional proj_fn) const { + auto ndim = dim(); + + if (proj_fn != nullptr) { + assert(!transformed()); + auto point = proj_fn.value()(proj::create_symbolic_point(launch_ndim)); + return Runtime::get_runtime()->get_projection(launch_ndim, point); + } + if (transform_->identity()) { if (launch_ndim != dim()) return Runtime::get_runtime()->get_delinearizing_projection(); @@ -557,7 +566,6 @@ Legion::ProjectionID LogicalStore::compute_projection(int32_t launch_ndim) const return 0; } - auto ndim = dim(); auto point = transform_->invert(proj::create_symbolic_point(ndim)); // TODO: We can't currently mix affine projections with delinearizing projections #ifdef DEBUG_LEGATE @@ -678,8 +686,9 @@ LogicalStorePartition::LogicalStorePartition(std::shared_ptr partitio { } +// FIXME pass projection functor std::unique_ptr LogicalStorePartition::create_projection_info( - const Domain* launch_domain) + const Domain* launch_domain, std::optional proj_fn) { if (nullptr == launch_domain || store_->has_scalar_storage()) return std::make_unique(); @@ -687,7 +696,7 @@ std::unique_ptr LogicalStorePartition::create_projection_info( // We're about to create a legion partition for this store, so the store should have its region // created. auto legion_partition = storage_partition_->get_legion_partition(); - auto proj_id = store_->compute_projection(launch_domain->dim); + auto proj_id = store_->compute_projection(launch_domain->dim, proj_fn); return std::make_unique(legion_partition, proj_id); } diff --git a/src/core/data/detail/logical_store.h b/src/core/data/detail/logical_store.h index 0bc08f463e..52147a2a50 100644 --- a/src/core/data/detail/logical_store.h +++ b/src/core/data/detail/logical_store.h @@ -21,6 +21,7 @@ #include "core/mapping/detail/machine.h" #include "core/partitioning/partition.h" #include "core/partitioning/restriction.h" +#include "core/runtime/detail/projection.h" #include "core/utilities/detail/buffer_builder.h" namespace legate::detail { @@ -212,7 +213,8 @@ class LogicalStore : public std::enable_shared_from_this { public: std::shared_ptr create_partition( std::shared_ptr partition, std::optional complete = std::nullopt); - Legion::ProjectionID compute_projection(int32_t launch_ndim) const; + Legion::ProjectionID compute_projection( + int32_t launch_ndim, std::optional proj_fn = nullptr) const; public: void pack(BufferBuilder& buffer) const; @@ -242,7 +244,8 @@ class LogicalStorePartition : public std::enable_shared_from_this partition() const { return partition_; } std::shared_ptr storage_partition() const { return storage_partition_; } std::shared_ptr store() const { return store_; } - std::unique_ptr create_projection_info(const Domain* launch_domain); + std::unique_ptr create_projection_info( + const Domain* launch_domain, std::optional proj_fn = nullptr); bool is_disjoint_for(const Domain* launch_domain) const; private: diff --git a/src/core/operation/detail/reduce.cc b/src/core/operation/detail/reduce.cc new file mode 100644 index 0000000000..a494ce16c7 --- /dev/null +++ b/src/core/operation/detail/reduce.cc @@ -0,0 +1,127 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "core/operation/detail/reduce.h" + +#include "core/data/detail/logical_store.h" +#include "core/operation/detail/projection.h" +#include "core/operation/detail/task_launcher.h" +#include "core/partitioning/detail/constraint.h" +#include "core/partitioning/detail/constraint_solver.h" +#include "core/partitioning/detail/partitioner.h" +#include "core/partitioning/partition.h" +#include "core/runtime/detail/library.h" +#include "core/runtime/detail/runtime.h" + +namespace legate::detail { + +Reduce::Reduce(const Library* library, + std::shared_ptr store, + std::shared_ptr out_store, + int64_t task_id, + int64_t unique_id, + int64_t radix, + mapping::detail::Machine&& machine) + : Operation(unique_id, std::move(machine)), + radix_(radix), + library_(library), + task_id_(task_id), + input_(std::move(store)), + output_(std::move(out_store)) +{ + input_part_ = find_or_declare_partition(input_); + output_part_ = declare_partition(); + record_partition(input_part_, input_); + record_partition(output_part_, output_); +} + +void Reduce::launch(Strategy* p_strategy) +{ + auto& strategy = *p_strategy; + auto launch_domain = *(strategy.launch_domain(this)); + auto n_tasks = launch_domain.get_volume(); + + auto input_part = strategy[input_part_]; + auto input_partition = input_->create_partition(input_part); + + // generating projection functions to use in tree_reduction task + std::vector proj_fns; + if (n_tasks > 1) { + for (size_t i = 0; i < radix_; i++) proj_fns.push_back(proj::RadixProjectionFunctor(radix_, i)); + } + + std::shared_ptr new_output; + bool done = false; + while (!done) { + detail::TaskLauncher launcher( + library_, machine_, provenance_, task_id_, LEGATE_CORE_TREE_REDUCE_TAG); + if (n_tasks > 1) { + // if there are more than 1 sub-task, we add several slices of the input + // for each sub-task + for (auto& proj_fn : proj_fns) { + launcher.add_input(input_.get(), + input_partition->create_projection_info(&launch_domain, proj_fn)); + } + } else { + // otherwise we just add an entire input region to the task + auto proj = input_partition->create_projection_info(&launch_domain); + launcher.add_input(input_.get(), std::move(proj)); + } + + // calculating #of sub-tasks in the reduction task + n_tasks = (n_tasks + radix_ - 1) / radix_; + done = (n_tasks == 1); + + // adding output region + auto runtime = detail::Runtime::get_runtime(); + + auto field_space = runtime->create_field_space(); + auto field_size = input_->type()->size(); + auto field_id = runtime->allocate_field(field_space, field_size); + // if this is not the last iteration of the while loop, we generate + // a new output region + if (n_tasks != 1) { + new_output = runtime->create_store(input_->type(), 1); + launcher.add_unbound_output(new_output.get(), field_space, field_id); + } else { + launcher.add_unbound_output(output_.get(), field_space, field_id); + } + + launch_domain = Domain(DomainPoint(0), DomainPoint(n_tasks - 1)); + auto result = launcher.execute(launch_domain); + + if (n_tasks != 1) { + Weighted weighted(result, launch_domain); + new_output->set_key_partition(machine_, &weighted); + auto output_partition = + new_output->create_partition(std::make_shared(std::move(weighted))); + input_ = new_output; + input_partition = output_partition; + } + } +} + +void Reduce::validate() {} + +void Reduce::add_to_solver(detail::ConstraintSolver& solver) +{ + solver.add_partition_symbol(output_part_, true); + solver.add_partition_symbol(input_part_); +} + +std::string Reduce::to_string() const { return "Reduce:" + std::to_string(unique_id_); } + +} // namespace legate::detail diff --git a/src/core/operation/detail/reduce.h b/src/core/operation/detail/reduce.h new file mode 100644 index 0000000000..528c1aa02d --- /dev/null +++ b/src/core/operation/detail/reduce.h @@ -0,0 +1,57 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +#include "core/operation/detail/operation.h" + +#include "core/mapping/detail/machine.h" + +namespace legate::detail { + +class Reduce : public Operation { + private: + friend class Runtime; + Reduce(const Library* library, + std::shared_ptr store, + std::shared_ptr out_store, + int64_t task_id, + int64_t unique_id, + int64_t radix, + mapping::detail::Machine&& machine); + + public: + void launch(Strategy*) override; + + public: + void validate() override; + void add_to_solver(ConstraintSolver& solver) override; + + public: + std::string to_string() const override; + + private: + int64_t radix_; + const Library* library_; + int64_t task_id_; + std::shared_ptr input_; + std::shared_ptr output_; + const Variable* input_part_; + const Variable* output_part_; +}; + +} // namespace legate::detail diff --git a/src/core/runtime/detail/projection.cc b/src/core/runtime/detail/projection.cc index 4b8d56eaeb..aa70153eb6 100644 --- a/src/core/runtime/detail/projection.cc +++ b/src/core/runtime/detail/projection.cc @@ -83,11 +83,26 @@ std::ostream& operator<<(std::ostream& out, const SymbolicExpr& expr) return out; } +RadixProjectionFunctor::RadixProjectionFunctor(int32_t radix, int32_t offset) + : radix_(radix), offset_(offset) +{ +} + +SymbolicPoint RadixProjectionFunctor::operator()(const SymbolicPoint& in_p) const +{ + auto ndim = in_p.size(); + std::vector exprs; + exprs.resize(ndim); + for (int32_t dim = 0; dim < ndim; ++dim) { exprs[dim] = in_p[dim] * radix_ + offset_; } + return SymbolicPoint(std::move(exprs)); +} + SymbolicPoint create_symbolic_point(int32_t ndim) { std::vector exprs; exprs.resize(ndim); - for (int32_t dim = 0; dim < ndim; ++dim) exprs[dim] = proj::SymbolicExpr(dim); + + for (int32_t dim = 0; dim < ndim; ++dim) { exprs[dim] = proj::SymbolicExpr(dim); } return SymbolicPoint(std::move(exprs)); } diff --git a/src/core/runtime/detail/projection.h b/src/core/runtime/detail/projection.h index a1040213ba..d89de23870 100644 --- a/src/core/runtime/detail/projection.h +++ b/src/core/runtime/detail/projection.h @@ -11,6 +11,7 @@ */ #pragma once +#include #include "legion.h" @@ -48,7 +49,17 @@ class SymbolicExpr { std::ostream& operator<<(std::ostream& out, const SymbolicExpr& expr); using SymbolicPoint = tuple; -using SymbolicFunctor = SymbolicPoint (*)(const SymbolicPoint&); +using SymbolicFunctor = std::function; +; + +struct RadixProjectionFunctor { + RadixProjectionFunctor(int32_t radix, int32_t offset); + + SymbolicPoint operator()(const SymbolicPoint& exprs) const; + + private: + int32_t offset_, radix_; +}; SymbolicPoint create_symbolic_point(int32_t ndim); diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index cbcad4610b..bb1cba5acc 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -21,6 +21,7 @@ #include "core/operation/detail/copy.h" #include "core/operation/detail/fill.h" #include "core/operation/detail/gather.h" +#include "core/operation/detail/reduce.h" #include "core/operation/detail/scatter.h" #include "core/operation/detail/scatter_gather.h" #include "core/operation/detail/task.h" @@ -255,6 +256,22 @@ void Runtime::issue_fill(std::shared_ptr lhs, std::shared_ptr store, + std::shared_ptr out_store, + int64_t radix) +{ + auto machine = machine_manager_->get_machine(); + submit(std::unique_ptr(new Reduce(library, + std::move(store), + std::move(out_store), + task_id, + next_unique_id_++, + radix, + std::move(machine)))); +} + void Runtime::flush_scheduling_window() { if (operations_.size() == 0) return; diff --git a/src/core/runtime/detail/runtime.h b/src/core/runtime/detail/runtime.h index 4e6c3d6005..fba1cd029a 100644 --- a/src/core/runtime/detail/runtime.h +++ b/src/core/runtime/detail/runtime.h @@ -88,6 +88,11 @@ class Runtime { std::shared_ptr source_indirect, std::optional redop); void issue_fill(std::shared_ptr lhs, std::shared_ptr value); + void tree_reduce(const Library* library, + int64_t task_id, + std::shared_ptr store, + std::shared_ptr out_store, + int64_t radix); void flush_scheduling_window(); void submit(std::unique_ptr op); diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index c53b45655b..d29102a536 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -231,6 +231,16 @@ void Runtime::issue_fill(LogicalStore lhs, const Scalar& value) issue_fill(std::move(lhs), create_store(value)); } +LogicalStore Runtime::tree_reduce(Library library, + int64_t task_id, + LogicalStore store, + int64_t radix) +{ + auto out_store = create_store(store.type(), 1); + impl_->tree_reduce(library.impl(), task_id, store.impl(), out_store.impl(), radix); + return out_store; +} + void Runtime::submit(AutoTask&& task) { impl_->submit(std::move(task.impl_)); } void Runtime::submit(ManualTask&& task) { impl_->submit(std::move(task.impl_)); } diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 78d789bef7..34ecd71485 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -324,6 +324,14 @@ class Runtime { * @param value Value to fill the store with */ void issue_fill(LogicalStore lhs, const Scalar& value); + /** + * @brief tree_reduce given store and task id + * + * @param task_id reduction task ID + * @param store Logical store to reduce + */ + LogicalStore tree_reduce(Library library, int64_t task_id, LogicalStore store, int64_t radix = 4); + /** * @brief Submits an AutoTask for execution * diff --git a/tests/cpp/integration/tree_reduce.cc b/tests/cpp/integration/tree_reduce.cc new file mode 100644 index 0000000000..ad1653d616 --- /dev/null +++ b/tests/cpp/integration/tree_reduce.cc @@ -0,0 +1,171 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "legate.h" + +namespace tree_reduce { + +static const char* library_name = "test_tree_reduce"; + +static const size_t TILE_SIZE = 10; + +enum TaskIDs { + TASK_PRODUCE_NORMAL = 1, + TASK_PRODUCE_UNBOUND, + TASK_REDUCE_NORMAL, + TASK_REDUCE_UNBOUND, +}; + +struct ProduceNormalTask : public legate::LegateTask { + static const int32_t TASK_ID = TASK_PRODUCE_NORMAL; + static void cpu_variant(legate::TaskContext& context) {} +}; + +struct ProduceUnboundTask : public legate::LegateTask { + static const int32_t TASK_ID = TASK_PRODUCE_UNBOUND; + static void cpu_variant(legate::TaskContext& context) + { + auto& output = context.outputs().at(0); + auto size = context.get_task_index()[0] + 1; + auto buffer = output.create_output_buffer(legate::Point<1>(size), true /*bind*/); + for (int64_t idx = 0; idx < size; ++idx) buffer[idx] = size; + } +}; + +struct ReduceNormalTask : public legate::LegateTask { + static const int32_t TASK_ID = TASK_REDUCE_NORMAL; + static void cpu_variant(legate::TaskContext& context) + { + auto& inputs = context.inputs(); + auto& output = context.outputs().at(0); + for (auto& input : inputs) { + auto shape = input.shape<1>(); + EXPECT_TRUE(shape.empty() || shape.volume() == TILE_SIZE); + } + output.create_output_buffer(legate::Point<1>(0), true); + } +}; + +struct ReduceUnboundTask : public legate::LegateTask { + static const int32_t TASK_ID = TASK_REDUCE_UNBOUND; + static void cpu_variant(legate::TaskContext& context) + { + auto& inputs = context.inputs(); + auto& output = context.outputs().at(0); + uint32_t expected = 1; + for (auto& input : inputs) { + auto shape = input.shape<1>(); + ASSERT_EQ(shape.volume(), expected); + ++expected; + } + output.create_output_buffer(legate::Point<1>(0), true); + } +}; + +void register_tasks() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + ProduceNormalTask::register_variants(context); + ProduceUnboundTask::register_variants(context); + ReduceNormalTask::register_variants(context); + ReduceUnboundTask::register_variants(context); +} + +TEST(Integration, TreeReduceNormal) +{ + legate::Core::perform_registration(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + size_t num_tasks = 3; + size_t tile_size = TILE_SIZE; + + auto store = runtime->create_store({num_tasks * tile_size}, legate::int64()); + + auto task = runtime->create_task(context, TASK_PRODUCE_NORMAL, {num_tasks}); + auto part = store.partition_by_tiling({tile_size}); + task.add_output(part); + runtime->submit(std::move(task)); + + auto result = runtime->tree_reduce(context, TASK_REDUCE_NORMAL, store, 4); + + EXPECT_FALSE(result.unbound()); +} + +TEST(Integration, TreeReduceNormalTwoSteps) +{ + legate::Core::perform_registration(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + size_t num_tasks = 6; + size_t tile_size = TILE_SIZE; + + auto store = runtime->create_store({num_tasks * tile_size}, legate::int64()); + + auto task = runtime->create_task(context, TASK_PRODUCE_NORMAL, {num_tasks}); + auto part = store.partition_by_tiling({tile_size}); + task.add_output(part); + runtime->submit(std::move(task)); + + auto result = runtime->tree_reduce(context, TASK_REDUCE_NORMAL, store, 4); + + EXPECT_FALSE(result.unbound()); +} + +TEST(Integration, TreeReduceUnboud) +{ + legate::Core::perform_registration(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + // unbound store + auto store = runtime->create_store(legate::int64()); + size_t num_tasks = 4; + + auto task = runtime->create_task(context, TASK_PRODUCE_UNBOUND, {num_tasks}); + task.add_output(store); + runtime->submit(std::move(task)); + + auto result = runtime->tree_reduce(context, TASK_REDUCE_UNBOUND, store, num_tasks); + EXPECT_FALSE(result.unbound()); +} + +TEST(Integration, TreeReduceSingleProc) +{ + legate::Core::perform_registration(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + // unbound store + auto store = runtime->create_store(legate::int64()); + + auto task = runtime->create_task(context, TASK_PRODUCE_UNBOUND, {1}); + task.add_output(store); + runtime->submit(std::move(task)); + + auto result = runtime->tree_reduce(context, TASK_REDUCE_UNBOUND, store, 4); + EXPECT_FALSE(result.unbound()); +} + +} // namespace tree_reduce diff --git a/tests/cpp/integration/tree_reduce_unique.cc b/tests/cpp/integration/tree_reduce_unique.cc new file mode 100644 index 0000000000..ed44a73700 --- /dev/null +++ b/tests/cpp/integration/tree_reduce_unique.cc @@ -0,0 +1,140 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "legate.h" + +namespace tree_reduce_unique { + +static const char* library_name = "test_tree_reduce_unique"; + +static const size_t TILE_SIZE = 10; + +enum TaskIDs { TASK_FILL = 1, TASK_UNIQUE, TASK_UNIQUE_REDUCE, TASK_CHECK }; + +struct FillTask : public legate::LegateTask { + static const int32_t TASK_ID = TASK_FILL; + static void cpu_variant(legate::TaskContext& context) + { + auto& output = context.outputs().at(0); + auto rect = output.shape<1>(); + auto volume = rect.volume(); + auto out = output.write_accessor(rect); + for (size_t idx = 0; idx < volume; ++idx) { out[idx] = idx / 2; } + } +}; + +struct UniqueTask : public legate::LegateTask { + static const int32_t TASK_ID = TASK_UNIQUE; + static void cpu_variant(legate::TaskContext& context) + { + auto& input = context.inputs().at(0); + auto& output = context.outputs().at(0); + auto rect = input.shape<1>(); + auto volume = rect.volume(); + auto in = input.read_accessor(rect); + std::set dedup_set; + for (size_t idx = 0; idx < volume; ++idx) dedup_set.insert(in[idx]); + + auto result = output.create_output_buffer(dedup_set.size(), true); + size_t pos = 0; + for (auto e : dedup_set) result[pos++] = e; + } +}; + +struct UniqueReduceTask : public legate::LegateTask { + static const int32_t TASK_ID = TASK_UNIQUE_REDUCE; + static void cpu_variant(legate::TaskContext& context) + { + auto& output = context.outputs().at(0); + std::vector, Legion::Rect<1>>> inputs; + for (auto& input_arr : context.inputs()) { + auto shape = input_arr.shape<1>(); + auto acc = input_arr.read_accessor(shape); + inputs.push_back(std::make_pair(acc, shape)); + } + std::set dedup_set; + for (auto& pair : inputs) { + auto& input = pair.first; + auto& shape = pair.second; + for (Legion::coord_t idx = shape.lo[0]; idx <= shape.hi[0]; ++idx) + dedup_set.insert(input[idx]); + } + + size_t size = dedup_set.size(); + size_t pos = 0; + auto result = output.create_output_buffer(Legion::Point<1>(size), true); + for (auto e : dedup_set) result[pos++] = e; + } +}; + +struct CheckTask : public legate::LegateTask { + static const int32_t TASK_ID = TASK_CHECK; + static void cpu_variant(legate::TaskContext& context) + { + auto& input = context.inputs().at(0); + auto rect = input.shape<1>(); + auto volume = rect.volume(); + auto in = input.read_accessor(rect); + ASSERT_EQ(volume, TILE_SIZE / 2); + for (size_t idx = 0; idx < volume; ++idx) { ASSERT_EQ(in[idx], idx); } + } +}; + +void register_tasks() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + FillTask::register_variants(context); + UniqueTask::register_variants(context); + UniqueReduceTask::register_variants(context); + CheckTask::register_variants(context); +} + +TEST(Integration, TreeReduceUnique) +{ + legate::Core::perform_registration(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + size_t num_tasks = 6; + size_t tile_size = TILE_SIZE; + + auto store = runtime->create_store({num_tasks * tile_size}, legate::int64()); + + auto task_fill = runtime->create_task(context, TASK_FILL, {num_tasks}); + auto part = store.partition_by_tiling({tile_size}); + task_fill.add_output(part); + runtime->submit(std::move(task_fill)); + + auto task_unique = runtime->create_task(context, TASK_UNIQUE, {num_tasks}); + auto intermediate_result = runtime->create_store(legate::int64(), 1); + task_unique.add_input(part); + task_unique.add_output(intermediate_result); + runtime->submit(std::move(task_unique)); + + auto result = runtime->tree_reduce(context, TASK_UNIQUE_REDUCE, intermediate_result, 4); + + EXPECT_FALSE(result.unbound()); + + auto task_check = runtime->create_task(context, TASK_CHECK, {1}); + task_check.add_input(result); + runtime->submit(std::move(task_check)); +} + +} // namespace tree_reduce_unique From f844cf336bc6a5dccfb0018657d513e1e609fbad Mon Sep 17 00:00:00 2001 From: Manolis Papadakis Date: Fri, 28 Jul 2023 16:46:01 -0700 Subject: [PATCH 0182/1425] Consensus match * Avoid unbounded loop * Fix unit test failure * Make deleted member functions private * Don't expose allow_out_of_order_destruction on public API * Another fix for compile issues * Fix a compile issue * Update the guidance on padding on the issue_consensus_match docs * Skip consensus match on 1 rank * Testcase for field reuse, including out-of-order destruction * Fix a typo in a print method * Handle destruction properly on moved-from ConsensusMatchResults * Explicitly clear the padding bits * Don't run with mpirun by default * Run consensus match for fields freed potentially out-of-order * Consensus match test: Have shards insert items in different orders * Be more careful about zero-initialization of vector elements * Add testcase for consensus_match operation * More unnecessary `extern` declarations * 'extern' not needed, just remove anonymous namespaces * WIP getting MPI-enabled test runs * Add consensus match operation * Track 'non-det-destruction' property on Storage and RegionField * Move field freeing to LogicalRegionField destructor Co-authored-by: Wonchan Lee See merge request legate/legate.core.internal!66 --- legate/settings.py | 7 +- src/core/data/detail/logical_region_field.cc | 25 ++- src/core/data/detail/logical_region_field.h | 22 ++- src/core/data/detail/logical_store.cc | 28 ++- src/core/data/detail/logical_store.h | 16 ++ src/core/runtime/detail/field_manager.cc | 111 +++++++++--- src/core/runtime/detail/field_manager.h | 30 +++- src/core/runtime/detail/runtime.cc | 18 +- src/core/runtime/detail/runtime.h | 49 ++++++ src/core/runtime/detail/runtime.inl | 71 ++++++++ src/env_defaults.h | 5 +- tests/cpp/CMakeLists.txt | 2 +- tests/cpp/integration/consensus_match.cc | 93 ++++++++++ tests/cpp/integration/field_reuse.cc | 169 +++++++++++++++++++ 14 files changed, 604 insertions(+), 42 deletions(-) create mode 100644 src/core/runtime/detail/runtime.inl create mode 100644 tests/cpp/integration/consensus_match.cc create mode 100644 tests/cpp/integration/field_reuse.cc diff --git a/legate/settings.py b/legate/settings.py index 4768612b29..e4bdf21e98 100644 --- a/legate/settings.py +++ b/legate/settings.py @@ -23,10 +23,11 @@ class LegateRuntimeSettings(Settings): - consensus: PrioritizedSetting[bool] = PrioritizedSetting( + consensus: EnvOnlySetting[bool] = EnvOnlySetting( "consensus", "LEGATE_CONSENSUS", default=False, + test_default=False, convert=convert_bool, help=""" Whether to perform the RegionField consensus match operation on @@ -34,6 +35,8 @@ class LegateRuntimeSettings(Settings): multi-node runs, where all processes must collectively agree that a RegionField has been garbage collected at the Python level before it can be reused. + + This is a read-only environment variable setting used by the runtime. """, ) @@ -189,7 +192,7 @@ class LegateRuntimeSettings(Settings): "field_reuse_freq", "LEGATE_FIELD_REUSE_FREQ", default=32, - test_default=32, + test_default=8, convert=convert_int, help=""" Every how many RegionField allocations to perform a consensus match diff --git a/src/core/data/detail/logical_region_field.cc b/src/core/data/detail/logical_region_field.cc index 42ec25b5a2..dda2508c4b 100644 --- a/src/core/data/detail/logical_region_field.cc +++ b/src/core/data/detail/logical_region_field.cc @@ -12,17 +12,25 @@ #include "core/data/detail/logical_region_field.h" #include "core/partitioning/partition.h" +#include "core/runtime/detail/field_manager.h" #include "core/runtime/detail/runtime.h" namespace legate::detail { -LogicalRegionField::LogicalRegionField(const Legion::LogicalRegion& lr, +LogicalRegionField::LogicalRegionField(FieldManager* manager, + const Legion::LogicalRegion& lr, Legion::FieldID fid, std::shared_ptr parent) - : lr_(lr), fid_(fid), parent_(std::move(parent)) + : manager_(manager), lr_(lr), fid_(fid), parent_(std::move(parent)) { } +LogicalRegionField::~LogicalRegionField() +{ + // Only free the field once the top-level region is deleted. + if (parent_ == nullptr) manager_->free_field(lr_, fid_, destroyed_out_of_order_); +} + int32_t LogicalRegionField::dim() const { return lr_.get_dim(); } const LogicalRegionField& LogicalRegionField::get_root() const @@ -35,6 +43,14 @@ Domain LogicalRegionField::domain() const return Runtime::get_runtime()->get_index_space_domain(lr_.get_index_space()); } +void LogicalRegionField::allow_out_of_order_destruction() +{ + if (parent_ != nullptr) + parent_->allow_out_of_order_destruction(); + else + destroyed_out_of_order_ = true; +} + std::shared_ptr LogicalRegionField::get_child(const Tiling* tiling, const Shape& color, bool complete) @@ -42,7 +58,10 @@ std::shared_ptr LogicalRegionField::get_child(const Tiling* auto legion_partition = get_legion_partition(tiling, complete); auto color_point = to_domain_point(color); return std::make_shared( - Runtime::get_runtime()->get_subregion(legion_partition, color_point), fid_, shared_from_this()); + manager_, + Runtime::get_runtime()->get_subregion(legion_partition, color_point), + fid_, + shared_from_this()); } Legion::LogicalPartition LogicalRegionField::get_legion_partition(const Partition* partition, diff --git a/src/core/data/detail/logical_region_field.h b/src/core/data/detail/logical_region_field.h index cd9ce6a024..3c083a9db5 100644 --- a/src/core/data/detail/logical_region_field.h +++ b/src/core/data/detail/logical_region_field.h @@ -18,6 +18,7 @@ #include "legion.h" #include "core/data/shape.h" +#include "core/runtime/detail/field_manager.h" namespace legate { class Partition; @@ -27,12 +28,18 @@ class Tiling; namespace legate::detail { class LogicalRegionField : public std::enable_shared_from_this { + private: + friend class FieldManager; + public: - LogicalRegionField() {} - LogicalRegionField(const Legion::LogicalRegion& lr, + LogicalRegionField(FieldManager* manager, + const Legion::LogicalRegion& lr, Legion::FieldID fid, std::shared_ptr parent = nullptr); + public: + ~LogicalRegionField(); + public: LogicalRegionField(const LogicalRegionField& other) = default; LogicalRegionField& operator=(const LogicalRegionField& other) = default; @@ -42,9 +49,10 @@ class LogicalRegionField : public std::enable_shared_from_this get_child(const Tiling* tiling, @@ -53,9 +61,11 @@ class LogicalRegionField : public std::enable_shared_from_this parent_{nullptr}; + FieldManager* manager_; + Legion::LogicalRegion lr_; + Legion::FieldID fid_; + std::shared_ptr parent_; + bool destroyed_out_of_order_{false}; }; } // namespace legate::detail diff --git a/src/core/data/detail/logical_store.cc b/src/core/data/detail/logical_store.cc index f61efbd48e..71d5e12713 100644 --- a/src/core/data/detail/logical_store.cc +++ b/src/core/data/detail/logical_store.cc @@ -159,9 +159,10 @@ LogicalRegionField* Storage::get_region_field() #endif if (region_field_ != nullptr) return region_field_.get(); - if (nullptr == parent_) + if (nullptr == parent_) { region_field_ = Runtime::get_runtime()->create_region_field(extents_, type_->size()); - else + if (destroyed_out_of_order_) region_field_->allow_out_of_order_destruction(); + } else region_field_ = parent_->get_child_data(color_); return region_field_.get(); @@ -177,8 +178,12 @@ Legion::Future Storage::get_future() const void Storage::set_region_field(std::shared_ptr&& region_field) { + assert(unbound_ && region_field_ == nullptr); + assert(parent_ == nullptr); + unbound_ = false; region_field_ = std::move(region_field); + if (destroyed_out_of_order_) region_field_->allow_out_of_order_destruction(); // TODO: this is a blocking operation auto domain = region_field_->domain(); @@ -199,6 +204,21 @@ RegionField Storage::map() return Runtime::get_runtime()->map_region_field(get_region_field()); } +void Storage::allow_out_of_order_destruction() +{ + // Technically speaking this property only needs to be tracked on (root) LogicalRegionFields, but + // a Storage may not have instantiated its region_field_ yet, so we note this also on the (root) + // Storage, in case we need to propagate later. We only need to note this on the root Storage, + // because any call that sets region_field_ (get_region_field(), set_region_field()) will end up + // touching the root Storage. + if (parent_ != nullptr) + get_root()->allow_out_of_order_destruction(); + else if (!destroyed_out_of_order_) { + destroyed_out_of_order_ = true; + if (region_field_ != nullptr) region_field_->allow_out_of_order_destruction(); + } +} + Restrictions Storage::compute_restrictions() const { return Restrictions(dim_, Restriction::ALLOW); @@ -244,7 +264,7 @@ std::string Storage::to_string() const else ss << extents_; ss << ", dim: " << dim_ << ", kind: " << (kind_ == Kind::REGION_FIELD ? "Region" : "Future") - << ", type: " << type_->to_string() << ", level: " << level_; + << ", type: " << type_->to_string() << ", level: " << level_ << "}"; return std::move(ss).str(); } @@ -543,6 +563,8 @@ std::shared_ptr LogicalStore::get_physical_store() return mapped_; } +void LogicalStore::allow_out_of_order_destruction() { storage_->allow_out_of_order_destruction(); } + Restrictions LogicalStore::compute_restrictions() const { return transform_->convert(storage_->compute_restrictions()); diff --git a/src/core/data/detail/logical_store.h b/src/core/data/detail/logical_store.h index 52147a2a50..52bea7f326 100644 --- a/src/core/data/detail/logical_store.h +++ b/src/core/data/detail/logical_store.h @@ -75,6 +75,7 @@ class Storage : public std::enable_shared_from_this { public: RegionField map(); + void allow_out_of_order_destruction(); public: Restrictions compute_restrictions() const; @@ -94,6 +95,7 @@ class Storage : public std::enable_shared_from_this { private: uint64_t storage_id_{0}; bool unbound_{false}; + bool destroyed_out_of_order_{false}; int32_t dim_{-1}; Shape extents_; size_t volume_; @@ -200,6 +202,20 @@ class LogicalStore : public std::enable_shared_from_this { public: std::shared_ptr get_physical_store(); + // Informs the runtime that references to this store may be removed in non-deterministic order + // (e.g. by an asynchronous garbage collector). + // + // Normally the top-level code must perform all Legate operations in a deterministic order (at + // least when running on multiple ranks/nodes). This includes destruction of objects managing + // Legate state, like stores. Passing a reference to such an object to a garbage-collected + // language violates this assumption, because (in general) garbage collection can occur at + // indeterminate points during the execution, and thus the point when the object's reference count + // drops to 0 (which triggers object destruction) is not deterministic. + // + // Before passing a store to a garbage collected language, it must first be marked using this + // function, so that the runtime knows to work around the potentially non-determintic removal of + // references. + void allow_out_of_order_destruction(); public: Restrictions compute_restrictions() const; diff --git a/src/core/runtime/detail/field_manager.cc b/src/core/runtime/detail/field_manager.cc index d28c025d2f..30e7e7eb4b 100644 --- a/src/core/runtime/detail/field_manager.cc +++ b/src/core/runtime/detail/field_manager.cc @@ -21,28 +21,32 @@ namespace legate::detail { FieldManager::FieldManager(Runtime* runtime, const Domain& shape, uint32_t field_size) : runtime_(runtime), shape_(shape), field_size_(field_size) { + log_legate.debug() << "Field manager " << this << " created for shape " << shape + << ", field_size " << field_size; } std::shared_ptr FieldManager::allocate_field() { - LogicalRegionField* rf = nullptr; - if (!free_fields_.empty()) { - auto field = free_fields_.front(); - log_legate.debug("Field %u recycled in field manager %p", field.second, this); - free_fields_.pop_front(); - rf = new LogicalRegionField(field.first, field.second); - } else { - auto rgn_mgr = runtime_->find_or_create_region_manager(shape_); - auto [lr, fid] = rgn_mgr->allocate_field(field_size_); - rf = new LogicalRegionField(lr, fid); - log_legate.debug("Field %u created in field manager %p", fid, this); + issue_field_match(); + while (!ordered_free_fields_.empty() || !matches_.empty()) { + // If there's a field that every shard is guaranteed to have, re-use that. + if (!ordered_free_fields_.empty()) { + const auto& field = ordered_free_fields_.front(); + auto* rf = new LogicalRegionField(this, field.first, field.second); + log_legate.debug("Field %u recycled in field manager %p", field.second, this); + ordered_free_fields_.pop_front(); + return std::shared_ptr(rf); + } + // If there are any field matches we haven't processed yet, process the next one, then go back + // and check if any fields were just added to the "ordered" queue. + process_next_field_match(); } - assert(rf != nullptr); - return std::shared_ptr(rf, [this](auto* field) { - log_legate.debug("Field %u freed in field manager %p", field->field_id(), this); - this->free_fields_.push_back(FreeField(field->region(), field->field_id())); - delete field; - }); + // If there are no more field matches to process, then we completely failed to reuse a field. + auto rgn_mgr = runtime_->find_or_create_region_manager(shape_); + auto [lr, fid] = rgn_mgr->allocate_field(field_size_); + auto* rf = new LogicalRegionField(this, lr, fid); + log_legate.debug("Field %u created in field manager %p", fid, this); + return std::shared_ptr(rf); } std::shared_ptr FieldManager::import_field(const Legion::LogicalRegion& region, @@ -51,15 +55,74 @@ std::shared_ptr FieldManager::import_field(const Legion::Log // Import the region only if the region manager is created fresh auto rgn_mgr = runtime_->find_or_create_region_manager(shape_); if (!rgn_mgr->has_space()) rgn_mgr->import_region(region); - log_legate.debug("Field %u imported in field manager %p", field_id, this); + return std::make_shared(this, region, field_id); +} + +void FieldManager::free_field(const Legion::LogicalRegion& region, + Legion::FieldID field_id, + bool unordered) +{ + if (unordered && runtime_->consensus_match_required()) { + log_legate.debug("Field %u freed locally in field manager %p", field_id, this); + unordered_free_fields_.push_back(FreeField(region, field_id)); + } else { + log_legate.debug("Field %u freed in-order in field manager %p", field_id, this); + ordered_free_fields_.push_back(FreeField(region, field_id)); + } +} - auto* rf = new LogicalRegionField(region, field_id); - return std::shared_ptr(rf, [this](auto* field) { - log_legate.debug("Field %u freed in field manager %p", field->field_id(), this); - this->free_fields_.push_back(FreeField(field->region(), field->field_id())); - delete field; - }); +void FieldManager::issue_field_match() +{ + // Check if there are any freed fields that are shared across all the shards. We have to + // test this deterministically no matter what, even if we don't have any fields to offer + // ourselves, because this is a collective with other shards. + if (++field_match_counter_ < runtime_->field_reuse_freq()) return; + field_match_counter_ = 0; + // We need to separately record the region that corresponds to each item in this match, because + // the match itself only uses a subset of the full region info. + auto& regions = regions_for_match_items_.emplace_back(); + std::vector input; + input.reserve(unordered_free_fields_.size()); + for (const auto& field : unordered_free_fields_) { + MatchItem item{field.first.get_tree_id(), field.second}; + input.push_back(item); + regions[item] = field.first; + } + assert(regions.size() == unordered_free_fields_.size()); + unordered_free_fields_.clear(); + // Dispatch the match and put it on the queue of outstanding matches, but don't block on it yet. + // We'll do that when we run out of ordered fields. + matches_.push_back(runtime_->issue_consensus_match(std::move(input))); + log_legate.debug( + "Consensus match emitted with %zu local fields in field manager %p", regions.size(), this); +} + +void FieldManager::process_next_field_match() +{ + assert(!matches_.empty()); + auto& match = matches_.front(); + auto& regions = regions_for_match_items_.front(); + match.wait(); + log_legate.debug("Consensus match result in field manager %p: %zu/%zu fields matched", + this, + match.output().size(), + match.input().size()); + // Put all the matched fields into the ordered queue, in the same order as the match result, + // which is the same order that all shards will see. + for (const auto& item : match.output()) { + auto it = regions.find(item); + assert(it != regions.end()); + ordered_free_fields_.push_back(FreeField(it->second, item.fid)); + regions.erase(it); + } + // All fields that weren't matched can go back into the unordered queue, to be included in the + // next consensus match that we run. + for (const auto& [item, lr] : regions) { + unordered_free_fields_.push_back(FreeField(lr, item.fid)); + } + matches_.pop_front(); + regions_for_match_items_.pop_front(); } } // namespace legate::detail diff --git a/src/core/runtime/detail/field_manager.h b/src/core/runtime/detail/field_manager.h index d1abd1b23d..8e58124bfc 100644 --- a/src/core/runtime/detail/field_manager.h +++ b/src/core/runtime/detail/field_manager.h @@ -20,8 +20,13 @@ namespace legate::detail { class LogicalRegionField; class Runtime; +template +class ConsensusMatchResult; class FieldManager { + private: + friend LogicalRegionField; + public: FieldManager(Runtime* runtime, const Domain& shape, uint32_t field_size); @@ -30,14 +35,37 @@ class FieldManager { std::shared_ptr import_field(const Legion::LogicalRegion& region, Legion::FieldID field_id); + private: + void free_field(const Legion::LogicalRegion& region, Legion::FieldID field_id, bool unordered); + void issue_field_match(); + void process_next_field_match(); + private: Runtime* runtime_; Domain shape_; uint32_t field_size_; + uint32_t field_match_counter_{0}; private: using FreeField = std::pair; - std::deque free_fields_; + // This is a sanitized list of (region,field_id) pairs that is guaranteed to be ordered across all + // the shards even with control replication. + std::deque ordered_free_fields_; + // This list contains the fields that we know have been freed on this shard, but may not have been + // freed yet on other shards. + std::vector unordered_free_fields_; + + private: + struct MatchItem { + Legion::RegionTreeID tid; + Legion::FieldID fid; + friend bool operator<(const MatchItem& l, const MatchItem& r) + { + return std::tie(l.tid, l.fid) < std::tie(r.tid, r.fid); + } + }; + std::deque> matches_; + std::deque> regions_for_match_items_; }; } // namespace legate::detail diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index bb1cba5acc..c39ffa826e 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -57,6 +57,9 @@ const char* TOPLEVEL_NAME = "Legate Core Toplevel Task"; Runtime::Runtime() : legion_runtime_(Legion::Runtime::get_runtime()), next_type_uid_(CUSTOM_TYPE_UID_BASE), + field_reuse_freq_( + extract_env("LEGATE_FIELD_REUSE_FREQ", FIELD_REUSE_FREQ_DEFAULT, FIELD_REUSE_FREQ_TEST)), + force_consensus_match_(extract_env("LEGATE_CONSENSUS", CONSENSUS_DEFAULT, CONSENSUS_TEST)), max_pending_exceptions_(extract_env( "LEGATE_MAX_PENDING_EXCEPTIONS", MAX_PENDING_EXCEPTIONS_DEFAULT, MAX_PENDING_EXCEPTIONS_TEST)) { @@ -346,7 +349,7 @@ std::optional Runtime::check_pending_task_exception() return result; } - // Othrewise, we unpack all pending exceptions and push them to the outstanding exception queue + // Otherwise, we unpack all pending exceptions and push them to the outstanding exception queue for (auto& pending_exception : pending_exceptions_) { auto returned_exception = pending_exception.get_result(); auto result = returned_exception.to_task_exception(); @@ -418,6 +421,12 @@ RegionField Runtime::map_region_field(const LogicalRegionField* rf) void Runtime::unmap_physical_region(Legion::PhysicalRegion pr) { + // TODO: Unmapping doesn't go through the Legion pipeline, so from that perspective it's not + // critical that all shards call `unmap_region` in the same order. However, if shard A unmaps + // region R and shard B doesn't, then both shards launch a task that uses R (or any region that + // overlaps with R), then B will unmap/remap around the task, whereas A will not. To be safe, we + // should consider delaying the unmapping until the field has gone through consensus match, or + // have a full consensus matching process just for unmapping. if (physical_region_refs_.remove(pr)) { // The last user of this inline mapping was removed, so remove it from our cache and unmap. std::vector fields; @@ -433,6 +442,13 @@ void Runtime::unmap_physical_region(Legion::PhysicalRegion pr) size_t Runtime::num_inline_mapped() const { return inline_mapped_.size(); } +uint32_t Runtime::field_reuse_freq() const { return field_reuse_freq_; } + +bool Runtime::consensus_match_required() const +{ + return force_consensus_match_ || Legion::Machine::get_machine().get_address_space_count() > 1; +} + RegionManager* Runtime::find_or_create_region_manager(const Domain& shape) { auto finder = region_managers_.find(shape); diff --git a/src/core/runtime/detail/runtime.h b/src/core/runtime/detail/runtime.h index fba1cd029a..6bbbd62d10 100644 --- a/src/core/runtime/detail/runtime.h +++ b/src/core/runtime/detail/runtime.h @@ -22,6 +22,7 @@ #include "core/mapping/machine.h" #include "core/runtime/detail/communicator_manager.h" #include "core/runtime/detail/field_manager.h" +#include "core/runtime/detail/library.h" #include "core/runtime/detail/machine_manager.h" #include "core/runtime/detail/partition_manager.h" #include "core/runtime/detail/projection.h" @@ -42,6 +43,33 @@ class LogicalStore; class ManualTask; class Operation; +template +class ConsensusMatchResult { + private: + friend class Runtime; + ConsensusMatchResult(std::vector&& input, Legion::Context ctx, Legion::Runtime* runtime); + + public: + ~ConsensusMatchResult(); + ConsensusMatchResult(ConsensusMatchResult&&) = default; + ConsensusMatchResult& operator=(ConsensusMatchResult&&) = default; + + private: + ConsensusMatchResult(const ConsensusMatchResult&) = delete; + ConsensusMatchResult& operator=(const ConsensusMatchResult&) = delete; + + public: + void wait(); + const std::vector& input() const; + const std::vector& output() const; + + private: + std::vector input_; + std::vector output_; + Legion::Future future_; + bool complete_{false}; +}; + class Runtime { public: Runtime(); @@ -123,6 +151,8 @@ class Runtime { RegionField map_region_field(const LogicalRegionField* region_field); void unmap_physical_region(Legion::PhysicalRegion pr); size_t num_inline_mapped() const; + uint32_t field_reuse_freq() const; + bool consensus_match_required() const; public: RegionManager* find_or_create_region_manager(const Legion::Domain& shape); @@ -173,6 +203,11 @@ class Runtime { { return get_tunable(mapper_id, tunable_id, sizeof(T)).get_result(); } + template + T get_core_tunable(int64_t tunable_id) + { + return get_tunable(core_library_->get_mapper_id(), tunable_id); + } public: Legion::Future dispatch(Legion::TaskLauncher* launcher, @@ -194,6 +229,16 @@ class Runtime { public: void issue_execution_fence(bool block = false); + // NOTE: If the type T contains any padding bits, make sure the entries *in the vector* are + // deterministically zero'd out on all shards, e.g. by doing the initialization as follows: + // struct Fred { bool flag; int number; }; + // std::vector input; + // input.emplace_back(); + // memset(&input.back(), 0, sizeof(Fred)); + // input.back().flag = true; + // input.back().flag = number; + template + ConsensusMatchResult issue_consensus_match(std::vector&& input); public: void initialize_toplevel_machine(); @@ -255,6 +300,8 @@ class Runtime { MultiSet physical_region_refs_; uint64_t next_store_id_{1}; uint64_t next_storage_id_{1}; + const uint32_t field_reuse_freq_; + const bool force_consensus_match_; private: std::map libraries_{}; @@ -278,3 +325,5 @@ void registration_callback_for_python(Legion::Machine machine, const std::set& local_procs); } // namespace legate::detail + +#include "core/runtime/detail/runtime.inl" diff --git a/src/core/runtime/detail/runtime.inl b/src/core/runtime/detail/runtime.inl new file mode 100644 index 0000000000..37914a71e7 --- /dev/null +++ b/src/core/runtime/detail/runtime.inl @@ -0,0 +1,71 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#pragma once + +// Useful for IDEs +#include "core/runtime/detail/runtime.h" + +namespace legate::detail { + +template +ConsensusMatchResult::ConsensusMatchResult(std::vector&& input, + Legion::Context ctx, + Legion::Runtime* runtime) + : input_(std::move(input)), + output_(input_.size()), + future_(runtime->consensus_match(ctx, input_.data(), output_.data(), input_.size(), sizeof(T))) +{ +} + +template +ConsensusMatchResult::~ConsensusMatchResult() +{ + // Make sure the consensus match operation has completed, because it will be scribbling over the + // buffers in this object. + if (future_.valid()) wait(); +} + +template +void ConsensusMatchResult::wait() +{ + if (complete_) return; + size_t num_matched = future_.get_result(); + assert(num_matched <= output_.size()); + output_.resize(num_matched); + complete_ = true; +}; + +template +const std::vector& ConsensusMatchResult::input() const +{ + return input_; +}; + +template +const std::vector& ConsensusMatchResult::output() const +{ + assert(complete_); + return output_; +}; + +template +ConsensusMatchResult Runtime::issue_consensus_match(std::vector&& input) +{ + return ConsensusMatchResult(std::move(input), legion_context_, legion_runtime_); +} + +} // namespace legate::detail diff --git a/src/env_defaults.h b/src/env_defaults.h index 8da014dc94..dfda055c5d 100644 --- a/src/env_defaults.h +++ b/src/env_defaults.h @@ -45,10 +45,13 @@ #define FIELD_REUSE_FRAC_TEST 256 #define FIELD_REUSE_FREQ_DEFAULT 32 -#define FIELD_REUSE_FREQ_TEST 32 +#define FIELD_REUSE_FREQ_TEST 8 #define MAX_LRU_LENGTH_DEFAULT 5 #define MAX_LRU_LENGTH_TEST 1 #define DISABLE_MPI_DEFAULT 0 #define DISABLE_MPI_TEST 0 + +#define CONSENSUS_DEFAULT 0 +#define CONSENSUS_TEST 0 diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt index 36b4df7193..305bef2298 100644 --- a/tests/cpp/CMakeLists.txt +++ b/tests/cpp/CMakeLists.txt @@ -69,4 +69,4 @@ include(CPack) include(GNUInstallDirs) install(TARGETS cpp_tests DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/cmake-install") -install( TARGETS cpp_tests DESTINATION ${CMAKE_INSTALL_BINDIR}) +install(TARGETS cpp_tests DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/tests/cpp/integration/consensus_match.cc b/tests/cpp/integration/consensus_match.cc new file mode 100644 index 0000000000..41fd6f3257 --- /dev/null +++ b/tests/cpp/integration/consensus_match.cc @@ -0,0 +1,93 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "core/runtime/detail/runtime.h" +#include "legate.h" + +namespace consensus_match { + +static const char* library_name = "consensus_match"; + +void register_tasks() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); +} + +struct Thing { + bool flag; + int32_t number; + bool operator==(const Thing& other) const { return flag == other.flag && number == other.number; } +}; + +TEST(Integration, ConsensusMatch) +{ + auto runtime = legate::Runtime::get_runtime(); + legate::Core::perform_registration(); + auto context = runtime->find_library(library_name); + + Legion::Runtime* legion_runtime = Legion::Runtime::get_runtime(); + Legion::Context legion_context = legion_runtime->get_context(); + Legion::ShardID sid = legion_runtime->get_shard_id(legion_context, true); + + std::vector input; + // All shards insert 4 items, but in a different order. + for (int i = 0; i < 4; ++i) { + input.emplace_back(); + // Make sure the padding bits have deterministic values. Apparently there is no reliable way to + // force the compiler to do zero initialization. + memset(&input.back(), 0, sizeof(Thing)); + switch ((i + sid) % 4) { + case 0: // shared between shards + input.back().flag = true; + input.back().number = -1; + break; + case 1: // unique among shards + input.back().flag = true; + input.back().number = sid; + break; + case 2: // shared between shards + input.back().flag = false; + input.back().number = -2; + break; + case 3: // unique among shards + input.back().flag = false; + input.back().number = sid; + break; + } + } + + auto result = runtime->impl()->issue_consensus_match(std::move(input)); + result.wait(); + + if (legion_runtime->get_num_shards(legion_context, true) < 2) { + EXPECT_EQ(result.output().size(), 4); + EXPECT_EQ(result.output()[0], result.input()[0]); + EXPECT_EQ(result.output()[1], result.input()[1]); + EXPECT_EQ(result.output()[2], result.input()[2]); + EXPECT_EQ(result.output()[3], result.input()[3]); + } else { + Thing ta{true, -1}; + Thing tb{false, -2}; + EXPECT_EQ(result.output().size(), 2); + EXPECT_TRUE(result.output()[0] == ta && result.output()[1] == tb || + result.output()[0] == tb && result.output()[1] == ta); + } +} + +} // namespace consensus_match diff --git a/tests/cpp/integration/field_reuse.cc b/tests/cpp/integration/field_reuse.cc new file mode 100644 index 0000000000..6df63aad9b --- /dev/null +++ b/tests/cpp/integration/field_reuse.cc @@ -0,0 +1,169 @@ +/* Copyright 2023 NVIDIA Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "core/data/detail/logical_store.h" +#include "core/runtime/detail/runtime.h" +#include "legate.h" + +namespace field_reuse { + +static const char* library_name = "field_reuse"; + +void register_tasks() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); +} + +void check_field_is_new(Legion::FieldID fid) +{ + static std::set unique_fields; + size_t prev_size = unique_fields.size(); + unique_fields.insert(fid); + EXPECT_EQ(unique_fields.size(), prev_size + 1); +} + +TEST(Integration, FieldReuse) +{ + // TODO: Also test the reuse of a field originally returned by an unbounded-output task. + + auto runtime = legate::Runtime::get_runtime(); + legate::Core::perform_registration(); + auto context = runtime->find_library(library_name); + + Legion::Runtime* legion_runtime = Legion::Runtime::get_runtime(); + Legion::Context legion_context = legion_runtime->get_context(); + Legion::ShardID sid = legion_runtime->get_shard_id(legion_context, true); + std::vector shard_local_stores; + + if (legion_runtime->get_num_shards(legion_context, true) < 2) return; + + uint32_t field_reuse_freq = runtime->impl()->field_reuse_freq(); + EXPECT_GE(field_reuse_freq, 7); // otherwise the consensus match would be triggered too early + uint32_t num_allocations = 0; + auto make_store = [&]() { + ++num_allocations; + return runtime->create_store({5, 5}, legate::int64()); + }; + + // A Store freed in-order will be reused immediately. + Legion::FieldID fid1; + { + auto store = make_store(); + fid1 = store.impl()->get_region_field()->field_id(); + check_field_is_new(fid1); + } + auto store1 = make_store(); + EXPECT_EQ(fid1, store1.impl()->get_region_field()->field_id()); + + // This store is marked for out-of-order destruction, but all shards actually free it in-order. + Legion::FieldID fid2; + { + auto store = make_store(); + store.impl()->allow_out_of_order_destruction(); + fid2 = store.impl()->get_region_field()->field_id(); + check_field_is_new(fid2); + } + + // This store is only freed on even shards. + Legion::FieldID fid3; + { + auto store = make_store(); + store.impl()->allow_out_of_order_destruction(); + fid3 = store.impl()->get_region_field()->field_id(); + check_field_is_new(fid3); + if (sid % 2 == 0) shard_local_stores.push_back(store); + } + + // This store is only freed on odd shards. + Legion::FieldID fid4; + { + auto store = make_store(); + store.impl()->allow_out_of_order_destruction(); + fid4 = store.impl()->get_region_field()->field_id(); + check_field_is_new(fid4); + if (sid % 2 == 1) shard_local_stores.push_back(store); + } + + // This store is kept alive on all shards. + Legion::FieldID fid5; + { + auto store = make_store(); + store.impl()->allow_out_of_order_destruction(); + fid5 = store.impl()->get_region_field()->field_id(); + check_field_is_new(fid5); + shard_local_stores.push_back(store); + } + + // None of the previous 4 fields should be reusable yet, so the next allocation will need to + // create a new field. Free and reuse this field enough times to trigger a consensus match. + Legion::FieldID fid6 = 0; + while (num_allocations % field_reuse_freq != 0) { + auto store = make_store(); + if (fid6 == 0) { + fid6 = store.impl()->get_region_field()->field_id(); + check_field_is_new(fid6); + } else + EXPECT_EQ(fid6, store.impl()->get_region_field()->field_id()); + } + + // At this point the consensus match has been triggered, but fid6 is still available, so the next + // allocation will just reuse that. + auto store6 = make_store(); + EXPECT_EQ(fid6, store6.impl()->get_region_field()->field_id()); + + // No more in-order-freed fields remain, so we will block on the consensus match, and reuse the + // only field that was universally freed. + auto store2 = make_store(); + EXPECT_EQ(fid2, store2.impl()->get_region_field()->field_id()); + + // Free any locally cached fields, so they will be included in the next consensus match. + shard_local_stores.clear(); + + // The next allocation will need to create a new field. Free and reuse this field enough times to + // trigger a consensus match. + Legion::FieldID fid7 = 0; + while (num_allocations % field_reuse_freq != 0) { + auto store = make_store(); + if (fid7 == 0) { + fid7 = store.impl()->get_region_field()->field_id(); + check_field_is_new(fid7); + } else + EXPECT_EQ(fid7, store.impl()->get_region_field()->field_id()); + } + + // At this point the consensus match has been triggered, but fid7 is still available, so the next + // allocation will just reuse that. + auto store7 = make_store(); + EXPECT_EQ(fid7, store7.impl()->get_region_field()->field_id()); + + // No more in-order-freed fields remain, so we will block on the consensus match. The next three + // allocations should reuse fid3, fid4 and fid5, but in an undefined order. + std::vector stores345 = {make_store(), make_store(), make_store()}; + std::set fields345; + for (const auto& store : stores345) + fields345.insert(store.impl()->get_region_field()->field_id()); + EXPECT_TRUE(fields345.count(fid3) && fields345.count(fid4) && fields345.count(fid5)); + + // The next allocation will need to create a new field. + auto store8 = make_store(); + Legion::FieldID fid8 = store8.impl()->get_region_field()->field_id(); + check_field_is_new(fid8); +} + +} // namespace field_reuse From 094573e085c701f26e82c5ebc885c4ac080ff3ee Mon Sep 17 00:00:00 2001 From: Parag Kulkarni Date: Wed, 2 Aug 2023 19:55:45 +0530 Subject: [PATCH 0183/1425] - Retarget cpu/gpu builds to internal repository. --- .github/workflows/ci-gh-cpu-build-and-test.yml | 8 ++++---- .github/workflows/ci-gh-gpu-build-and-test.yml | 6 +++--- .github/workflows/gh-build.yml | 6 +++--- .github/workflows/gh-test.yml | 8 ++++---- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci-gh-cpu-build-and-test.yml b/.github/workflows/ci-gh-cpu-build-and-test.yml index e7f492c861..267fd5a750 100644 --- a/.github/workflows/ci-gh-cpu-build-and-test.yml +++ b/.github/workflows/ci-gh-cpu-build-and-test.yml @@ -1,4 +1,4 @@ -name: Build and test CPU legate.core on GH +name: Build and test CPU legate.core.internal on GH concurrency: group: ci-cpu-on-${{ github.event_name }}-from-${{ github.ref_name }} @@ -8,7 +8,7 @@ on: push: branches: - "pull-request/[0-9]+" - - "branch-*" + - "cpp-branch-*" jobs: build-cpu: @@ -17,7 +17,7 @@ jobs: with: build-target: cpu # Ref: https://docs.rapids.ai/resources/github-actions/#cpu-labels for `linux-amd64-cpu4` - runs-on: ${{ github.repository == 'nv-legate/legate.core' && 'linux-amd64-cpu4' || 'ubuntu-latest' }} + runs-on: ${{ github.repository == 'nv-legate/legate.core.internal' && 'linux-amd64-cpu4' || 'ubuntu-latest' }} sha: ${{ github.sha }} test-cpu: @@ -27,7 +27,7 @@ jobs: ./.github/workflows/gh-test.yml with: build-target: cpu - runs-on: ${{ github.repository == 'nv-legate/legate.core' && 'linux-amd64-cpu4' || 'ubuntu-latest' }} + runs-on: ${{ github.repository == 'nv-legate/legate.core.internal' && 'linux-amd64-cpu4' || 'ubuntu-latest' }} sha: ${{ github.sha }} test-scope: unit diff --git a/.github/workflows/ci-gh-gpu-build-and-test.yml b/.github/workflows/ci-gh-gpu-build-and-test.yml index 2231fddb92..45d220c802 100644 --- a/.github/workflows/ci-gh-gpu-build-and-test.yml +++ b/.github/workflows/ci-gh-gpu-build-and-test.yml @@ -1,4 +1,4 @@ -name: Build and test GPU legate.core on GH +name: Build and test GPU legate.core.internal on GH concurrency: group: ci-gpu-on-${{ github.event_name }}-from-${{ github.ref_name }} @@ -8,7 +8,7 @@ on: push: branches: - "pull-request/[0-9]+" - - "branch-*" + - "cpp-branch-*" jobs: build-gpu: @@ -17,7 +17,7 @@ jobs: with: build-target: gpu # Ref: https://docs.rapids.ai/resources/github-actions/#cpu-labels for `linux-amd64-cpu4` - runs-on: ${{ github.repository == 'nv-legate/legate.core' && 'linux-amd64-cpu4' || 'ubuntu-latest' }} + runs-on: ${{ github.repository == 'nv-legate/legate.core.internal' && 'linux-amd64-cpu4' || 'ubuntu-latest' }} sha: ${{ github.sha }} test-gpu: diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index 6daf3f2287..f24a693f74 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -1,4 +1,4 @@ -name: Build legate.core on GH +name: Build legate.core.internal on GH on: workflow_call: @@ -39,7 +39,7 @@ jobs: VAULT_S3_TTL: "28800s" # 8 hours steps: - - name: Checkout legate.core (= this repo) + - name: Checkout legate.core.internal (= this repo) uses: actions/checkout@v3 with: fetch-depth: 0 @@ -86,7 +86,7 @@ jobs: - name: Upload build artifacts uses: actions/upload-artifact@v3 with: - name: "legate.core-${{ inputs.build-target }}-${{ inputs.sha }}" + name: "legate.core.internal-${{ inputs.build-target }}-${{ inputs.sha }}" path: | /tmp/conda-build /tmp/out diff --git a/.github/workflows/gh-test.yml b/.github/workflows/gh-test.yml index f73d2860fd..a701f48cb7 100644 --- a/.github/workflows/gh-test.yml +++ b/.github/workflows/gh-test.yml @@ -1,4 +1,4 @@ -name: Test legate.core on GH +name: Test legate.core.internal on GH on: workflow_call: @@ -34,17 +34,17 @@ jobs: name: Run nvidia-smi to make sure GPU is working run: nvidia-smi - - name: Checkout legate.core + - name: Checkout legate.core.internal uses: actions/checkout@v3 with: - repository: nv-legate/legate.core + repository: nv-legate/legate.core.internal fetch-depth: 0 path: legate - name: Download build artifacts uses: actions/download-artifact@v3 with: - name: "legate.core-${{inputs.build-target}}-${{ inputs.sha }}" + name: "legate.core.internal-${{inputs.build-target}}-${{ inputs.sha }}" path: artifacts - name: Display structure of downloaded artifacts From 5d6f695c5d60ec418a9d8dd1d575652afdeb1465 Mon Sep 17 00:00:00 2001 From: Parag Kulkarni Date: Mon, 7 Aug 2023 20:03:12 +0530 Subject: [PATCH 0184/1425] - Since it is not open source, we will not be taking contributions. Remove the file and its relevant references. --- README.md | 1 - docs/legate/core/source/CONTRIBUTING.md | 1 - docs/legate/core/source/index.rst | 1 - 3 files changed, 3 deletions(-) delete mode 120000 docs/legate/core/source/CONTRIBUTING.md diff --git a/README.md b/README.md index fbee060e02..46ee4cf665 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,6 @@ If you have questions, please contact us at legate(at)nvidia.com. - [Configuring the Jupyter Notebook](#configuring-the-jupyter-notebook) - [Magic Command](#magic-command) - [Other FAQs](#other-faqs) - - [Contributing](#contributing) - [Documentation](#documentation) - [Next Steps](#next-steps) diff --git a/docs/legate/core/source/CONTRIBUTING.md b/docs/legate/core/source/CONTRIBUTING.md deleted file mode 120000 index 069558fad2..0000000000 --- a/docs/legate/core/source/CONTRIBUTING.md +++ /dev/null @@ -1 +0,0 @@ -../../../../CONTRIBUTING.md \ No newline at end of file diff --git a/docs/legate/core/source/index.rst b/docs/legate/core/source/index.rst index d9ecfa2c31..1e930e8323 100644 --- a/docs/legate/core/source/index.rst +++ b/docs/legate/core/source/index.rst @@ -19,7 +19,6 @@ supercomputer without any code modifications. Build instructions Python API Reference C++ API Reference - Contributing .. toctree:: :maxdepth: 1 From ef3a2942a4b6e93124656421c02ce9a36dc6fd0d Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Tue, 15 Aug 2023 23:13:34 -0700 Subject: [PATCH 0185/1425] A first-cut implementation of Legate Array (#23) * Creation APIs for Legate arrays * Revise the callee-side calling convention * Update the mapping API * Post-merge clean-up * Minor clean-up before making changes for Legate Array * Duplicate Task::launch to start changing it for AutoTask * Factor out deserializer for Domain * Serialize/deserialize future indices directly * Catch up the calling convention * Refactor requirement analyzers * Consolidate task launch logic in one function * Clean up other launchers * More clean-up before adding legate arrays to the launcher * Extend AutoTask to handle Legate arrays * Code changes to make string arrays work end-to-end * Minor fixes to handle nested arrays correctly * List types and list arrays * Find-or-create key partition shouldn't mess up with the current key partition * Offset-based list array implementation * A simple accessor implementation * Use unique pointers in task launcher * Temporary launcher fix for the Python core * Split LogicalArray to multiple classes, one for each kind * Refine launcher args for array kinds * Use special array classes in physical and mapping views * Clean up constraint generation for arrays and put back constraint validation * Use a more conrete type for descriptor sub-arrays * Make sure concrete array types can be constructed only from Array * Clean up array accessors * Tie nullability of list arrays with descriptor sub-array's * Code clean-up and unit tests for array creation * Post-merge clean-up * Reject reductions with list/string arrays * Raise an exception when the size of a variable size type is queried * Remove unnecessary noexcept clauses * CUDA & OpenMP variants for offset-range conversion * Fix the repartitioning logic for compressed images * Partition cache for image partitions * Invalidate the cached image partition when the function is updated * Use arrays' primary stores in find_or_declare_partition for arrays * Make AutoTask and ManualTask use the same launch logic * Use ranges for list arrays and disallow nested lists for now * Inline mapping for logical arrays * Make accessors require concrete array types * Update the task signature to receive the TaskContext object directly * Add add_{input,output,reduction} overloads that don't take partition symbols * Move special type constructors to the detail namespace * Missing file * Struct array support * A method to create a nullable base array from stores * Reject variable size types and struct types in create_array_like for now * Transformations for arrays * Remove obsolete declaration * Missing header files in the install list * Use the right variable to check if the openmp module is available * Turn off the build for examples for now --- .../home/coder/.local/bin/build-all | 6 +- examples/cpp/hello/task_hello.cc | 24 +- examples/cpp/hello/task_hello.h | 8 +- examples/cpp/io/task_io.cc | 44 +- examples/cpp/io/task_io.h | 12 +- examples/hello/src/hello_world.cc | 4 +- examples/hello/src/iota.cc | 4 +- examples/hello/src/square.cc | 6 +- examples/hello/src/sum.cc | 10 +- examples/io/src/read_even_tiles.cc | 6 +- examples/io/src/read_file.cc | 6 +- examples/io/src/read_uneven_tiles.cc | 6 +- examples/io/src/util.cc | 4 +- examples/io/src/util.h | 4 +- examples/io/src/write_even_tiles.cc | 10 +- examples/io/src/write_file.cc | 6 +- examples/io/src/write_uneven_tiles.cc | 6 +- examples/reduction/src/bincount.cc | 6 +- examples/reduction/src/categorize.cc | 8 +- examples/reduction/src/histogram.cc | 8 +- examples/reduction/src/matmul.cc | 8 +- examples/reduction/src/mul.cc | 8 +- examples/reduction/src/sum_over_axis.cc | 6 +- examples/reduction/src/unique.cc | 20 +- legate/core/_lib/types.pyx | 2 +- legate/core/launcher.py | 43 +- legate_core_cpp.cmake | 23 +- src/core/cuda/cuda_help.h | 1 + src/core/data/accessors.h | 84 ++++ src/core/data/accessors.inl | 116 +++++ src/core/data/array.cc | 88 ++++ src/core/data/array.h | 185 +++++++ src/core/data/array.inl | 32 ++ src/core/data/detail/array.cc | 156 ++++++ src/core/data/detail/array.h | 158 ++++++ src/core/data/detail/array_kind.h | 25 + src/core/data/detail/array_tasks.cc | 92 ++++ src/core/data/detail/array_tasks.cu | 151 ++++++ src/core/data/detail/array_tasks.h | 51 ++ src/core/data/detail/array_tasks_omp.cc | 88 ++++ src/core/data/detail/logical_array.cc | 451 ++++++++++++++++++ src/core/data/detail/logical_array.h | 244 ++++++++++ src/core/data/detail/logical_region_field.cc | 28 +- src/core/data/detail/logical_region_field.h | 6 + src/core/data/detail/logical_store.cc | 75 ++- src/core/data/detail/logical_store.h | 15 +- src/core/data/detail/store.h | 2 + src/core/data/logical_array.cc | 127 +++++ src/core/data/logical_array.h | 282 +++++++++++ src/core/data/logical_store.cc | 2 +- src/core/data/logical_store.h | 18 +- src/core/data/store.cc | 13 +- src/core/data/store.h | 20 +- src/core/data/store.inl | 7 +- src/core/legate_c.h | 4 + src/core/mapping/array.cc | 50 ++ src/core/mapping/array.h | 101 ++++ src/core/mapping/detail/array.cc | 121 +++++ src/core/mapping/detail/array.h | 152 ++++++ src/core/mapping/detail/base_mapper.cc | 83 ++-- src/core/mapping/detail/operation.cc | 8 +- src/core/mapping/detail/operation.h | 55 +-- src/core/mapping/detail/store.h | 1 + src/core/mapping/operation.cc | 21 +- src/core/mapping/operation.h | 54 +-- src/core/operation/detail/copy.cc | 2 +- src/core/operation/detail/copy_launcher.cc | 99 ++-- src/core/operation/detail/copy_launcher.h | 13 +- src/core/operation/detail/fill_launcher.cc | 91 ++-- src/core/operation/detail/fill_launcher.h | 15 +- src/core/operation/detail/gather.cc | 2 +- src/core/operation/detail/launcher_arg.cc | 201 +++++++- src/core/operation/detail/launcher_arg.h | 134 ++++-- src/core/operation/detail/operation.cc | 1 - src/core/operation/detail/operation.h | 9 - src/core/operation/detail/reduce.cc | 20 +- src/core/operation/detail/req_analyzer.cc | 52 +- src/core/operation/detail/req_analyzer.h | 94 +++- src/core/operation/detail/scatter.cc | 2 +- src/core/operation/detail/scatter_gather.cc | 2 +- src/core/operation/detail/task.cc | 221 +++++---- src/core/operation/detail/task.h | 53 +- src/core/operation/detail/task_launcher.cc | 309 ++++++------ src/core/operation/detail/task_launcher.h | 64 +-- src/core/operation/task.cc | 40 +- src/core/operation/task.h | 107 +++-- src/core/partitioning/detail/constraint.cc | 20 +- .../partitioning/detail/constraint_solver.cc | 4 +- .../partitioning/detail/constraint_solver.h | 4 +- src/core/partitioning/partition.cc | 36 +- src/core/runtime/detail/library.cc | 2 +- src/core/runtime/detail/library.h | 2 +- src/core/runtime/detail/partition_manager.cc | 33 ++ src/core/runtime/detail/partition_manager.h | 16 + src/core/runtime/detail/runtime.cc | 160 ++++++- src/core/runtime/detail/runtime.h | 59 ++- src/core/runtime/library.h | 5 +- src/core/runtime/runtime.cc | 21 +- src/core/runtime/runtime.h | 40 +- src/core/task/detail/task_context.cc | 41 +- src/core/task/detail/task_context.h | 16 +- src/core/task/task_context.cc | 38 +- src/core/task/task_context.h | 79 ++- src/core/type/detail/type_info.cc | 224 ++++++++- src/core/type/detail/type_info.h | 61 ++- src/core/type/type_info.cc | 149 ++---- src/core/type/type_info.h | 44 +- src/core/utilities/deserializer.cc | 187 +++++--- src/core/utilities/deserializer.h | 35 +- src/core/utilities/deserializer.inl | 30 +- src/core/utilities/detail/buffer_builder.h | 6 + src/core/utilities/typedefs.h | 2 +- .../cpp/integration/alignment_constraints.cc | 45 +- .../cpp/integration/broadcast_constraints.cc | 24 +- tests/cpp/integration/copy_gather.cc | 12 +- tests/cpp/integration/copy_gather_scatter.cc | 16 +- tests/cpp/integration/copy_normal.cc | 22 +- tests/cpp/integration/copy_scatter.cc | 14 +- tests/cpp/integration/copy_util.inl | 14 +- tests/cpp/integration/cpu_communicator.cc | 2 +- tests/cpp/integration/exception.cc | 6 +- tests/cpp/integration/fill.cc | 20 +- tests/cpp/integration/image_constraints.cc | 59 ++- tests/cpp/integration/inline_map.cc | 8 +- tests/cpp/integration/inout.cc | 8 +- tests/cpp/integration/machine_scope.cc | 10 +- tests/cpp/integration/nccl.cu | 2 +- tests/cpp/integration/provenance.cc | 6 +- .../integration/tasks/task_region_manager.h | 2 +- tests/cpp/integration/tasks/task_simple.cc | 18 +- tests/cpp/integration/tasks/task_simple.h | 6 +- tests/cpp/integration/tree_reduce.cc | 22 +- tests/cpp/integration/tree_reduce_unique.cc | 39 +- tests/cpp/integration/weighted.cc | 16 +- tests/cpp/run.sh | 2 +- tests/cpp/unit/logical_array_creation.cc | 234 +++++++++ tests/cpp/unit/logical_store.cc | 3 - tests/cpp/unit/registration.cc | 4 +- tests/cpp/unit/scalar.cc | 6 +- .../collective/src/collective_test.cc | 2 +- .../collective/src/collective_test.h | 2 +- .../integration/region_manager/src/tester.cc | 2 +- tests/integration/region_manager/src/tester.h | 2 +- tests/integration/registry/src/hello.cc | 2 +- tests/integration/registry/src/world.cc | 2 +- tests/integration/registry/src/world.h | 2 +- tests/integration/scoping/src/library.cc | 10 +- .../tree_reduce/src/produce_normal.cc | 2 +- .../tree_reduce/src/produce_normal.h | 2 +- .../tree_reduce/src/produce_unbound.cc | 9 +- .../tree_reduce/src/produce_unbound.h | 2 +- .../tree_reduce/src/reduce_normal.cc | 8 +- .../tree_reduce/src/reduce_normal.h | 2 +- .../tree_reduce/src/reduce_unbound.cc | 8 +- .../tree_reduce/src/reduce_unbound.h | 2 +- 155 files changed, 5567 insertions(+), 1422 deletions(-) create mode 100644 src/core/data/accessors.h create mode 100644 src/core/data/accessors.inl create mode 100644 src/core/data/array.cc create mode 100644 src/core/data/array.h create mode 100644 src/core/data/array.inl create mode 100644 src/core/data/detail/array.cc create mode 100644 src/core/data/detail/array.h create mode 100644 src/core/data/detail/array_kind.h create mode 100644 src/core/data/detail/array_tasks.cc create mode 100644 src/core/data/detail/array_tasks.cu create mode 100644 src/core/data/detail/array_tasks.h create mode 100644 src/core/data/detail/array_tasks_omp.cc create mode 100644 src/core/data/detail/logical_array.cc create mode 100644 src/core/data/detail/logical_array.h create mode 100644 src/core/data/logical_array.cc create mode 100644 src/core/data/logical_array.h create mode 100644 src/core/mapping/array.cc create mode 100644 src/core/mapping/array.h create mode 100644 src/core/mapping/detail/array.cc create mode 100644 src/core/mapping/detail/array.h create mode 100644 tests/cpp/unit/logical_array_creation.cc diff --git a/continuous_integration/home/coder/.local/bin/build-all b/continuous_integration/home/coder/.local/bin/build-all index 95b41b0d88..ac75f45498 100755 --- a/continuous_integration/home/coder/.local/bin/build-all +++ b/continuous_integration/home/coder/.local/bin/build-all @@ -12,7 +12,11 @@ build_all() { build-legate-cpp; build-legate-wheel; build-legate-conda; - build-cpp-example; + # FIXME: C++ Legate examples need a working cuNumeric installation for the + # build, but the current CI image doesn't have it. This needs to get + # fixed by either including cuNumeric in CI image or modifying the + # examples so they are self-contained + #build-cpp-example; build-cpp-test; } diff --git a/examples/cpp/hello/task_hello.cc b/examples/cpp/hello/task_hello.cc index ff5323ff14..8d315ecc93 100644 --- a/examples/cpp/hello/task_hello.cc +++ b/examples/cpp/hello/task_hello.cc @@ -28,15 +28,15 @@ void register_tasks() SumTask::register_variants(library); } -/*static*/ void HelloWorldTask::cpu_variant(legate::TaskContext& context) +/*static*/ void HelloWorldTask::cpu_variant(legate::TaskContext context) { - std::string message = context.scalars().at(0).value(); + std::string message = context.scalar(0).value(); std::cout << message << std::endl; } -/*static*/ void SumTask::cpu_variant(legate::TaskContext& context) +/*static*/ void SumTask::cpu_variant(legate::TaskContext context) { - legate::Store& input = context.inputs().at(0); + legate::Store input = context.input(0); legate::Rect<1> input_shape = input.shape<1>(); // should be a 1-Dim array auto in = input.read_accessor(); @@ -54,9 +54,9 @@ void register_tasks() to add our local contribution. After all point tasks return, the runtime will make sure to combine all their buffers into the single final result. */ - using Reduce = Legion::SumReduction; - legate::Store& output = context.reductions().at(0); - auto sum = output.reduce_accessor(); + using Reduce = Legion::SumReduction; + legate::Store output = context.reduction(0); + auto sum = output.reduce_accessor(); // Best-practice is to validate types assert(output.code() == legate::Type::Code::FLOAT32); assert(output.dim() == 1); @@ -64,16 +64,16 @@ void register_tasks() sum.reduce(0, total); } -/*static*/ void SquareTask::cpu_variant(legate::TaskContext& context) +/*static*/ void SquareTask::cpu_variant(legate::TaskContext context) { - legate::Store& output = context.outputs().at(0); + legate::Store output = context.output(0); // Best-practice to validate the store types assert(output.code() == legate::Type::Code::FLOAT32); assert(output.dim() == 1); legate::Rect<1> output_shape = output.shape<1>(); auto out = output.write_accessor(); - legate::Store& input = context.inputs().at(0); + legate::Store input = context.input(0); // Best-practice to validate the store types assert(input.code() == legate::Type::Code::FLOAT32); assert(input.dim() == 1); @@ -88,9 +88,9 @@ void register_tasks() for (size_t i = input_shape.lo; i <= input_shape.hi; ++i) { out[i] = in[i] * in[i]; } } -/*static*/ void IotaTask::cpu_variant(legate::TaskContext& context) +/*static*/ void IotaTask::cpu_variant(legate::TaskContext context) { - legate::Store& output = context.outputs().at(0); + legate::Store output = context.output(0); legate::Rect<1> output_shape = output.shape<1>(); auto out = output.write_accessor(); diff --git a/examples/cpp/hello/task_hello.h b/examples/cpp/hello/task_hello.h index 8333800eb7..ef55e181c7 100644 --- a/examples/cpp/hello/task_hello.h +++ b/examples/cpp/hello/task_hello.h @@ -34,22 +34,22 @@ void register_tasks(); struct HelloWorldTask : public legate::LegateTask { static const int32_t TASK_ID = HELLO_WORLD; - static void cpu_variant(legate::TaskContext& context); + static void cpu_variant(legate::TaskContext context); }; struct SumTask : public legate::LegateTask { static const int32_t TASK_ID = SUM; - static void cpu_variant(legate::TaskContext& context); + static void cpu_variant(legate::TaskContext context); }; struct SquareTask : public legate::LegateTask { static const int32_t TASK_ID = SQUARE; - static void cpu_variant(legate::TaskContext& context); + static void cpu_variant(legate::TaskContext context); }; struct IotaTask : public legate::LegateTask { static const int32_t TASK_ID = IOTA; - static void cpu_variant(legate::TaskContext& context); + static void cpu_variant(legate::TaskContext context); }; } // namespace hello diff --git a/examples/cpp/io/task_io.cc b/examples/cpp/io/task_io.cc index 83f9c3bc73..afda8e9c2c 100644 --- a/examples/cpp/io/task_io.cc +++ b/examples/cpp/io/task_io.cc @@ -69,7 +69,7 @@ struct write_util_fn { } }; -std::filesystem::path get_unique_path_for_task_index(const legate::TaskContext& context, +std::filesystem::path get_unique_path_for_task_index(const legate::TaskContext context, int32_t ndim, const std::string& dirname) { @@ -90,7 +90,7 @@ std::filesystem::path get_unique_path_for_task_index(const legate::TaskContext& return fs::path(dirname) / filename; } -void write_to_file(legate::TaskContext& task_context, +void write_to_file(legate::TaskContext task_context, const std::string& dirname, const legate::Store& store) { @@ -131,10 +131,10 @@ struct read_even_fn { } }; -/*static*/ void ReadEvenTilesTask::cpu_variant(legate::TaskContext& context) +/*static*/ void ReadEvenTilesTask::cpu_variant(legate::TaskContext context) { - auto dirname = context.scalars().at(0).value(); - auto& output = context.outputs().at(0); + auto dirname = context.scalar(0).value(); + auto output = context.output(0).data(); auto path = utils::get_unique_path_for_task_index(context, output.dim(), dirname); // double_dispatch converts the first two arguments to non-type template arguments @@ -193,10 +193,10 @@ struct read_fn { } }; -/*static*/ void ReadFileTask::cpu_variant(legate::TaskContext& context) +/*static*/ void ReadFileTask::cpu_variant(legate::TaskContext context) { - auto filename = context.scalars().at(0).value(); - auto& output = context.outputs().at(0); + auto filename = context.scalar(0).value(); + auto output = context.output(0).data(); // The task context contains metadata about the launch so each reader task can figure out // which part of the file it needs to read into the output. @@ -238,10 +238,10 @@ struct read_uneven_fn { } }; -/*static*/ void ReadUnevenTilesTask::cpu_variant(legate::TaskContext& context) +/*static*/ void ReadUnevenTilesTask::cpu_variant(legate::TaskContext context) { - auto dirname = context.scalars().at(0).value(); - auto& output = context.outputs().at(0); + auto dirname = context.scalar(0).value(); + auto output = context.output(0).data(); auto path = utils::get_unique_path_for_task_index(context, output.dim(), dirname); // double_dispatch converts the first two arguments to non-type template arguments @@ -262,12 +262,12 @@ void write_header(std::ofstream& out, for (auto& v : tile_shape) out.write(reinterpret_cast(&v), sizeof(int32_t)); } -/*static*/ void WriteEvenTilesTask::cpu_variant(legate::TaskContext& context) +/*static*/ void WriteEvenTilesTask::cpu_variant(legate::TaskContext context) { - auto dirname = context.scalars().at(0).value(); - legate::Span shape = context.scalars().at(1).values(); - legate::Span tile_shape = context.scalars().at(2).values(); - auto& input = context.inputs().at(0); + auto dirname = context.scalar(0).value(); + legate::Span shape = context.scalar(1).values(); + legate::Span tile_shape = context.scalar(2).values(); + auto input = context.input(0).data(); auto launch_domain = context.get_launch_domain(); auto task_index = context.get_task_index(); @@ -306,10 +306,10 @@ struct write_fn { } }; -/*statis*/ void WriteFileTask::cpu_variant(legate::TaskContext& context) +/*statis*/ void WriteFileTask::cpu_variant(legate::TaskContext context) { - auto filename = context.scalars().at(0).value(); - auto& input = context.inputs().at(0); + auto filename = context.scalar(0).value(); + auto input = context.input(0).data(); logger.debug() << "Write to " << filename; legate::type_dispatch(input.code(), write_fn{}, input, filename); @@ -332,10 +332,10 @@ struct header_write_fn { } }; -/*statis*/ void WriteUnevenTilesTask::cpu_variant(legate::TaskContext& context) +/*statis*/ void WriteUnevenTilesTask::cpu_variant(legate::TaskContext context) { - auto dirname = context.scalars().at(0).value(); - auto& input = context.inputs().at(0); + auto dirname = context.scalar(0).value(); + auto input = context.input(0).data(); auto launch_domain = context.get_launch_domain(); auto task_index = context.get_task_index(); diff --git a/examples/cpp/io/task_io.h b/examples/cpp/io/task_io.h index d9bbe63507..84ccfa79b0 100644 --- a/examples/cpp/io/task_io.h +++ b/examples/cpp/io/task_io.h @@ -36,32 +36,32 @@ void register_tasks(); struct ReadEvenTilesTask : public legate::LegateTask { static const int32_t TASK_ID = READ_EVEN_TILES; - static void cpu_variant(legate::TaskContext& context); + static void cpu_variant(legate::TaskContext context); }; struct ReadFileTask : public legate::LegateTask { static const int32_t TASK_ID = READ_FILE; - static void cpu_variant(legate::TaskContext& context); + static void cpu_variant(legate::TaskContext context); }; struct ReadUnevenTilesTask : public legate::LegateTask { static const int32_t TASK_ID = READ_UNEVEN_TILES; - static void cpu_variant(legate::TaskContext& context); + static void cpu_variant(legate::TaskContext context); }; struct WriteEvenTilesTask : public legate::LegateTask { static const int32_t TASK_ID = WRITE_EVEN_TILES; - static void cpu_variant(legate::TaskContext& context); + static void cpu_variant(legate::TaskContext context); }; struct WriteFileTask : public legate::LegateTask { static const int32_t TASK_ID = WRITE_FILE; - static void cpu_variant(legate::TaskContext& context); + static void cpu_variant(legate::TaskContext context); }; struct WriteUnevenTilesTask : public legate::LegateTask { static const int32_t TASK_ID = WRITE_UNEVEN_TILES; - static void cpu_variant(legate::TaskContext& context); + static void cpu_variant(legate::TaskContext context); }; } // namespace legateio diff --git a/examples/hello/src/hello_world.cc b/examples/hello/src/hello_world.cc index 9d06e941c3..65127291e0 100644 --- a/examples/hello/src/hello_world.cc +++ b/examples/hello/src/hello_world.cc @@ -19,9 +19,9 @@ Legion::Logger logger("legate.hello"); class HelloWorldTask : public Task { public: - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - std::string message = context.scalars().at(0).value(); + std::string message = context.scalar(0).value(); std::cout << message << std::endl; } }; diff --git a/examples/hello/src/iota.cc b/examples/hello/src/iota.cc index 5a4b39bcdd..d049f70b92 100644 --- a/examples/hello/src/iota.cc +++ b/examples/hello/src/iota.cc @@ -17,9 +17,9 @@ namespace hello { class IotaTask : public Task { public: - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - legate::Store& output = context.outputs().at(0); + legate::Store output = context.output(0).data(); legate::Rect<1> output_shape = output.shape<1>(); auto out = output.write_accessor(); diff --git a/examples/hello/src/square.cc b/examples/hello/src/square.cc index d4a569cc37..4a3c6e5be7 100644 --- a/examples/hello/src/square.cc +++ b/examples/hello/src/square.cc @@ -17,13 +17,13 @@ namespace hello { class SquareTask : public Task { public: - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - legate::Store& output = context.outputs().at(0); + legate::Store output = context.output(0).data(); legate::Rect<1> output_shape = output.shape<1>(); auto out = output.write_accessor(); - legate::Store& input = context.inputs().at(0); + legate::Store input = context.input(0).data(); legate::Rect<1> input_shape = input.shape<1>(); // should be a 1-Dim array auto in = input.read_accessor(); diff --git a/examples/hello/src/sum.cc b/examples/hello/src/sum.cc index dfbfedc8ad..17d0564191 100644 --- a/examples/hello/src/sum.cc +++ b/examples/hello/src/sum.cc @@ -17,9 +17,9 @@ namespace hello { class SumTask : public Task { public: - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - legate::Store& input = context.inputs().at(0); + legate::Store input = context.input(0).data(); legate::Rect<1> input_shape = input.shape<1>(); // should be a 1-Dim array auto in = input.read_accessor(); @@ -37,9 +37,9 @@ class SumTask : public Task { to add our local contribution. After all point tasks return, the runtime will make sure to combine all their buffers into the single final result. */ - using Reduce = Legion::SumReduction; - legate::Store& output = context.reductions().at(0); - auto sum = output.reduce_accessor(); + using Reduce = Legion::SumReduction; + legate::Store output = context.reduction(0).data(); + auto sum = output.reduce_accessor(); assert(output.shape<1>() == legate::Rect<1>(0, 0)); sum.reduce(0, total); } diff --git a/examples/io/src/read_even_tiles.cc b/examples/io/src/read_even_tiles.cc index 80ebd5b21b..206672c8aa 100644 --- a/examples/io/src/read_even_tiles.cc +++ b/examples/io/src/read_even_tiles.cc @@ -57,10 +57,10 @@ struct read_fn { class ReadEvenTilesTask : public Task { public: - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto dirname = context.scalars().at(0).value(); - auto& output = context.outputs().at(0); + auto dirname = context.scalar(0).value(); + auto output = context.output(0).data(); auto path = get_unique_path_for_task_index(context, output.dim(), dirname); // double_dispatch converts the first two arguments to non-type template arguments diff --git a/examples/io/src/read_file.cc b/examples/io/src/read_file.cc index 7e67362293..4efe4450b4 100644 --- a/examples/io/src/read_file.cc +++ b/examples/io/src/read_file.cc @@ -75,10 +75,10 @@ struct read_fn { class ReadFileTask : public Task { public: - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto filename = context.scalars().at(0).value(); - auto& output = context.outputs().at(0); + auto filename = context.scalar(0).value(); + auto output = context.output(0).data(); // The task context contains metadata about the launch so each reader task can figure out // which part of the file it needs to read into the output. diff --git a/examples/io/src/read_uneven_tiles.cc b/examples/io/src/read_uneven_tiles.cc index 559c3f70bb..090bd4b55d 100644 --- a/examples/io/src/read_uneven_tiles.cc +++ b/examples/io/src/read_uneven_tiles.cc @@ -57,10 +57,10 @@ struct read_fn { class ReadUnevenTilesTask : public Task { public: - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto dirname = context.scalars().at(0).value(); - auto& output = context.outputs().at(0); + auto dirname = context.scalar(0).value(); + auto output = context.output(0).data(); auto path = get_unique_path_for_task_index(context, output.dim(), dirname); // double_dispatch converts the first two arguments to non-type template arguments diff --git a/examples/io/src/util.cc b/examples/io/src/util.cc index 11ad75fa63..cc993b7b03 100644 --- a/examples/io/src/util.cc +++ b/examples/io/src/util.cc @@ -58,7 +58,7 @@ struct write_fn { } // namespace -std::filesystem::path get_unique_path_for_task_index(const legate::TaskContext& context, +std::filesystem::path get_unique_path_for_task_index(legate::TaskContext context, int32_t ndim, const std::string& dirname) { @@ -79,7 +79,7 @@ std::filesystem::path get_unique_path_for_task_index(const legate::TaskContext& return fs::path(dirname) / filename; } -void write_to_file(legate::TaskContext& task_context, +void write_to_file(legate::TaskContext task_context, const std::string& dirname, const legate::Store& store) { diff --git a/examples/io/src/util.h b/examples/io/src/util.h index bbe29dc45d..2b697fca1e 100644 --- a/examples/io/src/util.h +++ b/examples/io/src/util.h @@ -19,11 +19,11 @@ namespace legateio { -std::filesystem::path get_unique_path_for_task_index(const legate::TaskContext& task_context, +std::filesystem::path get_unique_path_for_task_index(legate::TaskContext task_context, int32_t ndim, const std::string& dirname); -void write_to_file(legate::TaskContext& task_context, +void write_to_file(legate::TaskContext task_context, const std::string& dirname, const legate::Store& store); diff --git a/examples/io/src/write_even_tiles.cc b/examples/io/src/write_even_tiles.cc index d9bd42921a..9ec3a177a0 100644 --- a/examples/io/src/write_even_tiles.cc +++ b/examples/io/src/write_even_tiles.cc @@ -43,12 +43,12 @@ void write_header(std::ofstream& out, class WriteEvenTilesTask : public Task { public: - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto dirname = context.scalars().at(0).value(); - legate::Span shape = context.scalars().at(1).values(); - legate::Span tile_shape = context.scalars().at(2).values(); - auto& input = context.inputs().at(0); + auto dirname = context.scalar(0).value(); + legate::Span shape = context.scalar(1).values(); + legate::Span tile_shape = context.scalar(2).values(); + auto input = context.input(0).data(); auto launch_domain = context.get_launch_domain(); auto task_index = context.get_task_index(); diff --git a/examples/io/src/write_file.cc b/examples/io/src/write_file.cc index 69aedc88a9..6e995471a7 100644 --- a/examples/io/src/write_file.cc +++ b/examples/io/src/write_file.cc @@ -48,10 +48,10 @@ struct write_fn { class WriteFileTask : public Task { public: - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto filename = context.scalars().at(0).value(); - auto& input = context.inputs().at(0); + auto filename = context.scalar(0).value(); + auto input = context.input(0).data(); logger.print() << "Write to " << filename; legate::type_dispatch(input.code(), write_fn{}, input, filename); diff --git a/examples/io/src/write_uneven_tiles.cc b/examples/io/src/write_uneven_tiles.cc index ef62dc0a81..f81bc6859a 100644 --- a/examples/io/src/write_uneven_tiles.cc +++ b/examples/io/src/write_uneven_tiles.cc @@ -46,10 +46,10 @@ struct header_write_fn { class WriteUnevenTilesTask : public Task { public: - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto dirname = context.scalars().at(0).value(); - auto& input = context.inputs().at(0); + auto dirname = context.scalar(0).value(); + auto input = context.input(0).data(); auto launch_domain = context.get_launch_domain(); auto task_index = context.get_task_index(); diff --git a/examples/reduction/src/bincount.cc b/examples/reduction/src/bincount.cc index 266dadd6b3..36816cc992 100644 --- a/examples/reduction/src/bincount.cc +++ b/examples/reduction/src/bincount.cc @@ -19,10 +19,10 @@ namespace reduction { class BincountTask : public Task { public: - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto& input = context.inputs().at(0); - auto& output = context.reductions().at(0); + auto input = context.input(0).data(); + auto output = context.reduction(0).data(); auto in_shape = input.shape<1>(); auto out_shape = output.shape<1>(); diff --git a/examples/reduction/src/categorize.cc b/examples/reduction/src/categorize.cc index 94ac13cb6a..6a81c6ca7d 100644 --- a/examples/reduction/src/categorize.cc +++ b/examples/reduction/src/categorize.cc @@ -60,11 +60,11 @@ struct categorize_fn { class CategorizeTask : public Task { public: - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto& input = context.inputs().at(0); - auto& bins = context.inputs().at(1); - auto& result = context.outputs().at(0); + auto input = context.input(0).data(); + auto bins = context.input(1).data(); + auto result = context.output(0).data(); legate::type_dispatch(input.code(), categorize_fn{}, result, input, bins); } diff --git a/examples/reduction/src/histogram.cc b/examples/reduction/src/histogram.cc index f922126238..eaefe41e14 100644 --- a/examples/reduction/src/histogram.cc +++ b/examples/reduction/src/histogram.cc @@ -59,11 +59,11 @@ struct histogram_fn { class HistogramTask : public Task { public: - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto& input = context.inputs().at(0); - auto& bins = context.inputs().at(1); - auto& result = context.reductions().at(0); + auto input = context.input(0).data(); + auto bins = context.input(1).data(); + auto result = context.reduction(0).data(); legate::type_dispatch(input.code(), histogram_fn{}, result, input, bins); } diff --git a/examples/reduction/src/matmul.cc b/examples/reduction/src/matmul.cc index 5fb1064eb9..e5293a728f 100644 --- a/examples/reduction/src/matmul.cc +++ b/examples/reduction/src/matmul.cc @@ -44,11 +44,11 @@ struct matmul_fn { class MatMulTask : public Task { public: - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto& rhs1 = context.inputs().at(0); - auto& rhs2 = context.inputs().at(1); - auto& lhs = context.reductions().at(0); + auto rhs1 = context.input(0).data(); + auto rhs2 = context.input(1).data(); + auto lhs = context.reduction(0).data(); legate::type_dispatch(lhs.code(), matmul_fn{}, lhs, rhs1, rhs2); } diff --git a/examples/reduction/src/mul.cc b/examples/reduction/src/mul.cc index d203b5b1a4..29d6508ef5 100644 --- a/examples/reduction/src/mul.cc +++ b/examples/reduction/src/mul.cc @@ -44,11 +44,11 @@ struct mul_fn { class MultiplyTask : public Task { public: - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto& rhs1 = context.inputs().at(0); - auto& rhs2 = context.inputs().at(1); - auto& lhs = context.outputs().at(0); + auto rhs1 = context.input(0).data(); + auto rhs2 = context.input(1).data(); + auto lhs = context.output(0).data(); legate::double_dispatch(lhs.dim(), lhs.code(), mul_fn{}, lhs, rhs1, rhs2); } diff --git a/examples/reduction/src/sum_over_axis.cc b/examples/reduction/src/sum_over_axis.cc index abd9631bf6..d33c94887e 100644 --- a/examples/reduction/src/sum_over_axis.cc +++ b/examples/reduction/src/sum_over_axis.cc @@ -53,10 +53,10 @@ struct reduction_fn { class SumOverAxisTask : public Task { public: - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto& input = context.inputs().at(0); - auto& ouptut = context.reductions().at(0); + auto input = context.input(0).data(); + auto ouptut = context.reduction(0).data(); legate::double_dispatch(input.dim(), input.code(), reduction_fn{}, ouptut, input); } diff --git a/examples/reduction/src/unique.cc b/examples/reduction/src/unique.cc index ab44360e0e..e85ebd7169 100644 --- a/examples/reduction/src/unique.cc +++ b/examples/reduction/src/unique.cc @@ -22,7 +22,7 @@ namespace reduction { namespace { template -void add_to_set(std::unordered_set& unique_values, legate::Store& input) +void add_to_set(std::unordered_set& unique_values, legate::Store input) { auto shape = input.shape<1>(); if (shape.empty()) return; @@ -32,7 +32,7 @@ void add_to_set(std::unordered_set& unique_values, legate::Store& input) } template -void copy_to_output(legate::Store& output, const std::unordered_set& values) +void copy_to_output(legate::Store output, const std::unordered_set& values) { if (values.empty()) { output.bind_empty_data(); @@ -51,19 +51,19 @@ constexpr bool is_supported = !(legate::is_floating_point::value || legate::is_complex::value); struct unique_fn { template >* = nullptr> - void operator()(legate::Store& output, std::vector& inputs) + void operator()(legate::Array& output, std::vector& inputs) { using VAL = legate::legate_type_of; std::unordered_set unique_values; // Find unique values across all inputs - for (auto& input : inputs) add_to_set(unique_values, input); + for (auto& input : inputs) add_to_set(unique_values, input.data()); // Copy the set of unique values to the output store - copy_to_output(output, unique_values); + copy_to_output(output.data(), unique_values); } template >* = nullptr> - void operator()(legate::Store& output, std::vector& inputs) + void operator()(legate::Array& output, std::vector& inputs) { LEGATE_ABORT; } @@ -73,11 +73,11 @@ struct unique_fn { class UniqueTask : public Task { public: - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto& inputs = context.inputs(); - auto& output = context.outputs().at(0); - legate::type_dispatch(output.code(), unique_fn{}, output, inputs); + auto inputs = context.inputs(); + auto output = context.output(0); + legate::type_dispatch(output.type().code(), unique_fn{}, output, inputs); } }; diff --git a/legate/core/_lib/types.pyx b/legate/core/_lib/types.pyx index 062e539e4f..a9f91bbb08 100644 --- a/legate/core/_lib/types.pyx +++ b/legate/core/_lib/types.pyx @@ -103,7 +103,7 @@ cdef extern from "core/type/detail/type_info.h" namespace "legate::detail" nogil ctypedef enum Code: pass int code - unsigned int size() + unsigned int size() except+ unsigned int alignment() int uid() bool variable_size() diff --git a/legate/core/launcher.py b/legate/core/launcher.py index d95350c7a2..bcd2d8d192 100644 --- a/legate/core/launcher.py +++ b/legate/core/launcher.py @@ -134,18 +134,18 @@ def __str__(self) -> str: class FutureStoreArg: def __init__( - self, store: Store, read_only: bool, has_storage: bool, redop: int + self, store: Store, read_only: bool, future_index: int, redop: int ) -> None: self._store = store self._read_only = read_only - self._has_storage = has_storage + self._future_index = future_index self._redop = redop def pack(self, buf: BufferBuilder) -> None: self._store.serialize(buf) buf.pack_32bit_int(self._redop) buf.pack_bool(self._read_only) - buf.pack_bool(self._has_storage) + buf.pack_32bit_int(self._future_index) buf.pack_32bit_uint(self._store.type.size) extents = self._store.extents buf.pack_32bit_uint(len(extents)) @@ -187,6 +187,19 @@ def __str__(self) -> str: return f"RegionFieldArg({self._dim}, {self._req}, {self._field_id})" +class ArrayAdaptor: + def __init__( + self, + arg: LauncherArg, + ) -> None: + self._arg = arg + + def pack(self, buf: BufferBuilder) -> None: + buf.pack_32bit_int(0) # array kind + self._arg.pack(buf) + buf.pack_bool(False) # nullable + + def pack_args( argbuf: BufferBuilder, args: Sequence[LauncherArg], @@ -787,11 +800,11 @@ def add_store( # current value whenever the store has a storage. (if this # was a write-only store, it would not have a storage yet, but # the inverse isn't true.) - has_storage = store.has_storage + future_index = -1 read_only = perm == Permission.READ - if has_storage: - self.add_future(store.storage) - args.append(FutureStoreArg(store, read_only, has_storage, redop)) + if store.has_storage: + future_index = self.add_future(store.storage) + args.append(FutureStoreArg(store, read_only, future_index, redop)) else: if TYPE_CHECKING: @@ -866,8 +879,10 @@ def add_unbound_output( ) ) - def add_future(self, future: Future) -> None: + def add_future(self, future: Future) -> int: + idx = len(self._future_args) self._future_args.append(future) + return idx def add_future_map(self, future_map: FutureMap) -> None: self._future_map_args.append(future_map) @@ -905,9 +920,9 @@ def build_task( self._req_analyzer.analyze_requirements() self._out_analyzer.analyze_requirements() - pack_args(argbuf, self._inputs) - pack_args(argbuf, self._outputs) - pack_args(argbuf, self._reductions) + pack_args(argbuf, [ArrayAdaptor(arg) for arg in self._inputs]) + pack_args(argbuf, [ArrayAdaptor(arg) for arg in self._outputs]) + pack_args(argbuf, [ArrayAdaptor(arg) for arg in self._reductions]) pack_args(argbuf, self._scalars) argbuf.pack_bool(self._can_raise_exception) argbuf.pack_bool(self._insert_barrier) @@ -949,9 +964,9 @@ def build_single_task(self, argbuf: BufferBuilder) -> SingleTask: self._req_analyzer.analyze_requirements() self._out_analyzer.analyze_requirements() - pack_args(argbuf, self._inputs) - pack_args(argbuf, self._outputs) - pack_args(argbuf, self._reductions) + pack_args(argbuf, [ArrayAdaptor(arg) for arg in self._inputs]) + pack_args(argbuf, [ArrayAdaptor(arg) for arg in self._outputs]) + pack_args(argbuf, [ArrayAdaptor(arg) for arg in self._reductions]) pack_args(argbuf, self._scalars) argbuf.pack_bool(self._can_raise_exception) diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index 21836b6558..269a5bed31 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -199,19 +199,26 @@ list(APPEND legate_core_SOURCES src/core/comm/comm_cpu.cc src/core/comm/coll.cc src/core/data/allocator.cc + src/core/data/array.cc + src/core/data/logical_array.cc src/core/data/logical_store.cc src/core/data/scalar.cc src/core/data/shape.cc src/core/data/store.cc + src/core/data/detail/array.cc + src/core/data/detail/array_tasks.cc + src/core/data/detail/logical_array.cc src/core/data/detail/logical_region_field.cc src/core/data/detail/logical_store.cc src/core/data/detail/scalar.cc src/core/data/detail/store.cc src/core/data/detail/transform.cc + src/core/mapping/array.cc src/core/mapping/machine.cc src/core/mapping/mapping.cc src/core/mapping/operation.cc src/core/mapping/store.cc + src/core/mapping/detail/array.cc src/core/mapping/detail/base_mapper.cc src/core/mapping/detail/core_mapper.cc src/core/mapping/detail/default_mapper.cc @@ -279,10 +286,16 @@ else() src/core/comm/local_comm.cc) endif() +if(Legion_USE_OpenMP) + list(APPEND legate_core_SOURCES + src/core/data/detail/array_tasks_omp.cc) +endif() + if(Legion_USE_CUDA) list(APPEND legate_core_SOURCES src/core/comm/comm_nccl.cu - src/core/cuda/stream_pool.cu) + src/core/cuda/stream_pool.cu + src/core/data/detail/array_tasks.cu) endif() add_library(legate_core ${legate_core_SOURCES}) @@ -379,11 +392,13 @@ if (legate_core_BUILD_DOCS) src/core/task/exception.h src/core/cuda/stream_pool.h # data + src/core/data/array.h src/core/data/store.h src/core/data/scalar.h src/core/data/buffer.h src/core/utilities/span.h src/core/data/allocator.h + src/core/data/logical_array.h src/core/data/logical_store.h # runtime src/core/runtime/library.h @@ -461,7 +476,10 @@ install( install( FILES src/core/data/allocator.h + src/core/data/array.h + src/core/data/array.inl src/core/data/buffer.h + src/core/data/logical_array.h src/core/data/logical_store.h src/core/data/scalar.h src/core/data/scalar.inl @@ -472,7 +490,8 @@ install( DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/legate/core/data) install( - FILES src/core/mapping/machine.h + FILES src/core/mapping/array.h + src/core/mapping/machine.h src/core/mapping/mapping.h src/core/mapping/operation.h src/core/mapping/store.h diff --git a/src/core/cuda/cuda_help.h b/src/core/cuda/cuda_help.h index b7171e96ec..fe81370871 100644 --- a/src/core/cuda/cuda_help.h +++ b/src/core/cuda/cuda_help.h @@ -16,6 +16,7 @@ #include #include +#define THREADS_PER_BLOCK 128 #define CHECK_CUDA(expr) \ do { \ cudaError_t __result__ = (expr); \ diff --git a/src/core/data/accessors.h b/src/core/data/accessors.h new file mode 100644 index 0000000000..8491b71764 --- /dev/null +++ b/src/core/data/accessors.h @@ -0,0 +1,84 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#pragma once + +#include "core/data/array.h" +#include "core/utilities/span.h" +#include "core/utilities/typedefs.h" + +/** + * @file + * @brief Class definitions for array accessors + */ + +namespace legate { + +template +class ListArrayAccessor; + +template +class ListArrayAccessor { + public: + ListArrayAccessor(ListArray array); + virtual ~ListArrayAccessor(); + + public: + Span operator[](const Point<1>& p); + + private: + AccessorRO, 1> desc_acc_; + AccessorRO vardata_acc_; +}; + +template +class ListArrayAccessor { + public: + ListArrayAccessor(ListArray array); + virtual ~ListArrayAccessor(); + + public: + void insert(const legate::Span& value); + + private: + void check_overflow(); + + private: + Rect<1> desc_shape_; + AccessorWO, 1> desc_acc_; + Store vardata_store_; + std::vector> values_{}; +}; + +template +class StringArrayAccessor; + +template <> +struct StringArrayAccessor : public ListArrayAccessor { + StringArrayAccessor(StringArray array); + + using ListArrayAccessor::operator[]; + std::string_view operator[](const Point<1>& p); +}; + +template <> +struct StringArrayAccessor : public ListArrayAccessor { + StringArrayAccessor(StringArray array); + + using ListArrayAccessor::insert; + void insert(const std::string& value); + void insert(const std::string_view& value); +}; + +} // namespace legate + +#include "core/data/accessors.inl" diff --git a/src/core/data/accessors.inl b/src/core/data/accessors.inl new file mode 100644 index 0000000000..fd0f175f03 --- /dev/null +++ b/src/core/data/accessors.inl @@ -0,0 +1,116 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#pragma once + +// Useful for IDEs +#include "core/data/accessors.h" + +namespace legate { + +template +ListArrayAccessor::ListArrayAccessor(ListArray array) +{ + desc_acc_ = array.descriptor().data().read_accessor, 1>(); + vardata_acc_ = array.vardata().data().read_accessor(); +} + +template +ListArrayAccessor::~ListArrayAccessor() +{ +} + +template +Span ListArrayAccessor::operator[](const Point<1>& p) +{ + auto& desc = desc_acc_[p]; + return Span(vardata_acc_.ptr(desc.lo), desc.volume()); +} + +template +ListArrayAccessor::ListArrayAccessor(ListArray array) +{ + desc_shape_ = array.shape<1>(); + desc_acc_ = array.descriptor().data().write_accessor, 1>(); + vardata_store_ = array.vardata().data(); +} + +template +ListArrayAccessor::~ListArrayAccessor() +{ + if (desc_shape_.empty()) { + vardata_store_.bind_empty_data(); + return; + } + + int64_t vardata_size = 0; + for (auto& value : values_) { vardata_size += value.size(); } + auto buffer = vardata_store_.create_output_buffer(Point<1>(vardata_size), true); + auto* p_buffer = buffer.ptr(0); + int64_t vardata_pos = 0; + int64_t desc_pos = desc_shape_.lo[0]; + for (auto& value : values_) { + auto len = value.size(); + memcpy(p_buffer, value.data(), len * sizeof(VAL)); + p_buffer += len; + + auto& desc = desc_acc_[desc_pos++]; + desc.lo[0] = vardata_pos; + desc.hi[0] = vardata_pos + len - 1; + vardata_pos += len; + } +} + +template +void ListArrayAccessor::insert(const legate::Span& value) +{ + check_overflow(); + values_.emplace_back(value.begin(), value.end()); +} + +template +void ListArrayAccessor::check_overflow() +{ + if (values_.size() >= desc_shape_.volume()) { + throw std::out_of_range("No space left in the array"); + } +} + +StringArrayAccessor::StringArrayAccessor(StringArray array) + : ListArrayAccessor(array.as_list_array()) +{ +} + +std::string_view StringArrayAccessor::operator[](const Point<1>& p) +{ + auto span = ListArrayAccessor::operator[](p); + return std::string_view(reinterpret_cast(span.ptr()), span.size()); +} + +StringArrayAccessor::StringArrayAccessor(StringArray array) + : ListArrayAccessor(array.as_list_array()) +{ +} + +void StringArrayAccessor::insert(const std::string& value) +{ + ListArrayAccessor::insert( + legate::Span(reinterpret_cast(value.data()), value.size())); +} + +void StringArrayAccessor::insert(const std::string_view& value) +{ + ListArrayAccessor::insert( + legate::Span(reinterpret_cast(value.data()), value.size())); +} + +} // namespace legate diff --git a/src/core/data/array.cc b/src/core/data/array.cc new file mode 100644 index 0000000000..0389dff1b9 --- /dev/null +++ b/src/core/data/array.cc @@ -0,0 +1,88 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include "core/data/array.h" +#include "core/data/detail/array.h" + +namespace legate { + +bool Array::nullable() const { return impl_->nullable(); } + +int32_t Array::dim() const { return impl_->dim(); } + +Type Array::type() const { return Type(impl_->type()); } + +bool Array::nested() const { return impl_->nested(); } + +Store Array::data() const { return Store(impl_->data()); } + +Store Array::null_mask() const { return Store(impl_->null_mask()); } + +Array Array::child(uint32_t index) const { return Array(impl_->child(index)); } + +Domain Array::domain() const { return impl_->domain(); } + +void Array::check_shape_dimension(const int32_t dim) const { impl_->check_shape_dimension(dim); } + +ListArray Array::as_list_array() const +{ + if (impl_->kind() != detail::ArrayKind::LIST) { + throw std::invalid_argument("Array is not a list array"); + } + return ListArray(impl_); +} + +StringArray Array::as_string_array() const +{ + if (type().code() != Type::Code::STRING) { + throw std::invalid_argument("Array is not a string array"); + } + return StringArray(impl_); +} + +Array::Array(std::shared_ptr impl) : impl_(std::move(impl)) {} + +Array::Array(const Array&) = default; + +Array& Array::operator=(const Array&) = default; + +Array::Array(Array&&) = default; + +Array& Array::operator=(Array&&) = default; + +Array::~Array() {} + +ListArray::ListArray(std::shared_ptr impl) : Array(std::move(impl)) {} + +Array ListArray::descriptor() const +{ + return Array(static_cast(impl_.get())->descriptor()); +} + +Array ListArray::vardata() const +{ + return Array(static_cast(impl_.get())->vardata()); +} + +StringArray::StringArray(std::shared_ptr impl) : Array(std::move(impl)) {} + +Array StringArray::ranges() const +{ + return Array(static_cast(impl_.get())->descriptor()); +} + +Array StringArray::chars() const +{ + return Array(static_cast(impl_.get())->vardata()); +} + +} // namespace legate diff --git a/src/core/data/array.h b/src/core/data/array.h new file mode 100644 index 0000000000..64f4aa2e41 --- /dev/null +++ b/src/core/data/array.h @@ -0,0 +1,185 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#pragma once + +#include "core/data/store.h" +#include "core/utilities/typedefs.h" + +/** + * @file + * @brief Class definition for legate::Array + */ + +namespace legate { + +namespace detail { +class Array; +} // namespace detail + +class ListArray; +class StringArray; + +class Array { + public: + /** + * @brief Indicates if the array is nullable + * + * @return true If the array is nullable + * @return false Otherwise + */ + bool nullable() const; + /** + * @brief Returns the dimension of the array + * + * @return Array's dimension + */ + int32_t dim() const; + /** + * @brief Returns the array's type + * + * @return Type + */ + Type type() const; + /** + * @brief Indicates if the array has child arrays + * + * @return true If the array has child arrays + * @return false Otherwise + */ + bool nested() const; + + public: + /** + * @brief Returns the store containing the array's data + * + * @return Store + * + * @throw std::invalid_argument If the array is not a base array + */ + Store data() const; + /** + * @brief Returns the store containing the array's null mask + * + * @return Store + * + * @throw std::invalid_argument If the array is not nullable + */ + Store null_mask() const; + /** + * @brief Returns the sub-array of a given index + * + * @param index Sub-array index + * + * @return Array + * + * @throw std::invalid_argument If the array has no child arrays + * @throw std::out_of_range If the index is out of range + */ + Array child(uint32_t index) const; + + public: + /** + * @brief Returns the array's domain + * + * @return Array's domain + */ + template + Rect shape() const; + /** + * @brief Returns the array's domain in a dimension-erased domain type + * + * @return Array's domain in a dimension-erased domain type + */ + Domain domain() const; + + public: + /** + * @brief Casts this array as a list array + * + * @return List array + * + * @throw std::invalid_argument If the array is not a list array + */ + ListArray as_list_array() const; + /** + * @brief Casts this array as a string array + * + * @return String array + * + * @throw std::invalid_argument If the array is not a string array + */ + StringArray as_string_array() const; + + private: + void check_shape_dimension(const int32_t dim) const; + + public: + Array(std::shared_ptr impl); + std::shared_ptr impl() const { return impl_; } + + public: + Array(const Array&); + Array& operator=(const Array&); + Array(Array&&); + Array& operator=(Array&&); + + public: + virtual ~Array(); + + protected: + std::shared_ptr impl_{nullptr}; +}; + +class ListArray : public Array { + public: + /** + * @brief Returns the sub-array for descriptors + * + * @return Store + */ + Array descriptor() const; + /** + * @brief Returns the sub-array for variable size data + * + * @return Store + */ + Array vardata() const; + + private: + friend class Array; + ListArray(std::shared_ptr impl); +}; + +class StringArray : public Array { + public: + /** + * @brief Returns the sub-array for ranges + * + * @return Store + */ + Array ranges() const; + /** + * @brief Returns the sub-array for characters + * + * @return Store + */ + Array chars() const; + + private: + friend class Array; + StringArray(std::shared_ptr impl); +}; + +} // namespace legate + +#include "core/data/array.inl" diff --git a/src/core/data/array.inl b/src/core/data/array.inl new file mode 100644 index 0000000000..2c7d4919b0 --- /dev/null +++ b/src/core/data/array.inl @@ -0,0 +1,32 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#pragma once + +// Useful for IDEs +#include "core/data/array.h" + +namespace legate { + +template +Rect Array::shape() const +{ + check_shape_dimension(DIM); + if (dim() > 0) { + return domain().bounds(); + } else { + auto p = Point::ZEROES(); + return Rect(p, p); + } +} + +} // namespace legate diff --git a/src/core/data/detail/array.cc b/src/core/data/detail/array.cc new file mode 100644 index 0000000000..37e73e35a7 --- /dev/null +++ b/src/core/data/detail/array.cc @@ -0,0 +1,156 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include "core/data/detail/array.h" + +namespace legate::detail { + +std::shared_ptr Array::data() const +{ + throw std::invalid_argument("Data store of a nested array cannot be retrieved"); + return nullptr; +} + +BaseArray::BaseArray(std::shared_ptr data, std::shared_ptr null_mask) + : data_(std::move(data)), null_mask_(std::move(null_mask)) +{ +} + +bool BaseArray::unbound() const +{ +#ifdef DEBUG_LEGATE + assert(!nullable() || data_->is_unbound_store() == null_mask_->is_unbound_store()); +#endif + return data_->is_unbound_store(); +} + +bool BaseArray::valid() const { return data_->valid(); } + +std::shared_ptr BaseArray::null_mask() const +{ + if (!nullable()) { + throw std::invalid_argument("Invalid to retrieve the null mask of a non-nullable array"); + } + return null_mask_; +} + +std::shared_ptr BaseArray::child(uint32_t index) const +{ + throw std::invalid_argument("Non-nested array has no child sub-array"); + return nullptr; +} + +void BaseArray::_stores(std::vector>& result) const +{ + result.push_back(data_); + if (nullable()) result.push_back(null_mask_); +} + +Domain BaseArray::domain() const { return data_->domain(); } + +void BaseArray::check_shape_dimension(const int32_t dim) const +{ + return data_->check_shape_dimension(dim); +} + +ListArray::ListArray(std::shared_ptr type, + std::shared_ptr descriptor, + std::shared_ptr vardata) + : type_(std::move(type)), descriptor_(std::move(descriptor)), vardata_(std::move(vardata)) +{ +} + +int32_t ListArray::dim() const { return descriptor_->dim(); } + +bool ListArray::unbound() const { return descriptor_->unbound() || vardata_->unbound(); } + +bool ListArray::valid() const +{ +#ifdef DEBUG_LEGATE + assert(descriptor_->valid() == vardata_->valid()); +#endif + return descriptor_->valid(); +} + +std::shared_ptr ListArray::child(uint32_t index) const +{ + switch (index) { + case 0: return descriptor_; + case 1: return vardata_; + default: { + throw std::out_of_range("List array does not have child " + std::to_string(index)); + break; + } + } + return nullptr; +} + +void ListArray::_stores(std::vector>& result) const +{ + descriptor_->_stores(result); + vardata_->_stores(result); +} + +Domain ListArray::domain() const { return descriptor_->domain(); } + +void ListArray::check_shape_dimension(const int32_t dim) const +{ + descriptor_->check_shape_dimension(dim); +} + +StructArray::StructArray(std::shared_ptr type, + std::shared_ptr null_mask, + std::vector>&& fields) + : type_(std::move(type)), null_mask_(std::move(null_mask)), fields_(std::move(fields)) +{ +} + +int32_t StructArray::dim() const { return fields_.front()->dim(); } + +bool StructArray::unbound() const +{ + return std::any_of(fields_.begin(), fields_.end(), [](auto& field) { return field->unbound(); }); +} + +bool StructArray::valid() const +{ + auto result = + std::all_of(fields_.begin(), fields_.end(), [](auto& field) { return field->valid(); }); +#ifdef DEBUG_LEGATE + assert(null_mask_->valid() == result); +#endif + return result; +} + +std::shared_ptr StructArray::null_mask() const +{ + if (!nullable()) { + throw std::invalid_argument("Invalid to retrieve the null mask of a non-nullable array"); + } + return null_mask_; +} + +std::shared_ptr StructArray::child(uint32_t index) const { return fields_.at(index); } + +void StructArray::_stores(std::vector>& result) const +{ + std::for_each(fields_.begin(), fields_.end(), [&result](auto& field) { field->_stores(result); }); +} + +Domain StructArray::domain() const { return fields_.front()->domain(); } + +void StructArray::check_shape_dimension(const int32_t dim) const +{ + fields_.front()->check_shape_dimension(dim); +} + +} // namespace legate::detail diff --git a/src/core/data/detail/array.h b/src/core/data/detail/array.h new file mode 100644 index 0000000000..17cccd06f1 --- /dev/null +++ b/src/core/data/detail/array.h @@ -0,0 +1,158 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#pragma once + +#include "core/data/detail/array.h" +#include "core/data/detail/array_kind.h" +#include "core/data/detail/store.h" + +namespace legate { +class Array; +} // namespace legate + +namespace legate::detail { + +struct Array { + virtual ~Array() {} + virtual int32_t dim() const = 0; + virtual ArrayKind kind() const = 0; + virtual std::shared_ptr type() const = 0; + virtual bool unbound() const = 0; + virtual bool nullable() const = 0; + virtual bool nested() const = 0; + virtual bool valid() const = 0; + + virtual std::shared_ptr data() const; + virtual std::shared_ptr null_mask() const = 0; + virtual std::shared_ptr child(uint32_t index) const = 0; + std::vector> stores() const + { + std::vector> result; + _stores(result); + return result; + } + virtual void _stores(std::vector>& result) const = 0; + + virtual Domain domain() const = 0; + virtual void check_shape_dimension(const int32_t dim) const = 0; +}; + +class BaseArray : public Array { + public: + BaseArray(std::shared_ptr data, std::shared_ptr null_mask); + + public: + BaseArray(const BaseArray& other) = default; + BaseArray& operator=(const BaseArray& other) = default; + BaseArray(BaseArray&& other) = default; + BaseArray& operator=(BaseArray&& other) = default; + + public: + int32_t dim() const override { return data_->dim(); } + ArrayKind kind() const override { return ArrayKind::BASE; } + std::shared_ptr type() const override { return data_->type(); } + bool unbound() const override; + bool nullable() const override { return null_mask_ != nullptr; } + bool nested() const override { return false; } + bool valid() const override; + + public: + std::shared_ptr data() const override { return data_; } + std::shared_ptr null_mask() const override; + std::shared_ptr child(uint32_t index) const override; + void _stores(std::vector>& result) const override; + + public: + virtual Domain domain() const override; + virtual void check_shape_dimension(const int32_t dim) const override; + + private: + std::shared_ptr data_; + std::shared_ptr null_mask_; +}; + +class ListArray : public Array { + public: + ListArray(std::shared_ptr type, + std::shared_ptr descriptor, + std::shared_ptr vardata); + + public: + ListArray(const ListArray& other) = default; + ListArray& operator=(const ListArray& other) = default; + ListArray(ListArray&& other) = default; + ListArray& operator=(ListArray&& other) = default; + + public: + int32_t dim() const override; + ArrayKind kind() const override { return ArrayKind::LIST; } + std::shared_ptr type() const override { return type_; } + bool unbound() const override; + bool nullable() const override { return vardata_->nullable(); } + bool nested() const override { return true; } + bool valid() const override; + + public: + std::shared_ptr null_mask() const override { return descriptor_->null_mask(); } + std::shared_ptr child(uint32_t index) const override; + void _stores(std::vector>& result) const override; + std::shared_ptr descriptor() const { return descriptor_; } + std::shared_ptr vardata() const { return vardata_; } + + public: + virtual Domain domain() const override; + virtual void check_shape_dimension(const int32_t dim) const override; + + private: + std::shared_ptr type_; + std::shared_ptr descriptor_; + std::shared_ptr vardata_; +}; + +class StructArray : public Array { + public: + StructArray(std::shared_ptr type, + std::shared_ptr null_mask, + std::vector>&& fields); + + public: + StructArray(const StructArray& other) = default; + StructArray& operator=(const StructArray& other) = default; + StructArray(StructArray&& other) = default; + StructArray& operator=(StructArray&& other) = default; + + public: + int32_t dim() const override; + ArrayKind kind() const override { return ArrayKind::STRUCT; } + std::shared_ptr type() const override { return type_; } + bool unbound() const override; + bool nullable() const override { return null_mask_ != nullptr; } + bool nested() const override { return true; } + bool valid() const override; + + public: + std::shared_ptr null_mask() const override; + std::shared_ptr child(uint32_t index) const override; + void _stores(std::vector>& result) const override; + + public: + virtual Domain domain() const override; + virtual void check_shape_dimension(const int32_t dim) const override; + + private: + std::shared_ptr type_; + std::shared_ptr null_mask_; + std::vector> fields_; +}; + +} // namespace legate::detail diff --git a/src/core/data/detail/array_kind.h b/src/core/data/detail/array_kind.h new file mode 100644 index 0000000000..7316c377d5 --- /dev/null +++ b/src/core/data/detail/array_kind.h @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#pragma once + +#include + +namespace legate::detail { + +enum class ArrayKind : int32_t { + BASE = 0, + LIST = 1, + STRUCT = 2, +}; + +} // namespace legate::detail diff --git a/src/core/data/detail/array_tasks.cc b/src/core/data/detail/array_tasks.cc new file mode 100644 index 0000000000..a45e86dfa8 --- /dev/null +++ b/src/core/data/detail/array_tasks.cc @@ -0,0 +1,92 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include "core/data/detail/array_tasks.h" +#include "core/legate_c.h" +#include "core/task/task_context.h" + +namespace legate::detail { + +/*static*/ void FixupRanges::cpu_variant(legate::TaskContext context) +{ + if (context.get_task_index()[0] == 0) return; + + auto outputs = context.outputs(); + // TODO: We need to extend this to nested cases + for (auto& output : outputs) { + auto list_arr = output.as_list_array(); + + auto desc = list_arr.descriptor(); + auto desc_shape = desc.shape<1>(); + if (desc_shape.empty()) continue; + + auto vardata_lo = list_arr.vardata().shape<1>().lo; + auto desc_acc = desc.data().read_write_accessor, 1>(); + + for (int64_t idx = desc_shape.lo[0]; idx <= desc_shape.hi[0]; ++idx) { + auto& desc = desc_acc[idx]; + desc.lo += vardata_lo; + desc.hi += vardata_lo; + } + } +} + +/*static*/ void OffsetsToRanges::cpu_variant(legate::TaskContext context) +{ + auto offsets = context.input(0).data(); + auto vardata = context.input(1).data(); + auto ranges = context.output(0).data(); + + auto shape = offsets.shape<1>(); + assert(shape == ranges.shape<1>()); + + if (shape.empty()) return; + + auto vardata_shape = vardata.shape<1>(); + auto vardata_lo = vardata_shape.lo[0]; + + auto offsets_acc = offsets.read_accessor(); + auto ranges_acc = ranges.write_accessor, 1>(); + for (int64_t idx = shape.lo[0]; idx < shape.hi[0]; ++idx) { + ranges_acc[idx].lo[0] = vardata_lo + offsets_acc[idx]; + ranges_acc[idx].hi[0] = vardata_lo + offsets_acc[idx + 1] - 1; + } + ranges_acc[shape.hi].lo[0] = vardata_lo + offsets_acc[shape.hi]; + ranges_acc[shape.hi].hi[0] = vardata_lo + static_cast(vardata_shape.volume()) - 1; +} + +/*static*/ void RangesToOffsets::cpu_variant(legate::TaskContext context) +{ + auto ranges = context.input(0).data(); + auto offsets = context.output(0).data(); + + auto shape = ranges.shape<1>(); + assert(shape == offsets.shape<1>()); + + if (shape.empty()) return; + + auto ranges_acc = ranges.read_accessor, 1>(); + auto offsets_acc = offsets.write_accessor(); + auto lo = ranges_acc[shape.lo].lo[0]; + for (int64_t idx = shape.lo[0]; idx <= shape.hi[0]; ++idx) { + offsets_acc[idx] = ranges_acc[idx].lo[0] - lo; + } +} + +void register_array_tasks(Library* core_lib) +{ + FixupRanges::register_variants(legate::Library(core_lib), LEGATE_CORE_FIXUP_RANGES); + OffsetsToRanges::register_variants(legate::Library(core_lib), LEGATE_CORE_OFFSETS_TO_RANGES); + RangesToOffsets::register_variants(legate::Library(core_lib), LEGATE_CORE_RANGES_TO_OFFSETS); +} + +} // namespace legate::detail diff --git a/src/core/data/detail/array_tasks.cu b/src/core/data/detail/array_tasks.cu new file mode 100644 index 0000000000..aa3a846a24 --- /dev/null +++ b/src/core/data/detail/array_tasks.cu @@ -0,0 +1,151 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include "core/data/detail/array_tasks.h" + +#include "core/cuda/cuda_help.h" +#include "core/cuda/stream_pool.h" +#include "core/task/task_context.h" + +namespace legate::detail { + +namespace { + +__device__ inline size_t global_tid_1d() +{ + return static_cast(blockIdx.x) * blockDim.x + threadIdx.x; +} + +template +__global__ void fixup_ranges(size_t desc_volume, + Point<1> desc_lo, + Point<1> vardata_lo, + DescAcc desc_acc) +{ + auto tid = global_tid_1d(); + if (tid >= desc_volume) return; + auto& desc = desc_acc[desc_lo + tid]; + desc.lo += vardata_lo; + desc.hi += vardata_lo; +} + +} // namespace + +/*static*/ void FixupRanges::gpu_variant(legate::TaskContext context) +{ + if (context.get_task_index()[0] == 0) return; + + auto outputs = context.outputs(); + auto stream = legate::cuda::StreamPool::get_stream_pool().get_stream(); + + // TODO: We need to extend this to nested cases + for (auto& output : outputs) { + auto list_arr = output.as_list_array(); + + auto desc = list_arr.descriptor(); + auto desc_shape = desc.shape<1>(); + if (desc_shape.empty()) continue; + + auto vardata_lo = list_arr.vardata().shape<1>().lo; + auto desc_acc = desc.data().read_write_accessor, 1>(); + + size_t desc_volume = desc_shape.volume(); + auto num_blocks = (desc_volume + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + fixup_ranges<<>>( + desc_volume, desc_shape.lo, vardata_lo, desc_acc); + } +} +namespace { + +template +__global__ void offsets_to_ranges(size_t offsets_volume, + int64_t vardata_volume, + Point<1> offsets_lo, + Point<1> vardata_lo, + RangesAcc ranges_acc, + OffsetsAcc offsets_acc) +{ + auto tid = global_tid_1d(); + if (tid >= offsets_volume) return; + auto p = offsets_lo + tid; + auto& range = ranges_acc[p]; + range.lo[0] = vardata_lo + offsets_acc[p]; + range.hi[0] = vardata_lo + (tid != offsets_volume - 1 ? offsets_acc[p + 1] : vardata_volume) - 1; +} + +} // namespace + +/*static*/ void OffsetsToRanges::gpu_variant(legate::TaskContext context) +{ + auto offsets = context.input(0).data(); + auto vardata = context.input(1).data(); + auto ranges = context.output(0).data(); + + auto offsets_shape = offsets.shape<1>(); + assert(offsets_shape == ranges.shape<1>()); + + if (offsets_shape.empty()) return; + + auto vardata_shape = vardata.shape<1>(); + auto vardata_lo = vardata_shape.lo[0]; + + auto offsets_acc = offsets.read_accessor(); + auto ranges_acc = ranges.write_accessor, 1>(); + + auto stream = legate::cuda::StreamPool::get_stream_pool().get_stream(); + + size_t offsets_volume = offsets_shape.volume(); + size_t vardata_volume = vardata_shape.volume(); + + auto num_blocks = (offsets_volume + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + offsets_to_ranges<<>>( + offsets_volume, vardata_volume, offsets_shape.lo, vardata_shape.lo, ranges_acc, offsets_acc); +} + +namespace { + +template +__global__ void ranges_to_offsets(size_t ranges_volume, + Point<1> ranges_lo, + OffsetsAcc offsets_acc, + RangesAcc ranges_acc) +{ + auto tid = global_tid_1d(); + if (tid >= ranges_volume) return; + auto p = ranges_lo + tid; + offsets_acc[p] = ranges_acc[p].lo[0] - ranges_acc[ranges_lo].lo[0]; +} + +} // namespace + +/*static*/ void RangesToOffsets::gpu_variant(legate::TaskContext context) +{ + auto ranges = context.input(0).data(); + auto offsets = context.output(0).data(); + + auto ranges_shape = ranges.shape<1>(); + assert(ranges_shape == offsets.shape<1>()); + + if (ranges_shape.empty()) return; + + auto ranges_acc = ranges.read_accessor, 1>(); + auto offsets_acc = offsets.write_accessor(); + + auto stream = legate::cuda::StreamPool::get_stream_pool().get_stream(); + + auto ranges_volume = ranges_shape.volume(); + auto num_blocks = (ranges_volume + THREADS_PER_BLOCK - 1) / THREADS_PER_BLOCK; + ranges_to_offsets<<>>( + ranges_volume, ranges_shape.lo, offsets_acc, ranges_acc); +} + +} // namespace legate::detail diff --git a/src/core/data/detail/array_tasks.h b/src/core/data/detail/array_tasks.h new file mode 100644 index 0000000000..ac0ed45db3 --- /dev/null +++ b/src/core/data/detail/array_tasks.h @@ -0,0 +1,51 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include "core/task/task.h" + +namespace legate::detail { + +class Library; + +struct FixupRanges : public LegateTask { + static void cpu_variant(legate::TaskContext context); +#ifdef LEGATE_USE_OPENMP + static void omp_variant(legate::TaskContext context); +#endif +#ifdef LEGATE_USE_CUDA + static void gpu_variant(legate::TaskContext context); +#endif +}; + +struct OffsetsToRanges : public LegateTask { + static void cpu_variant(legate::TaskContext context); +#ifdef LEGATE_USE_OPENMP + static void omp_variant(legate::TaskContext context); +#endif +#ifdef LEGATE_USE_CUDA + static void gpu_variant(legate::TaskContext context); +#endif +}; + +struct RangesToOffsets : public LegateTask { + static void cpu_variant(legate::TaskContext context); +#ifdef LEGATE_USE_OPENMP + static void omp_variant(legate::TaskContext context); +#endif +#ifdef LEGATE_USE_CUDA + static void gpu_variant(legate::TaskContext context); +#endif +}; + +void register_array_tasks(Library* core_lib); + +} // namespace legate::detail diff --git a/src/core/data/detail/array_tasks_omp.cc b/src/core/data/detail/array_tasks_omp.cc new file mode 100644 index 0000000000..c64867de1f --- /dev/null +++ b/src/core/data/detail/array_tasks_omp.cc @@ -0,0 +1,88 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include "core/data/detail/array_tasks.h" +#include "core/legate_c.h" +#include "core/task/task_context.h" + +namespace legate::detail { + +/*static*/ void FixupRanges::omp_variant(legate::TaskContext context) +{ + if (context.get_task_index()[0] == 0) return; + + auto outputs = context.outputs(); + // TODO: We need to extend this to nested cases + for (auto& output : outputs) { + auto list_arr = output.as_list_array(); + + auto desc = list_arr.descriptor(); + auto desc_shape = desc.shape<1>(); + if (desc_shape.empty()) continue; + + auto vardata_lo = list_arr.vardata().shape<1>().lo; + auto desc_acc = desc.data().read_write_accessor, 1>(); + +#pragma omp parallel for schedule(static) + for (int64_t idx = desc_shape.lo[0]; idx <= desc_shape.hi[0]; ++idx) { + auto& desc = desc_acc[idx]; + desc.lo += vardata_lo; + desc.hi += vardata_lo; + } + } +} + +/*static*/ void OffsetsToRanges::omp_variant(legate::TaskContext context) +{ + auto offsets = context.input(0).data(); + auto vardata = context.input(1).data(); + auto ranges = context.output(0).data(); + + auto shape = offsets.shape<1>(); + assert(shape == ranges.shape<1>()); + + if (shape.empty()) return; + + auto vardata_shape = vardata.shape<1>(); + auto vardata_lo = vardata_shape.lo[0]; + + auto offsets_acc = offsets.read_accessor(); + auto ranges_acc = ranges.write_accessor, 1>(); +#pragma omp parallel for schedule(static) + for (int64_t idx = shape.lo[0]; idx < shape.hi[0]; ++idx) { + ranges_acc[idx].lo[0] = vardata_lo + offsets_acc[idx]; + ranges_acc[idx].hi[0] = vardata_lo + offsets_acc[idx + 1] - 1; + } + ranges_acc[shape.hi].lo[0] = vardata_lo + offsets_acc[shape.hi]; + ranges_acc[shape.hi].hi[0] = vardata_lo + static_cast(vardata_shape.volume()) - 1; +} + +/*static*/ void RangesToOffsets::omp_variant(legate::TaskContext context) +{ + auto ranges = context.input(0).data(); + auto offsets = context.output(0).data(); + + auto shape = ranges.shape<1>(); + assert(shape == offsets.shape<1>()); + + if (shape.empty()) return; + + auto ranges_acc = ranges.read_accessor, 1>(); + auto offsets_acc = offsets.write_accessor(); + auto lo = ranges_acc[shape.lo].lo[0]; +#pragma omp parallel for schedule(static) + for (int64_t idx = shape.lo[0]; idx <= shape.hi[0]; ++idx) { + offsets_acc[idx] = ranges_acc[idx].lo[0] - lo; + } +} + +} // namespace legate::detail diff --git a/src/core/data/detail/logical_array.cc b/src/core/data/detail/logical_array.cc new file mode 100644 index 0000000000..48566984c7 --- /dev/null +++ b/src/core/data/detail/logical_array.cc @@ -0,0 +1,451 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include "core/data/detail/logical_array.h" +#include "core/data/detail/array.h" +#include "core/operation/detail/launcher_arg.h" +#include "core/operation/detail/task.h" +#include "core/partitioning/detail/constraint.h" +#include "core/partitioning/detail/constraint_solver.h" + +namespace legate::detail { + +std::shared_ptr LogicalArray::data() const +{ + throw std::invalid_argument("Data store of a nested array cannot be retrieved"); + return nullptr; +} + +BaseLogicalArray::BaseLogicalArray(std::shared_ptr data, + std::shared_ptr null_mask) + : data_(std::move(data)), null_mask_(std::move(null_mask)) +{ + assert(data_ != nullptr); +} + +bool BaseLogicalArray::unbound() const +{ +#ifdef DEBUG_LEGATE + assert(!nullable() || data_->unbound() == null_mask_->unbound()); +#endif + return data_->unbound(); +} + +std::shared_ptr BaseLogicalArray::promote(int32_t extra_dim, size_t dim_size) const +{ + auto null_mask = nullable() ? null_mask_->promote(extra_dim, dim_size) : nullptr; + auto data = data_->promote(extra_dim, dim_size); + return std::make_shared(std::move(data), std::move(null_mask)); +} + +std::shared_ptr BaseLogicalArray::project(int32_t dim, int64_t index) const +{ + auto null_mask = nullable() ? null_mask_->project(dim, index) : nullptr; + auto data = data_->project(dim, index); + return std::make_shared(std::move(data), std::move(null_mask)); +} + +std::shared_ptr BaseLogicalArray::slice(int32_t dim, Slice sl) const +{ + auto null_mask = nullable() ? null_mask_->slice(dim, sl) : nullptr; + auto data = data_->slice(dim, sl); + return std::make_shared(std::move(data), std::move(null_mask)); +} + +std::shared_ptr BaseLogicalArray::transpose(const std::vector& axes) const +{ + auto null_mask = nullable() ? null_mask_->transpose(axes) : nullptr; + auto data = data_->transpose(axes); + return std::make_shared(std::move(data), std::move(null_mask)); +} + +std::shared_ptr BaseLogicalArray::delinearize(int32_t dim, + const std::vector& sizes) const +{ + auto null_mask = nullable() ? null_mask_->delinearize(dim, sizes) : nullptr; + auto data = data_->delinearize(dim, sizes); + return std::make_shared(std::move(data), std::move(null_mask)); +} + +std::shared_ptr BaseLogicalArray::null_mask() const +{ + if (!nullable()) + throw std::invalid_argument("Invalid to retrieve the null mask of a non-nullable array"); + return null_mask_; +} + +std::shared_ptr BaseLogicalArray::get_physical_array() const +{ + return _get_physical_array(); +} + +std::shared_ptr BaseLogicalArray::_get_physical_array() const +{ + auto data_store = data_->get_physical_store(); + std::shared_ptr null_mask_store = nullptr; + if (null_mask_ != nullptr) { null_mask_store = null_mask_->get_physical_store(); } + return std::make_shared(std::move(data_store), std::move(null_mask_store)); +} + +std::shared_ptr BaseLogicalArray::child(uint32_t index) const +{ + throw std::invalid_argument("Non-nested array has no child sub-array"); + return nullptr; +} + +void BaseLogicalArray::record_scalar_or_unbound_outputs(AutoTask* task) const +{ + if (data_->unbound()) + task->record_unbound_output(data_); + else if (data_->has_scalar_storage()) + task->record_scalar_output(data_); + + if (!nullable()) return; + + if (null_mask_->unbound()) + task->record_unbound_output(null_mask_); + else if (null_mask_->has_scalar_storage()) + task->record_scalar_output(null_mask_); +} + +void BaseLogicalArray::record_scalar_reductions(AutoTask* task, Legion::ReductionOpID redop) const +{ + if (data_->has_scalar_storage()) { task->record_scalar_reduction(data_, redop); } + if (nullable() && null_mask_->has_scalar_storage()) { + auto null_redop = bool_()->find_reduction_operator(ReductionOpKind::MUL); + task->record_scalar_reduction(null_mask_, null_redop); + } +} + +void BaseLogicalArray::generate_constraints( + AutoTask* task, + std::map, const Variable*>& mapping, + const Variable* partition_symbol) const +{ + mapping.insert({data_, partition_symbol}); + + if (!nullable()) return; + auto part_null_mask = task->declare_partition(); + mapping.insert({null_mask_, part_null_mask}); + task->add_constraint(align(partition_symbol, part_null_mask)); +} + +std::unique_ptr BaseLogicalArray::to_launcher_arg( + const std::map, const Variable*>& mapping, + const Strategy& strategy, + const Domain* launch_domain, + Legion::PrivilegeMode privilege, + int32_t redop) const +{ + auto data_arg = + data_->to_launcher_arg(mapping.at(data_), strategy, launch_domain, privilege, redop); + std::unique_ptr null_mask_arg = nullptr; + if (nullable()) { + auto null_redop = + privilege == REDUCE ? bool_()->find_reduction_operator(ReductionOpKind::MUL) : -1; + null_mask_arg = null_mask_->to_launcher_arg( + mapping.at(null_mask_), strategy, launch_domain, privilege, null_redop); + } + + return std::make_unique(std::move(data_arg), std::move(null_mask_arg)); +} + +std::unique_ptr BaseLogicalArray::to_launcher_arg_for_fixup( + const Domain* launch_domain, Legion::PrivilegeMode privilege) const +{ + auto data_arg = data_->to_launcher_arg_for_fixup(launch_domain, privilege); + return std::make_unique(std::move(data_arg)); +} + +ListLogicalArray::ListLogicalArray(std::shared_ptr type, + std::shared_ptr descriptor, + std::shared_ptr vardata) + : type_(std::move(type)), descriptor_(std::move(descriptor)), vardata_(std::move(vardata)) +{ +} + +bool ListLogicalArray::unbound() const { return descriptor_->unbound() || vardata_->unbound(); } + +std::shared_ptr ListLogicalArray::promote(int32_t extra_dim, size_t dim_size) const +{ + throw std::runtime_error("List array does not support store transformations"); + return nullptr; +} + +std::shared_ptr ListLogicalArray::project(int32_t dim, int64_t index) const +{ + throw std::runtime_error("List array does not support store transformations"); + return nullptr; +} + +std::shared_ptr ListLogicalArray::slice(int32_t dim, Slice sl) const +{ + throw std::runtime_error("List array does not support store transformations"); + return nullptr; +} + +std::shared_ptr ListLogicalArray::transpose(const std::vector& axes) const +{ + throw std::runtime_error("List array does not support store transformations"); + return nullptr; +} + +std::shared_ptr ListLogicalArray::delinearize(int32_t dim, + const std::vector& sizes) const +{ + throw std::runtime_error("List array does not support store transformations"); + return nullptr; +} + +std::shared_ptr ListLogicalArray::get_physical_array() const +{ + auto desc_arr = descriptor_->_get_physical_array(); + auto vardata_arr = vardata_->get_physical_array(); + return std::make_shared(type_, std::move(desc_arr), std::move(vardata_arr)); +} + +std::shared_ptr ListLogicalArray::child(uint32_t index) const +{ + if (unbound()) { + throw std::invalid_argument("Invalid to retrieve a sub-array of an unbound array"); + } + switch (index) { + case 0: return descriptor_; + case 1: return vardata_; + default: { + throw std::out_of_range("List array does not have child " + std::to_string(index)); + break; + } + } + return nullptr; +} + +std::shared_ptr ListLogicalArray::descriptor() const +{ + if (unbound()) { + throw std::invalid_argument("Invalid to retrieve a sub-array of an unbound array"); + } + return descriptor_; +} + +std::shared_ptr ListLogicalArray::vardata() const +{ + if (unbound()) { + throw std::invalid_argument("Invalid to retrieve a sub-array of an unbound array"); + } + return vardata_; +} + +void ListLogicalArray::record_scalar_or_unbound_outputs(AutoTask* task) const +{ + descriptor_->record_scalar_or_unbound_outputs(task); + vardata_->record_scalar_or_unbound_outputs(task); +} + +void ListLogicalArray::record_scalar_reductions(AutoTask* task, Legion::ReductionOpID redop) const +{ + vardata_->record_scalar_reductions(task, redop); +} + +void ListLogicalArray::generate_constraints( + AutoTask* task, + std::map, const Variable*>& mapping, + const Variable* partition_symbol) const +{ + descriptor_->generate_constraints(task, mapping, partition_symbol); + auto part_vardata = task->declare_partition(); + vardata_->generate_constraints(task, mapping, part_vardata); + if (!unbound()) { task->add_constraint(image(partition_symbol, part_vardata)); } +} + +std::unique_ptr ListLogicalArray::to_launcher_arg( + const std::map, const Variable*>& mapping, + const Strategy& strategy, + const Domain* launch_domain, + Legion::PrivilegeMode privilege, + int32_t redop) const +{ + auto desc_priv = (READ_ONLY == privilege || vardata_->unbound()) ? privilege : READ_WRITE; + auto descriptor_arg = + descriptor_->to_launcher_arg(mapping, strategy, launch_domain, desc_priv, redop); + auto vardata_arg = vardata_->to_launcher_arg(mapping, strategy, launch_domain, privilege, redop); + + return std::make_unique(type(), std::move(descriptor_arg), std::move(vardata_arg)); +} + +std::unique_ptr ListLogicalArray::to_launcher_arg_for_fixup( + const Domain* launch_domain, Legion::PrivilegeMode privilege) const +{ + auto descriptor_arg = descriptor_->to_launcher_arg_for_fixup(launch_domain, READ_WRITE); + auto vardata_arg = vardata_->to_launcher_arg_for_fixup(launch_domain, privilege); + + return std::make_unique(type(), std::move(descriptor_arg), std::move(vardata_arg)); +} + +StructLogicalArray::StructLogicalArray(std::shared_ptr type, + std::shared_ptr null_mask, + std::vector>&& fields) + : type_(std::move(type)), null_mask_(std::move(null_mask)), fields_(std::move(fields)) +{ +} + +int32_t StructLogicalArray::dim() const { return fields_.front()->dim(); } + +const Shape& StructLogicalArray::extents() const { return fields_.front()->extents(); } + +size_t StructLogicalArray::volume() const { return fields_.front()->volume(); } + +bool StructLogicalArray::unbound() const +{ + return std::any_of(fields_.begin(), fields_.end(), [](auto& array) { return array->unbound(); }); +} + +std::shared_ptr StructLogicalArray::promote(int32_t extra_dim, size_t dim_size) const +{ + auto null_mask = nullable() ? null_mask_->promote(extra_dim, dim_size) : nullptr; + std::vector> fields; + for (auto& field : fields_) { fields.push_back(field->promote(extra_dim, dim_size)); } + return std::make_shared(type_, std::move(null_mask), std::move(fields)); +} + +std::shared_ptr StructLogicalArray::project(int32_t dim, int64_t index) const +{ + auto null_mask = nullable() ? null_mask_->project(dim, index) : nullptr; + std::vector> fields; + for (auto& field : fields_) { fields.push_back(field->project(dim, index)); } + return std::make_shared(type_, std::move(null_mask), std::move(fields)); +} + +std::shared_ptr StructLogicalArray::slice(int32_t dim, Slice sl) const +{ + auto null_mask = nullable() ? null_mask_->slice(dim, sl) : nullptr; + std::vector> fields; + for (auto& field : fields_) { fields.push_back(field->slice(dim, sl)); } + return std::make_shared(type_, std::move(null_mask), std::move(fields)); +} + +std::shared_ptr StructLogicalArray::transpose(const std::vector& axes) const +{ + auto null_mask = nullable() ? null_mask_->transpose(axes) : nullptr; + std::vector> fields; + for (auto& field : fields_) { fields.push_back(field->transpose(axes)); } + return std::make_shared(type_, std::move(null_mask), std::move(fields)); +} + +std::shared_ptr StructLogicalArray::delinearize( + int32_t dim, const std::vector& sizes) const +{ + auto null_mask = nullable() ? null_mask_->delinearize(dim, sizes) : nullptr; + std::vector> fields; + for (auto& field : fields_) { fields.push_back(field->delinearize(dim, sizes)); } + return std::make_shared(type_, std::move(null_mask), std::move(fields)); +} + +std::shared_ptr StructLogicalArray::null_mask() const +{ + if (!nullable()) + throw std::invalid_argument("Invalid to retrieve the null mask of a non-nullable array"); + return null_mask_; +} + +std::shared_ptr StructLogicalArray::get_physical_array() const +{ + std::shared_ptr null_mask_store = nullptr; + if (null_mask_ != nullptr) { null_mask_store = null_mask_->get_physical_store(); } + std::vector> field_arrays; + for (auto& field : fields_) { field_arrays.push_back(field->get_physical_array()); } + return std::make_shared(type_, std::move(null_mask_store), std::move(field_arrays)); +} + +std::shared_ptr StructLogicalArray::child(uint32_t index) const +{ + if (unbound()) { + throw std::invalid_argument("Invalid to retrieve a sub-array of an unbound array"); + } + return fields_.at(index); +} + +std::shared_ptr StructLogicalArray::primary_store() const +{ + return fields_.front()->primary_store(); +} + +void StructLogicalArray::record_scalar_or_unbound_outputs(AutoTask* task) const +{ + for (auto& field : fields_) { field->record_scalar_or_unbound_outputs(task); } + + if (!nullable()) return; + + if (null_mask_->unbound()) + task->record_unbound_output(null_mask_); + else if (null_mask_->has_scalar_storage()) + task->record_scalar_output(null_mask_); +} + +void StructLogicalArray::record_scalar_reductions(AutoTask* task, Legion::ReductionOpID redop) const +{ + for (auto& field : fields_) { field->record_scalar_reductions(task, redop); } + if (nullable() && null_mask_->has_scalar_storage()) { + auto null_redop = bool_()->find_reduction_operator(ReductionOpKind::MUL); + task->record_scalar_reduction(null_mask_, null_redop); + } +} + +void StructLogicalArray::generate_constraints( + AutoTask* task, + std::map, const Variable*>& mapping, + const Variable* partition_symbol) const +{ + auto it = fields_.begin(); + (*it++)->generate_constraints(task, mapping, partition_symbol); + for (; it != fields_.end(); ++it) { + auto part_field = task->declare_partition(); + (*it)->generate_constraints(task, mapping, part_field); + task->add_constraint(image(partition_symbol, part_field)); + } +} + +std::unique_ptr StructLogicalArray::to_launcher_arg( + const std::map, const Variable*>& mapping, + const Strategy& strategy, + const Domain* launch_domain, + Legion::PrivilegeMode privilege, + int32_t redop) const +{ + std::unique_ptr null_mask_arg = nullptr; + if (nullable()) { + auto null_redop = + privilege == REDUCE ? bool_()->find_reduction_operator(ReductionOpKind::MUL) : -1; + null_mask_arg = null_mask_->to_launcher_arg( + mapping.at(null_mask_), strategy, launch_domain, privilege, null_redop); + } + + std::vector> field_args; + for (auto& field : fields_) { + field_args.push_back( + field->to_launcher_arg(mapping, strategy, launch_domain, privilege, redop)); + } + + return std::make_unique(type(), std::move(null_mask_arg), std::move(field_args)); +} + +std::unique_ptr StructLogicalArray::to_launcher_arg_for_fixup( + const Domain* launch_domain, Legion::PrivilegeMode privilege) const +{ + std::vector> field_args; + for (auto& field : fields_) { + field_args.push_back(field->to_launcher_arg_for_fixup(launch_domain, privilege)); + } + return std::make_unique(type(), nullptr, std::move(field_args)); +} + +} // namespace legate::detail diff --git a/src/core/data/detail/logical_array.h b/src/core/data/detail/logical_array.h new file mode 100644 index 0000000000..38b13c0c64 --- /dev/null +++ b/src/core/data/detail/logical_array.h @@ -0,0 +1,244 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#pragma once + +#include "core/data/detail/array_kind.h" +#include "core/data/detail/logical_store.h" +#include "core/operation/detail/launcher_arg.h" + +namespace legate::detail { + +class Array; +class AutoTask; +class BaseArray; +class ConstraintSolver; +class ListLogicalArray; +class Variable; + +struct LogicalArray { + virtual ~LogicalArray() {} + virtual int32_t dim() const = 0; + virtual ArrayKind kind() const = 0; + virtual std::shared_ptr type() const = 0; + virtual const Shape& extents() const = 0; + virtual size_t volume() const = 0; + virtual bool unbound() const = 0; + virtual bool nullable() const = 0; + virtual bool nested() const = 0; + virtual uint32_t num_children() const = 0; + + virtual std::shared_ptr promote(int32_t extra_dim, size_t dim_size) const = 0; + virtual std::shared_ptr project(int32_t dim, int64_t index) const = 0; + virtual std::shared_ptr slice(int32_t dim, Slice sl) const = 0; + virtual std::shared_ptr transpose(const std::vector& axes) const = 0; + virtual std::shared_ptr delinearize(int32_t dim, + const std::vector& sizes) const = 0; + + virtual std::shared_ptr data() const; + virtual std::shared_ptr null_mask() const = 0; + virtual std::shared_ptr get_physical_array() const = 0; + virtual std::shared_ptr child(uint32_t index) const = 0; + virtual std::shared_ptr primary_store() const = 0; + + virtual void record_scalar_or_unbound_outputs(AutoTask* task) const = 0; + virtual void record_scalar_reductions(AutoTask* task, Legion::ReductionOpID redop) const = 0; + + virtual void generate_constraints( + AutoTask* task, + std::map, const Variable*>& mapping, + const Variable* partition_symbol) const = 0; + + virtual std::unique_ptr to_launcher_arg( + const std::map, const Variable*>& mapping, + const Strategy& strategy, + const Domain* launch_domain, + Legion::PrivilegeMode privilege, + int32_t redop) const = 0; + virtual std::unique_ptr to_launcher_arg_for_fixup( + const Domain* launch_domain, Legion::PrivilegeMode privilege) const = 0; +}; + +class BaseLogicalArray : public LogicalArray { + public: + BaseLogicalArray(std::shared_ptr data, + std::shared_ptr null_mask = nullptr); + + public: + int32_t dim() const override { return data_->dim(); } + ArrayKind kind() const override { return ArrayKind::BASE; } + std::shared_ptr type() const override { return data_->type(); } + const Shape& extents() const override { return data_->extents(); } + size_t volume() const override { return data_->volume(); } + bool unbound() const override; + bool nullable() const override { return null_mask_ != nullptr; } + bool nested() const override { return false; } + uint32_t num_children() const override { return 0; } + + public: + std::shared_ptr promote(int32_t extra_dim, size_t dim_size) const override; + std::shared_ptr project(int32_t dim, int64_t index) const override; + std::shared_ptr slice(int32_t dim, Slice sl) const override; + std::shared_ptr transpose(const std::vector& axes) const override; + std::shared_ptr delinearize(int32_t dim, + const std::vector& sizes) const override; + + public: + std::shared_ptr data() const override { return data_; } + std::shared_ptr null_mask() const override; + std::shared_ptr get_physical_array() const override; + std::shared_ptr _get_physical_array() const; + std::shared_ptr child(uint32_t index) const override; + std::shared_ptr primary_store() const override { return data(); } + + public: + void record_scalar_or_unbound_outputs(AutoTask* task) const override; + void record_scalar_reductions(AutoTask* task, Legion::ReductionOpID redop) const override; + + public: + void generate_constraints(AutoTask* task, + std::map, const Variable*>& mapping, + const Variable* partition_symbol) const override; + + public: + std::unique_ptr to_launcher_arg( + const std::map, const Variable*>& mapping, + const Strategy& strategy, + const Domain* launch_domain, + Legion::PrivilegeMode privilege, + int32_t redop) const override; + std::unique_ptr to_launcher_arg_for_fixup( + const Domain* launch_domain, Legion::PrivilegeMode privilege) const override; + + private: + std::shared_ptr data_; + std::shared_ptr null_mask_; +}; + +class ListLogicalArray : public LogicalArray { + public: + ListLogicalArray(std::shared_ptr type, + std::shared_ptr descriptor, + std::shared_ptr vardata); + + public: + int32_t dim() const override { return descriptor_->dim(); } + ArrayKind kind() const override { return ArrayKind::LIST; } + std::shared_ptr type() const override { return type_; } + const Shape& extents() const override { return descriptor_->extents(); } + size_t volume() const override { return descriptor_->volume(); } + bool unbound() const override; + bool nullable() const override { return descriptor_->nullable(); } + bool nested() const override { return true; } + uint32_t num_children() const override { return 2; } + + public: + std::shared_ptr promote(int32_t extra_dim, size_t dim_size) const override; + std::shared_ptr project(int32_t dim, int64_t index) const override; + std::shared_ptr slice(int32_t dim, Slice sl) const override; + std::shared_ptr transpose(const std::vector& axes) const override; + std::shared_ptr delinearize(int32_t dim, + const std::vector& sizes) const override; + + public: + std::shared_ptr null_mask() const override { return descriptor_->null_mask(); } + std::shared_ptr get_physical_array() const override; + std::shared_ptr child(uint32_t index) const override; + std::shared_ptr primary_store() const override + { + return descriptor_->primary_store(); + } + std::shared_ptr descriptor() const; + std::shared_ptr vardata() const; + + public: + void record_scalar_or_unbound_outputs(AutoTask* task) const override; + void record_scalar_reductions(AutoTask* task, Legion::ReductionOpID redop) const override; + + public: + void generate_constraints(AutoTask* task, + std::map, const Variable*>& mapping, + const Variable* partition_symbol) const override; + + public: + std::unique_ptr to_launcher_arg( + const std::map, const Variable*>& mapping, + const Strategy& strategy, + const Domain* launch_domain, + Legion::PrivilegeMode privilege, + int32_t redop) const override; + std::unique_ptr to_launcher_arg_for_fixup( + const Domain* launch_domain, Legion::PrivilegeMode privilege) const override; + + private: + std::shared_ptr type_; + std::shared_ptr descriptor_; + std::shared_ptr vardata_; +}; + +class StructLogicalArray : public LogicalArray { + public: + StructLogicalArray(std::shared_ptr type, + std::shared_ptr null_mask, + std::vector>&& fields); + + public: + int32_t dim() const override; + ArrayKind kind() const override { return ArrayKind::STRUCT; } + std::shared_ptr type() const override { return type_; } + const Shape& extents() const override; + size_t volume() const override; + bool unbound() const override; + bool nullable() const override { return null_mask_ != nullptr; } + bool nested() const override { return true; } + uint32_t num_children() const override { return fields_.size(); } + + public: + std::shared_ptr promote(int32_t extra_dim, size_t dim_size) const override; + std::shared_ptr project(int32_t dim, int64_t index) const override; + std::shared_ptr slice(int32_t dim, Slice sl) const override; + std::shared_ptr transpose(const std::vector& axes) const override; + std::shared_ptr delinearize(int32_t dim, + const std::vector& sizes) const override; + + public: + std::shared_ptr null_mask() const override; + std::shared_ptr get_physical_array() const override; + std::shared_ptr child(uint32_t index) const override; + std::shared_ptr primary_store() const override; + + public: + void record_scalar_or_unbound_outputs(AutoTask* task) const override; + void record_scalar_reductions(AutoTask* task, Legion::ReductionOpID redop) const override; + + public: + void generate_constraints(AutoTask* task, + std::map, const Variable*>& mapping, + const Variable* partition_symbol) const override; + + public: + std::unique_ptr to_launcher_arg( + const std::map, const Variable*>& mapping, + const Strategy& strategy, + const Domain* launch_domain, + Legion::PrivilegeMode privilege, + int32_t redop) const override; + std::unique_ptr to_launcher_arg_for_fixup( + const Domain* launch_domain, Legion::PrivilegeMode privilege) const override; + + private: + std::shared_ptr type_; + std::shared_ptr null_mask_; + std::vector> fields_; +}; + +} // namespace legate::detail diff --git a/src/core/data/detail/logical_region_field.cc b/src/core/data/detail/logical_region_field.cc index dda2508c4b..70f3c9f694 100644 --- a/src/core/data/detail/logical_region_field.cc +++ b/src/core/data/detail/logical_region_field.cc @@ -28,7 +28,10 @@ LogicalRegionField::LogicalRegionField(FieldManager* manager, LogicalRegionField::~LogicalRegionField() { // Only free the field once the top-level region is deleted. - if (parent_ == nullptr) manager_->free_field(lr_, fid_, destroyed_out_of_order_); + if (parent_ == nullptr) { + perform_invalidation_callbacks(); + manager_->free_field(lr_, fid_, destroyed_out_of_order_); + } } int32_t LogicalRegionField::dim() const { return lr_.get_dim(); } @@ -70,4 +73,27 @@ Legion::LogicalPartition LogicalRegionField::get_legion_partition(const Partitio return partition->construct(lr_, complete); } +void LogicalRegionField::add_invalidation_callback(std::function callback) +{ + if (parent_ != nullptr) { + parent_->add_invalidation_callback(callback); + } else { + callbacks_.push_back(callback); + } +} + +void LogicalRegionField::perform_invalidation_callbacks() +{ + if (parent_ != nullptr) { +#ifdef DEBUG_LEGATE + // Callbacks should exist only in the root + assert(callbacks_.empty()); +#endif + parent_->perform_invalidation_callbacks(); + } else { + for (auto& callback : callbacks_) { callback(); } + callbacks_.clear(); + } +} + } // namespace legate::detail diff --git a/src/core/data/detail/logical_region_field.h b/src/core/data/detail/logical_region_field.h index 3c083a9db5..0a446f8792 100644 --- a/src/core/data/detail/logical_region_field.h +++ b/src/core/data/detail/logical_region_field.h @@ -12,6 +12,7 @@ #pragma once +#include #include #include @@ -60,12 +61,17 @@ class LogicalRegionField : public std::enable_shared_from_this callback); + void perform_invalidation_callbacks(); + private: FieldManager* manager_; Legion::LogicalRegion lr_; Legion::FieldID fid_; std::shared_ptr parent_; bool destroyed_out_of_order_{false}; + std::vector> callbacks_{}; }; } // namespace legate::detail diff --git a/src/core/data/detail/logical_store.cc b/src/core/data/detail/logical_store.cc index 71d5e12713..efb8aea9c5 100644 --- a/src/core/data/detail/logical_store.cc +++ b/src/core/data/detail/logical_store.cc @@ -14,7 +14,10 @@ #include "core/data/detail/store.h" #include "core/data/detail/transform.h" +#include "core/operation/detail/launcher_arg.h" +#include "core/operation/detail/operation.h" #include "core/operation/detail/projection.h" +#include "core/partitioning/detail/partitioner.h" #include "core/runtime/detail/partition_manager.h" #include "core/runtime/detail/runtime.h" #include "core/type/detail/type_info.h" @@ -491,6 +494,11 @@ std::shared_ptr LogicalStore::slice(int32_t idx, Slice slice) std::move(exts), std::move(substorage), std::move(transform)); } +std::shared_ptr LogicalStore::transpose(const std::vector& axes) +{ + return transpose(std::vector(axes)); +} + std::shared_ptr LogicalStore::transpose(std::vector&& axes) { if (axes.size() != dim()) { @@ -515,6 +523,12 @@ std::shared_ptr LogicalStore::transpose(std::vector&& axe return std::make_shared(std::move(new_extents), storage_, std::move(transform)); } +std::shared_ptr LogicalStore::delinearize(int32_t idx, + const std::vector& sizes) +{ + return delinearize(idx, std::vector(sizes)); +} + std::shared_ptr LogicalStore::delinearize(int32_t idx, std::vector&& sizes) { if (idx < 0 || idx >= dim()) { @@ -604,11 +618,7 @@ std::shared_ptr LogicalStore::find_or_create_key_partition( key_partition_->satisfies_restrictions(restrictions)) return key_partition_; - if (has_scalar_storage()) { - num_pieces_ = new_num_pieces; - key_partition_ = create_no_partition(); - return key_partition_; - } + if (has_scalar_storage()) { return create_no_partition(); } Partition* storage_part = nullptr; if (transform_->is_convertible()) @@ -629,9 +639,7 @@ std::shared_ptr LogicalStore::find_or_create_key_partition( #ifdef DEBUG_LEGATE assert(store_part != nullptr); #endif - num_pieces_ = new_num_pieces; - key_partition_ = std::move(store_part); - return key_partition_; + return std::move(store_part); } bool LogicalStore::has_key_partition(const mapping::detail::Machine& machine, @@ -649,7 +657,8 @@ bool LogicalStore::has_key_partition(const mapping::detail::Machine& machine, void LogicalStore::set_key_partition(const mapping::detail::Machine& machine, const Partition* partition) { - num_pieces_ = machine.count(); + num_pieces_ = machine.count(); + key_partition_.reset(partition->clone().release()); auto inverted = transform_->invert(partition); storage_->set_key_partition(machine, std::move(inverted)); } @@ -681,6 +690,46 @@ void LogicalStore::pack(BufferBuilder& buffer) const transform_->pack(buffer); } +std::unique_ptr LogicalStore::to_launcher_arg(const Variable* variable, + const Strategy& strategy, + const Domain* launch_domain, + Legion::PrivilegeMode privilege, + int32_t redop) +{ + if (has_scalar_storage()) { + auto has_storage = privilege != WRITE_ONLY; + auto read_only = privilege == READ_ONLY; + return std::make_unique(this, read_only, has_storage, redop); + } else if (unbound()) { + return std::make_unique(this, strategy.find_field_space(variable)); + } else { + auto partition = strategy[variable]; + auto store_partition = create_partition(partition); + auto proj_info = store_partition->create_projection_info(launch_domain); + proj_info->tag = strategy.is_key_partition(variable) ? LEGATE_CORE_KEY_STORE_TAG : 0; + proj_info->redop = redop; + + if (privilege == REDUCE && store_partition->is_disjoint_for(launch_domain)) { + privilege = READ_WRITE; + } + if (privilege == WRITE_ONLY) { + set_key_partition(variable->operation()->machine(), partition.get()); + } + return std::make_unique(this, privilege, std::move(proj_info)); + } +} + +std::unique_ptr LogicalStore::to_launcher_arg_for_fixup(const Domain* launch_domain, + Legion::PrivilegeMode privilege) +{ +#ifdef DEBUG_LEGATE + assert(key_partition_ != nullptr); +#endif + auto store_partition = create_partition(key_partition_); + auto proj_info = store_partition->create_projection_info(launch_domain); + return std::make_unique(this, privilege, std::move(proj_info)); +} + std::string LogicalStore::to_string() const { std::stringstream ss; @@ -712,13 +761,15 @@ LogicalStorePartition::LogicalStorePartition(std::shared_ptr partitio std::unique_ptr LogicalStorePartition::create_projection_info( const Domain* launch_domain, std::optional proj_fn) { - if (nullptr == launch_domain || store_->has_scalar_storage()) - return std::make_unique(); + if (store_->has_scalar_storage()) return std::make_unique(); + + if (!partition_->has_launch_domain()) { return std::make_unique(); } // We're about to create a legion partition for this store, so the store should have its region // created. auto legion_partition = storage_partition_->get_legion_partition(); - auto proj_id = store_->compute_projection(launch_domain->dim, proj_fn); + auto proj_id = + launch_domain != nullptr ? store_->compute_projection(launch_domain->dim, proj_fn) : 0; return std::make_unique(legion_partition, proj_id); } diff --git a/src/core/data/detail/logical_store.h b/src/core/data/detail/logical_store.h index 52bea7f326..4fa0d015a3 100644 --- a/src/core/data/detail/logical_store.h +++ b/src/core/data/detail/logical_store.h @@ -26,10 +26,13 @@ namespace legate::detail { +class Analyzable; +class LogicalStorePartition; class ProjectionInfo; +class Strategy; class StoragePartition; class Store; -class LogicalStorePartition; +class Variable; class Storage : public std::enable_shared_from_this { public: @@ -194,7 +197,9 @@ class LogicalStore : public std::enable_shared_from_this { std::shared_ptr promote(int32_t extra_dim, size_t dim_size); std::shared_ptr project(int32_t dim, int64_t index); std::shared_ptr slice(int32_t dim, Slice sl); + std::shared_ptr transpose(const std::vector& axes); std::shared_ptr transpose(std::vector&& axes); + std::shared_ptr delinearize(int32_t dim, const std::vector& sizes); std::shared_ptr delinearize(int32_t dim, std::vector&& sizes); public: @@ -221,6 +226,7 @@ class LogicalStore : public std::enable_shared_from_this { Restrictions compute_restrictions() const; std::shared_ptr find_or_create_key_partition(const mapping::detail::Machine& machine, const Restrictions& restrictions); + std::shared_ptr get_current_key_partition() const { return key_partition_; } bool has_key_partition(const mapping::detail::Machine& machine, const Restrictions& restrictions) const; void set_key_partition(const mapping::detail::Machine& machine, const Partition* partition); @@ -234,6 +240,13 @@ class LogicalStore : public std::enable_shared_from_this { public: void pack(BufferBuilder& buffer) const; + std::unique_ptr to_launcher_arg(const Variable* variable, + const Strategy& strategy, + const Domain* launch_domain, + Legion::PrivilegeMode privilege, + int32_t redop = -1); + std::unique_ptr to_launcher_arg_for_fixup(const Domain* launch_domain, + Legion::PrivilegeMode privilege); public: std::string to_string() const; diff --git a/src/core/data/detail/store.h b/src/core/data/detail/store.h index 9cb9d0c1a4..2bbf6f1848 100644 --- a/src/core/data/detail/store.h +++ b/src/core/data/detail/store.h @@ -21,6 +21,7 @@ class Store; } // namespace legate namespace legate::detail { +class BaseArray; class TransformStack; class RegionField { @@ -209,6 +210,7 @@ class Store { private: friend class legate::Store; + friend class legate::detail::BaseArray; void check_accessor_dimension(const int32_t dim) const; void check_buffer_dimension(const int32_t dim) const; void check_shape_dimension(const int32_t dim) const; diff --git a/src/core/data/logical_array.cc b/src/core/data/logical_array.cc new file mode 100644 index 0000000000..3cdf0b6288 --- /dev/null +++ b/src/core/data/logical_array.cc @@ -0,0 +1,127 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include "core/data/logical_array.h" +#include "core/data/detail/logical_array.h" + +namespace legate { + +int32_t LogicalArray::dim() const { return impl_->dim(); } + +Type LogicalArray::type() const { return Type(impl_->type()); } + +const Shape& LogicalArray::extents() const { return impl_->extents(); } + +size_t LogicalArray::volume() const { return impl_->volume(); } + +bool LogicalArray::unbound() const { return impl_->unbound(); } + +bool LogicalArray::nullable() const { return impl_->nullable(); } + +bool LogicalArray::nested() const { return impl_->nested(); } + +uint32_t LogicalArray::num_children() const { return impl_->num_children(); } + +LogicalArray LogicalArray::promote(int32_t extra_dim, size_t dim_size) const +{ + return LogicalArray(impl_->promote(extra_dim, dim_size)); +} + +LogicalArray LogicalArray::project(int32_t dim, int64_t index) const +{ + return LogicalArray(impl_->project(dim, index)); +} + +LogicalArray LogicalArray::slice(int32_t dim, Slice sl) const +{ + return LogicalArray(impl_->slice(dim, sl)); +} + +LogicalArray LogicalArray::transpose(const std::vector& axes) const +{ + return LogicalArray(impl_->transpose(axes)); +} + +LogicalArray LogicalArray::delinearize(int32_t dim, const std::vector& sizes) const +{ + return LogicalArray(impl_->delinearize(dim, sizes)); +} + +LogicalStore LogicalArray::data() const { return LogicalStore(impl_->data()); } + +LogicalStore LogicalArray::null_mask() const { return LogicalStore(impl_->null_mask()); } + +LogicalArray LogicalArray::child(uint32_t index) const { return LogicalArray(impl_->child(index)); } + +Array LogicalArray::get_physical_array() const { return Array(impl_->get_physical_array()); } + +ListLogicalArray LogicalArray::as_list_array() const +{ + if (impl_->kind() != detail::ArrayKind::LIST) { + throw std::invalid_argument("Array is not a list array"); + } + return ListLogicalArray(impl_); +} + +StringLogicalArray LogicalArray::as_string_array() const +{ + if (type().code() != Type::Code::STRING) { + throw std::invalid_argument("Array is not a string array"); + } + return StringLogicalArray(impl_); +} + +LogicalArray::LogicalArray(std::shared_ptr impl) : impl_(std::move(impl)) {} + +LogicalArray::~LogicalArray() {} + +LogicalArray::LogicalArray(LogicalStore store) + : impl_(std::make_shared(store.impl())) +{ +} + +LogicalArray::LogicalArray(LogicalStore store, LogicalStore null_mask) + : impl_(std::make_shared(store.impl(), null_mask.impl())) +{ +} + +LogicalArray ListLogicalArray::descriptor() const +{ + return LogicalArray(static_cast(impl_.get())->descriptor()); +} + +LogicalArray ListLogicalArray::vardata() const +{ + return LogicalArray(static_cast(impl_.get())->vardata()); +} + +ListLogicalArray::ListLogicalArray(std::shared_ptr impl) + : LogicalArray(std::move(impl)) +{ +} + +LogicalArray StringLogicalArray::offsets() const +{ + return LogicalArray(static_cast(impl_.get())->descriptor()); +} + +LogicalArray StringLogicalArray::chars() const +{ + return LogicalArray(static_cast(impl_.get())->vardata()); +} + +StringLogicalArray::StringLogicalArray(std::shared_ptr impl) + : LogicalArray(std::move(impl)) +{ +} + +} // namespace legate diff --git a/src/core/data/logical_array.h b/src/core/data/logical_array.h new file mode 100644 index 0000000000..a32aeec074 --- /dev/null +++ b/src/core/data/logical_array.h @@ -0,0 +1,282 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#pragma once + +#include + +#include "core/data/array.h" +#include "core/data/logical_store.h" +#include "core/data/shape.h" +#include "core/type/type_info.h" +#include "core/utilities/typedefs.h" + +/** + * @file + * @brief Class definition for legate::LogicalArray + */ + +namespace legate::detail { + +class LogicalArray; + +} // namespace legate::detail + +namespace legate { + +class ListLogicalArray; +class StringLogicalArray; + +/** + * @ingroup data + * + * @brief A multi-dimensional array + */ +class LogicalArray { + public: + /** + * @brief Returns the number of dimensions of the array. + * + * @return The number of dimensions + */ + int32_t dim() const; + /** + * @brief Returns the element type of the array. + * + * @return Type of elements in the store + */ + Type type() const; + /** + * @brief Returns the shape of the array. + * + * Flushes the scheduling window if the store is unbound and has no shape assigned. + * + * @return The store's shape + */ + const Shape& extents() const; + /** + * @brief Returns the number of elements in the array. + * + * Flushes the scheduling window if the store is unbound and has no shape assigned. + * + * @return The number of elements in the store + */ + size_t volume() const; + /** + * @brief Indicates whether the array is unbound + * + * @return true The array is unbound + * @return false The array is normal + */ + bool unbound() const; + /** + * @brief Indicates whether the array is nullable + * + * @return true The array is nullable + * @return false The array is non-nullable + */ + bool nullable() const; + /** + * @brief Indicates whether the array has child arrays + * + * @return true The array has child arrays + * @return false Otherwise + */ + bool nested() const; + /** + * @brief Returns the number of child sub-arrays + * + * @return Number of child sub-arrays + */ + uint32_t num_children() const; + + public: + /** + * @brief Adds an extra dimension to the array. + * + * @param extra_dim Position for a new dimension + * @param dim_size Extent of the new dimension + * + * @return A new array with an extra dimension + * + * @throw std::invalid_argument When `extra_dim` is not a valid dimension name + * @throw std::runtime_error If the array or any of the sub-arrays is a list array + */ + LogicalArray promote(int32_t extra_dim, size_t dim_size) const; + /** + * @brief Projects out a dimension of the array. + * + * @param dim Dimension to project out + * @param index Index on the chosen dimension + * + * @return A new array with one fewer dimension + * + * @throw std::invalid_argument If `dim` is not a valid dimension name or `index` is out of bounds + * @throw std::runtime_error If the array or any of the sub-arrays is a list array + */ + LogicalArray project(int32_t dim, int64_t index) const; + /** + * @brief Slices a contiguous sub-section of the array. + * + * @param dim Dimension to slice + * @param sl Slice descriptor + * + * @return A new array that corresponds to the sliced section + * + * @throw std::invalid_argument If `dim` is not a valid dimension name + * @throw std::runtime_error If the array or any of the sub-arrays is a list array + */ + LogicalArray slice(int32_t dim, Slice sl) const; + /** + * @brief Reorders dimensions of the array. + * + * @param axes Mapping from dimensions of the resulting array to those of the input + * + * @return A new array with the dimensions transposed + * + * @throw std::invalid_argument If any of the following happens: 1) The length of `axes` doesn't + * match the array's dimension; 2) `axes` has duplicates; 3) Any axis in `axes` is an invalid + * axis name. + * @throw std::runtime_error If the array or any of the sub-arrays is a list array + */ + LogicalArray transpose(const std::vector& axes) const; + /** + * @brief Delinearizes a dimension into multiple dimensions. + * + * @param dim Dimension to delinearize + * @param sizes Extents for the resulting dimensions + * + * @return A new array with the chosen dimension delinearized + * + * @throw std::invalid_argument If `dim` is invalid for the array or `sizes` does not preserve + * the extent of the chosen dimenison + * @throw std::runtime_error If the array or any of the sub-arrays is a list array + */ + LogicalArray delinearize(int32_t dim, const std::vector& sizes) const; + + public: + /** + * @brief Returns the store of this array + * + * @return Logical store + */ + LogicalStore data() const; + /** + * @brief Returns the store of this array + * + * @return Logical store + */ + LogicalStore null_mask() const; + /** + * @brief Returns the sub-array of a given index + * + * @param index Sub-array index + * + * @return Logical array + * + * @throw std::invalid_argument If the array has no child arrays, or the array is an unbound + * struct array + * @throw std::out_of_range If the index is out of range + */ + LogicalArray child(uint32_t index) const; + + public: + /** + * @brief Creates a physical array for this logical array + * + * This call blocks the client's control flow and fetches the data for the whole array to the + * current node + * + * @return A physical array of the logical array + */ + Array get_physical_array() const; + + public: + /** + * @brief Casts this array as a list array + * + * @return List array + * + * @throw std::invalid_argument If the array is not a list array + */ + ListLogicalArray as_list_array() const; + /** + * @brief Casts this array as a string array + * + * @return String array + * + * @throw std::invalid_argument If the array is not a string array + */ + StringLogicalArray as_string_array() const; + + public: + LogicalArray(std::shared_ptr impl); + virtual ~LogicalArray(); + + public: + LogicalArray(const LogicalArray& other) = default; + LogicalArray& operator=(const LogicalArray& other) = default; + LogicalArray(LogicalArray&& other) = default; + LogicalArray& operator=(LogicalArray&& other) = default; + + public: + LogicalArray(LogicalStore store); + LogicalArray(LogicalStore store, LogicalStore null_mask); + + public: + std::shared_ptr impl() const { return impl_; } + + protected: + std::shared_ptr impl_{nullptr}; +}; + +class ListLogicalArray : public LogicalArray { + public: + /** + * @brief Returns the sub-array for descriptors + * + * @return Store + */ + LogicalArray descriptor() const; + /** + * @brief Returns the sub-array for variable size data + * + * @return Store + */ + LogicalArray vardata() const; + + private: + friend class LogicalArray; + ListLogicalArray(std::shared_ptr impl); +}; + +class StringLogicalArray : public LogicalArray { + public: + /** + * @brief Returns the sub-array for offsets + * + * @return Store + */ + LogicalArray offsets() const; + /** + * @brief Returns the sub-array for characters + * + * @return Store + */ + LogicalArray chars() const; + + private: + friend class LogicalArray; + StringLogicalArray(std::shared_ptr impl); +}; + +} // namespace legate diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index 8d5dcedbb5..b35366b550 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -67,7 +67,7 @@ LogicalStore LogicalStore::delinearize(int32_t dim, std::vector&& sizes return LogicalStore(impl_->delinearize(dim, std::move(sizes))); } -Store LogicalStore::get_physical_store() { return Store(impl_->get_physical_store()); } +Store LogicalStore::get_physical_store() const { return Store(impl_->get_physical_store()); } LogicalStorePartition::LogicalStorePartition(std::shared_ptr&& impl) : impl_(std::move(impl)) diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index 3022988ba0..dfa85e6c6d 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -13,7 +13,6 @@ #pragma once #include -#include #include "core/data/shape.h" #include "core/data/slice.h" @@ -28,6 +27,7 @@ */ namespace legate::detail { +class LogicalArray; class LogicalStore; class LogicalStorePartition; } // namespace legate::detail @@ -72,6 +72,7 @@ class Runtime; class LogicalStore { private: friend class Runtime; + friend class LogicalArray; friend class LogicalStorePartition; LogicalStore(std::shared_ptr&& impl); @@ -104,6 +105,13 @@ class LogicalStore { * @return The store's shape */ const Shape& extents() const; + /** + * @brief Returns the number of elements in the store. + * + * Flushes the scheduling window if the store is unbound and has no shape assigned. + * + * @return The number of elements in the store + */ size_t volume() const; /** * @brief Indicates whether the store is unbound @@ -214,7 +222,7 @@ class LogicalStore { * @param dim Dimension to slice * @param sl Slice descriptor * - * @return A new store that correponds to the sliced section + * @return A new store that corresponds to the sliced section * * @throw std::invalid_argument If `dim` is not a valid dimension name */ @@ -312,12 +320,12 @@ class LogicalStore { /** * @brief Creates a physical store for this logical store * - * This call blocks the client's control flow. And it fetches the data for the whole store on - * a single node. + * This call blocks the client's control flow and fetches the data for the whole store to the + * current node * * @return A physical store of the logical store */ - Store get_physical_store(); + Store get_physical_store() const; public: std::shared_ptr impl() const { return impl_; } diff --git a/src/core/data/store.cc b/src/core/data/store.cc index c78e749e50..4adcd6c95c 100644 --- a/src/core/data/store.cc +++ b/src/core/data/store.cc @@ -11,11 +11,13 @@ */ #include "core/data/store.h" + +#include "core/data/array.h" #include "core/data/detail/store.h" namespace legate { -void Store::bind_empty_data() { impl_->bind_empty_data(); } +void Store::bind_empty_data() const { impl_->bind_empty_data(); } int32_t Store::dim() const { return impl_->dim(); } @@ -39,6 +41,8 @@ bool Store::is_unbound_store() const { return impl_->is_unbound_store(); } void Store::unmap() { impl_->unmap(); } +Store::Store(const Array& array) : impl_(array.data().impl()) {} + void Store::check_accessor_dimension(const int32_t dim) const { impl_->check_accessor_dimension(dim); @@ -68,12 +72,15 @@ Legion::Future Store::get_future() const { return impl_->get_future(); } Legion::UntypedDeferredValue Store::get_buffer() const { return impl_->get_buffer(); } -void Store::get_output_field(Legion::OutputRegion& out, Legion::FieldID& fid) +void Store::get_output_field(Legion::OutputRegion& out, Legion::FieldID& fid) const { impl_->get_output_field(out, fid); } -void Store::update_num_elements(size_t num_elements) { impl_->update_num_elements(num_elements); } +void Store::update_num_elements(size_t num_elements) const +{ + impl_->update_num_elements(num_elements); +} Store::Store() {} diff --git a/src/core/data/store.h b/src/core/data/store.h index df248a62fb..04debcb7fb 100644 --- a/src/core/data/store.h +++ b/src/core/data/store.h @@ -30,6 +30,8 @@ class Store; namespace legate { +class Array; + /** * @ingroup data * @@ -196,7 +198,7 @@ class Store { * @return A reduction accessor to the store */ template - Buffer create_output_buffer(const Point& extents, bool bind_buffer = false); + Buffer create_output_buffer(const Point& extents, bool bind_buffer = false) const; /** * @brief Binds a buffer to the store. Valid only when the store is unbound and * has not yet been bound to another buffer. The buffer must be consistent with @@ -211,12 +213,12 @@ class Store { * */ template - void bind_data(Buffer& buffer, const Point& extents); + void bind_data(Buffer& buffer, const Point& extents) const; /** * @brief Makes the unbound store empty. Valid only when the store is unbound and * has not yet been bound to another buffer. */ - void bind_empty_data(); + void bind_empty_data() const; public: /** @@ -322,6 +324,14 @@ class Store { */ void unmap(); + public: + /** + * @brief Constructs a store out of an array + * + * @throw std::invalid_argument If the array is nullable or has sub-arrays + */ + Store(const Array& array); + private: void check_accessor_dimension(const int32_t dim) const; void check_buffer_dimension(const int32_t dim) const; @@ -345,8 +355,8 @@ class Store { Legion::UntypedDeferredValue get_buffer() const; private: - void get_output_field(Legion::OutputRegion& out, Legion::FieldID& fid); - void update_num_elements(size_t num_elements); + void get_output_field(Legion::OutputRegion& out, Legion::FieldID& fid) const; + void update_num_elements(size_t num_elements) const; public: Store(); diff --git a/src/core/data/store.inl b/src/core/data/store.inl index 27439cbe09..9097f9053b 100644 --- a/src/core/data/store.inl +++ b/src/core/data/store.inl @@ -163,7 +163,8 @@ AccessorRD Store::reduce_accessor(const Rect& bounds) c } template -Buffer Store::create_output_buffer(const Point& extents, bool bind_buffer /*= false*/) +Buffer Store::create_output_buffer(const Point& extents, + bool bind_buffer /*= false*/) const { check_valid_binding(bind_buffer); check_buffer_dimension(DIM); @@ -183,7 +184,7 @@ Rect Store::shape() const { check_shape_dimension(DIM); if (dim() > 0) { - return domain().bounds(); + return domain().bounds(); } else { auto p = Point::ZEROES(); return Rect(p, p); @@ -204,7 +205,7 @@ VAL Store::scalar() const } template -void Store::bind_data(Buffer& buffer, const Point& extents) +void Store::bind_data(Buffer& buffer, const Point& extents) const { check_valid_binding(true); check_buffer_dimension(DIM); diff --git a/src/core/legate_c.h b/src/core/legate_c.h index 8506e5c94b..f34095a2cd 100644 --- a/src/core/legate_c.h +++ b/src/core/legate_c.h @@ -28,6 +28,9 @@ typedef enum legate_core_task_id_t { LEGATE_CORE_INIT_CPUCOLL_MAPPING_TASK_ID, LEGATE_CORE_INIT_CPUCOLL_TASK_ID, LEGATE_CORE_FINALIZE_CPUCOLL_TASK_ID, + LEGATE_CORE_FIXUP_RANGES, + LEGATE_CORE_OFFSETS_TO_RANGES, + LEGATE_CORE_RANGES_TO_OFFSETS, LEGATE_CORE_NUM_TASK_IDS, // must be last } legate_core_task_id_t; @@ -91,6 +94,7 @@ typedef enum legate_core_type_code_t { STRUCT_LT, // Variable size types STRING_LT, + LIST_LT, INVALID_LT = -1, } legate_core_type_code_t; diff --git a/src/core/mapping/array.cc b/src/core/mapping/array.cc new file mode 100644 index 0000000000..d6f134a937 --- /dev/null +++ b/src/core/mapping/array.cc @@ -0,0 +1,50 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include "core/mapping/array.h" +#include "core/mapping/detail/array.h" + +namespace legate::mapping { + +bool Array::nullable() const { return impl_->nullable(); } + +int32_t Array::dim() const { return impl_->dim(); } + +Type Array::type() const { return Type(impl_->type()); } + +Store Array::data() const { return Store(impl_->data().get()); } + +Store Array::null_mask() const { return Store(impl_->null_mask().get()); } + +std::vector Array::stores() const +{ + std::vector result; + result.push_back(data()); + if (nullable()) result.push_back(null_mask()); + return std::move(result); +} + +Domain Array::domain() const { return impl_->domain(); } + +Array::Array(const detail::Array* impl) : impl_(impl) {} + +Array::Array(const Array&) = default; + +Array& Array::operator=(const Array&) = default; + +Array::Array(Array&&) = default; + +Array& Array::operator=(Array&&) = default; + +Array::~Array() {} + +} // namespace legate::mapping diff --git a/src/core/mapping/array.h b/src/core/mapping/array.h new file mode 100644 index 0000000000..40f18197b1 --- /dev/null +++ b/src/core/mapping/array.h @@ -0,0 +1,101 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#pragma once + +#include "core/mapping/store.h" + +namespace legate::mapping::detail { +class Array; +} // namespace legate::mapping::detail + +namespace legate::mapping { + +class Array { + public: + /** + * @brief Indicates if the array is nullable + * + * @return true If the array is nullable + * @return false Otherwise + */ + bool nullable() const; + /** + * @brief Returns the dimension of the array + * + * @return Array's dimension + */ + int32_t dim() const; + /** + * @brief Returns the array's type + * + * @return Type + */ + Type type() const; + + public: + /** + * @brief Returns metadata of the store containing the array's data + * + * @return Store metadata + */ + Store data() const; + /** + * @brief Returns metadata of the store containing the array's null mask + * + * @return Store metadata + * + * @throw std::invalid_argument If the array is not nullable + */ + Store null_mask() const; + /** + * @brief Returns metadat of all stores associated with this array + * + * @return Vector of store metadata + */ + std::vector stores() const; + + public: + /** + * @brief Returns the array's domain + * + * @return Array's domain + */ + template + Rect shape() const + { + return Rect(domain()); + } + /** + * @brief Returns the array's domain in a dimension-erased domain type + * + * @return Array's domain in a dimension-erased domain type + */ + Domain domain() const; + + public: + Array(const detail::Array* impl); + + public: + Array(const Array&); + Array& operator=(const Array&); + Array(Array&&); + Array& operator=(Array&&); + + public: + ~Array(); + + private: + const detail::Array* impl_{nullptr}; +}; + +} // namespace legate::mapping diff --git a/src/core/mapping/detail/array.cc b/src/core/mapping/detail/array.cc new file mode 100644 index 0000000000..b931cfa979 --- /dev/null +++ b/src/core/mapping/detail/array.cc @@ -0,0 +1,121 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include "core/mapping/detail/array.h" + +namespace legate::mapping::detail { + +std::shared_ptr Array::data() const +{ + throw std::invalid_argument("Data store of a nested array cannot be retrieved"); + return nullptr; +} + +BaseArray::BaseArray(std::shared_ptr data, std::shared_ptr null_mask) + : data_(std::move(data)), null_mask_(std::move(null_mask)) +{ +} + +bool BaseArray::unbound() const +{ +#ifdef DEBUG_LEGATE + assert(!nullable() || data_->unbound() == null_mask_->unbound()); +#endif + return data_->unbound(); +} + +std::shared_ptr BaseArray::null_mask() const +{ + if (!nullable()) { + throw std::invalid_argument("Invalid to retrieve the null mask of a non-nullable array"); + } + return null_mask_; +} + +std::shared_ptr BaseArray::child(uint32_t index) const +{ + throw std::invalid_argument("Non-nested array has no child sub-array"); + return nullptr; +} + +void BaseArray::_stores(std::vector>& result) const +{ + result.push_back(data_); + if (nullable()) result.push_back(null_mask_); +} + +Domain BaseArray::domain() const { return data_->domain(); } + +ListArray::ListArray(std::shared_ptr type, + std::shared_ptr descriptor, + std::shared_ptr vardata) + : type_(std::move(type)), descriptor_(std::move(descriptor)), vardata_(std::move(vardata)) +{ +} + +int32_t ListArray::dim() const { return descriptor_->dim(); } + +bool ListArray::unbound() const { return descriptor_->unbound() || vardata_->unbound(); } + +std::shared_ptr ListArray::child(uint32_t index) const +{ + switch (index) { + case 0: return descriptor_; + case 1: return vardata_; + default: { + throw std::out_of_range("List array does not have child " + std::to_string(index)); + break; + } + } + return nullptr; +} + +void ListArray::_stores(std::vector>& result) const +{ + descriptor_->_stores(result); + vardata_->_stores(result); +} + +Domain ListArray::domain() const { return descriptor_->domain(); } + +StructArray::StructArray(std::shared_ptr type, + std::shared_ptr null_mask, + std::vector>&& fields) + : type_(std::move(type)), null_mask_(std::move(null_mask)), fields_(std::move(fields)) +{ +} + +int32_t StructArray::dim() const { return fields_.front()->dim(); } + +bool StructArray::unbound() const +{ + return std::any_of(fields_.begin(), fields_.end(), [](auto& field) { return field->unbound(); }); +} + +std::shared_ptr StructArray::null_mask() const +{ + if (!nullable()) { + throw std::invalid_argument("Invalid to retrieve the null mask of a non-nullable array"); + } + return null_mask_; +} + +std::shared_ptr StructArray::child(uint32_t index) const { return fields_.at(index); } + +void StructArray::_stores(std::vector>& result) const +{ + std::for_each(fields_.begin(), fields_.end(), [&result](auto& field) { field->_stores(result); }); +} + +Domain StructArray::domain() const { return fields_.front()->domain(); } + +} // namespace legate::mapping::detail diff --git a/src/core/mapping/detail/array.h b/src/core/mapping/detail/array.h new file mode 100644 index 0000000000..0ba2087d96 --- /dev/null +++ b/src/core/mapping/detail/array.h @@ -0,0 +1,152 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#pragma once + +#include "core/data/detail/array_kind.h" +#include "core/mapping/detail/store.h" +#include "core/type/detail/type_info.h" + +namespace legate::mapping::detail { + +class Array { + public: + virtual ~Array() {} + virtual int32_t dim() const = 0; + virtual legate::detail::ArrayKind kind() const = 0; + virtual std::shared_ptr type() const = 0; + virtual bool unbound() const = 0; + virtual bool nullable() const = 0; + virtual bool nested() const = 0; + + public: + virtual std::shared_ptr data() const; + virtual std::shared_ptr null_mask() const = 0; + virtual std::shared_ptr child(uint32_t index) const = 0; + std::vector> stores() const + { + std::vector> result; + _stores(result); + return result; + } + virtual void _stores(std::vector>& result) const = 0; + + public: + virtual Domain domain() const = 0; + + protected: + std::shared_ptr null_mask_{nullptr}; +}; + +class BaseArray : public Array { + public: + BaseArray(std::shared_ptr data, std::shared_ptr null_mask); + + public: + BaseArray(const BaseArray& other) = default; + BaseArray& operator=(const BaseArray& other) = default; + BaseArray(BaseArray&& other) = default; + BaseArray& operator=(BaseArray&& other) = default; + + public: + int32_t dim() const override { return data_->dim(); } + legate::detail::ArrayKind kind() const override { return legate::detail::ArrayKind::BASE; } + std::shared_ptr type() const override { return data_->type(); } + bool unbound() const override; + bool nullable() const override { return null_mask_ != nullptr; } + bool nested() const override { return false; } + + public: + std::shared_ptr data() const override { return data_; } + std::shared_ptr null_mask() const override; + std::shared_ptr child(uint32_t index) const override; + void _stores(std::vector>& result) const override; + + public: + virtual Domain domain() const override; + + private: + std::shared_ptr data_; + std::shared_ptr null_mask_; +}; + +class ListArray : public Array { + public: + ListArray(std::shared_ptr type, + std::shared_ptr descriptor, + std::shared_ptr vardata); + + public: + ListArray(const ListArray& other) = default; + ListArray& operator=(const ListArray& other) = default; + ListArray(ListArray&& other) = default; + ListArray& operator=(ListArray&& other) = default; + + public: + int32_t dim() const override; + legate::detail::ArrayKind kind() const override { return legate::detail::ArrayKind::LIST; } + std::shared_ptr type() const override { return type_; } + bool unbound() const override; + bool nullable() const override { return vardata_->nullable(); } + bool nested() const override { return true; } + + public: + std::shared_ptr null_mask() const override { return descriptor_->null_mask(); } + std::shared_ptr child(uint32_t index) const override; + void _stores(std::vector>& result) const override; + std::shared_ptr descriptor() const { return descriptor_; } + std::shared_ptr vardata() const { return vardata_; } + + public: + virtual Domain domain() const override; + + private: + std::shared_ptr type_; + std::shared_ptr descriptor_; + std::shared_ptr vardata_; +}; + +class StructArray : public Array { + public: + StructArray(std::shared_ptr type, + std::shared_ptr null_mask, + std::vector>&& fields); + + public: + StructArray(const StructArray& other) = default; + StructArray& operator=(const StructArray& other) = default; + StructArray(StructArray&& other) = default; + StructArray& operator=(StructArray&& other) = default; + + public: + int32_t dim() const override; + legate::detail::ArrayKind kind() const override { return legate::detail::ArrayKind::STRUCT; } + std::shared_ptr type() const override { return type_; } + bool unbound() const override; + bool nullable() const override { return null_mask_ != nullptr; } + bool nested() const override { return true; } + + public: + std::shared_ptr null_mask() const override; + std::shared_ptr child(uint32_t index) const override; + void _stores(std::vector>& result) const override; + + public: + virtual Domain domain() const override; + + private: + std::shared_ptr type_; + std::shared_ptr null_mask_; + std::vector> fields_; +}; + +} // namespace legate::mapping::detail diff --git a/src/core/mapping/detail/base_mapper.cc b/src/core/mapping/detail/base_mapper.cc index 7bea9d3a80..abd8db7408 100644 --- a/src/core/mapping/detail/base_mapper.cc +++ b/src/core/mapping/detail/base_mapper.cc @@ -129,23 +129,29 @@ void BaseMapper::select_task_options(const Legion::Mapping::MapperContext ctx, #ifdef LEGATE_USE_COLLECTIVE auto hi = task.index_domain.hi(); auto lo = task.index_domain.lo(); - for (auto& store : legate_task.inputs()) { - if (store.is_future()) continue; - std::vector promoted_dims = store.find_imaginary_dims(); - for (auto& d : promoted_dims) { - if ((hi[d] - lo[d]) >= 1) { - output.check_collective_regions.insert(store.requirement_index()); - break; + for (auto& array : legate_task.inputs()) { + auto stores = array->stores(); + for (auto& store : stores) { + if (store->is_future()) continue; + std::vector promoted_dims = store->find_imaginary_dims(); + for (auto& d : promoted_dims) { + if ((hi[d] - lo[d]) >= 1) { + output.check_collective_regions.insert(store->requirement_index()); + break; + } } } } - for (auto& store : legate_task.reductions()) { - if (store.is_future()) continue; - auto idx = store.requirement_index(); - auto req = task.regions[idx]; - if (req.privilege & LEGION_WRITE_PRIV) continue; - if (req.handle_type == LEGION_SINGULAR_PROJECTION || req.projection != 0) { - output.check_collective_regions.insert(idx); + for (auto& array : legate_task.reductions()) { + auto stores = array->stores(); + for (auto& store : stores) { + if (store->is_future()) continue; + auto idx = store->requirement_index(); + auto req = task.regions[idx]; + if (req.privilege & LEGION_WRITE_PRIV) continue; + if (req.handle_type == LEGION_SINGULAR_PROJECTION || req.projection != 0) { + output.check_collective_regions.insert(idx); + } } } @@ -359,24 +365,28 @@ void BaseMapper::map_task(const Legion::Mapping::MapperContext ctx, // Generate default mappings for stores that are not yet mapped by the client mapper auto default_option = options.front(); - auto generate_default_mappings = [&](auto& stores, bool exact) { - for (auto& store : stores) { - auto mapping = StoreMapping::default_mapping(&store, default_option, exact); - if (store.is_future()) { - auto fut_idx = store.future_index(); - // Only need to map Future-backed Stores corresponding to inputs (i.e. one of task.futures) - if (fut_idx >= task.futures.size()) continue; - if (mapped_futures.find(fut_idx) != mapped_futures.end()) continue; - mapped_futures.insert(fut_idx); - for_futures.push_back(std::move(mapping)); - } else { - auto key = store.unique_region_field_id(); - if (mapped_regions.find(key) != mapped_regions.end()) continue; - mapped_regions.insert(key); - if (store.unbound()) - for_unbound_stores.push_back(std::move(mapping)); - else - for_stores.push_back(std::move(mapping)); + auto generate_default_mappings = [&](auto& arrays, bool exact) { + for (auto& array : arrays) { + auto stores = array->stores(); + for (auto& store : stores) { + auto mapping = StoreMapping::default_mapping(store.get(), default_option, exact); + if (store->is_future()) { + auto fut_idx = store->future_index(); + // Only need to map Future-backed Stores corresponding to inputs (i.e. one of + // task.futures) + if (fut_idx >= task.futures.size()) continue; + if (mapped_futures.find(fut_idx) != mapped_futures.end()) continue; + mapped_futures.insert(fut_idx); + for_futures.push_back(std::move(mapping)); + } else { + auto key = store->unique_region_field_id(); + if (mapped_regions.find(key) != mapped_regions.end()) continue; + mapped_regions.insert(key); + if (store->unbound()) + for_unbound_stores.push_back(std::move(mapping)); + else + for_stores.push_back(std::move(mapping)); + } } } }; @@ -547,9 +557,14 @@ bool BaseMapper::map_legate_store(const Legion::Mapping::MapperContext ctx, { if (reqs.empty()) return false; - const auto& policy = mapping.policy; std::vector regions; - for (auto* req : reqs) regions.push_back(req->region); + for (auto* req : reqs) { + if (LEGION_NO_ACCESS == req->privilege) continue; + regions.push_back(req->region); + } + if (regions.empty()) return false; + + const auto& policy = mapping.policy; auto target_memory = local_machine.get_memory(target_proc, policy.target); auto redop = (*reqs.begin())->redop; diff --git a/src/core/mapping/detail/operation.cc b/src/core/mapping/detail/operation.cc index 80c64380bc..f295533953 100644 --- a/src/core/mapping/detail/operation.cc +++ b/src/core/mapping/detail/operation.cc @@ -32,10 +32,10 @@ Task::Task(const Legion::Task* task, : Mappable(task), task_(task), library_(library) { TaskDeserializer dez(task, runtime, context); - inputs_ = dez.unpack(); - outputs_ = dez.unpack(); - reductions_ = dez.unpack(); - scalars_ = dez.unpack>(); + inputs_ = dez.unpack_arrays(); + outputs_ = dez.unpack_arrays(); + reductions_ = dez.unpack_arrays(); + scalars_ = dez.unpack_scalars(); } int64_t Task::task_id() const { return library_->get_local_task_id(task_->task_id); } diff --git a/src/core/mapping/detail/operation.h b/src/core/mapping/detail/operation.h index 9a812aa80a..1494fa7b76 100644 --- a/src/core/mapping/detail/operation.h +++ b/src/core/mapping/detail/operation.h @@ -12,15 +12,11 @@ #pragma once -#include "core/data/scalar.h" +#include "core/data/detail/scalar.h" +#include "core/mapping/detail/array.h" #include "core/mapping/detail/machine.h" #include "core/mapping/detail/store.h" -/** - * @file - * @brief Class definitions for operations and stores used in mapping - */ - namespace legate::detail { class Library; } // namespace legate::detail @@ -28,6 +24,7 @@ class Library; namespace legate::mapping::detail { namespace { +using Arrays = std::vector>; using Stores = std::vector; } // namespace @@ -47,10 +44,6 @@ class Mappable { uint32_t sharding_id_; }; -/** - * @ingroup mapping - * @brief A metadata class for tasks - */ class Task : public Mappable { public: Task(const Legion::Task* task, @@ -59,47 +52,15 @@ class Task : public Mappable { const Legion::Mapping::MapperContext context); public: - /** - * @brief Returns the task id - * - * @return Task id - */ int64_t task_id() const; public: - /** - * @brief Returns metadata for the task's input stores - * - * @return Vector of store metadata objects - */ - const Stores& inputs() const { return inputs_; } - /** - * @brief Returns metadata for the task's output stores - * - * @return Vector of store metadata objects - */ - const Stores& outputs() const { return outputs_; } - /** - * @brief Returns metadata for the task's reduction stores - * - * @return Vector of store metadata objects - */ - const Stores& reductions() const { return reductions_; } - /** - * @brief Returns the vector of the task's by-value arguments. Unlike `mapping::Store` - * objects that have no access to data in the stores, the returned `Scalar` objects - * contain valid arguments to the task - * - * @return Vector of `Scalar` objects - */ - const std::vector& scalars() const { return scalars_; } + const Arrays& inputs() const { return inputs_; } + const Arrays& outputs() const { return outputs_; } + const Arrays& reductions() const { return reductions_; } + const std::vector& scalars() const { return scalars_; } public: - /** - * @brief Returns the point of the task - * - * @return The point of the task - */ DomainPoint point() const { return task_->index_point; } public: @@ -110,7 +71,7 @@ class Task : public Mappable { const Legion::Task* task_; private: - Stores inputs_, outputs_, reductions_; + Arrays inputs_, outputs_, reductions_; std::vector scalars_; }; diff --git a/src/core/mapping/detail/store.h b/src/core/mapping/detail/store.h index 9478dd019d..e11e3db069 100644 --- a/src/core/mapping/detail/store.h +++ b/src/core/mapping/detail/store.h @@ -116,6 +116,7 @@ class Store { bool is_future() const { return is_future_; } bool unbound() const { return is_unbound_store_; } int32_t dim() const { return dim_; } + std::shared_ptr type() const { return type_; } public: bool is_reduction() const { return redop_id_ > 0; } diff --git a/src/core/mapping/operation.cc b/src/core/mapping/operation.cc index ddb23aff4a..13743bab53 100644 --- a/src/core/mapping/operation.cc +++ b/src/core/mapping/operation.cc @@ -11,6 +11,7 @@ */ #include "core/mapping/operation.h" +#include "core/mapping/detail/array.h" #include "core/mapping/detail/operation.h" namespace legate::mapping { @@ -19,29 +20,29 @@ int64_t Task::task_id() const { return impl_->task_id(); } namespace { -template -std::vector convert_stores(const Stores& stores) +template +std::vector convert_arrays(const Arrays& arrays) { - std::vector result; - for (auto& store : stores) { result.emplace_back(&store); } + std::vector result; + for (auto& array : arrays) { result.emplace_back(array.get()); } return std::move(result); } } // namespace -std::vector Task::inputs() const { return convert_stores(impl_->inputs()); } +std::vector Task::inputs() const { return convert_arrays(impl_->inputs()); } -std::vector Task::outputs() const { return convert_stores(impl_->outputs()); } +std::vector Task::outputs() const { return convert_arrays(impl_->outputs()); } -std::vector Task::reductions() const { return convert_stores(impl_->reductions()); } +std::vector Task::reductions() const { return convert_arrays(impl_->reductions()); } const std::vector& Task::scalars() const { return impl_->scalars(); } -Store Task::input(uint32_t index) const { return Store(&impl_->inputs().at(index)); } +Array Task::input(uint32_t index) const { return Array(impl_->inputs().at(index).get()); } -Store Task::output(uint32_t index) const { return Store(&impl_->outputs().at(index)); } +Array Task::output(uint32_t index) const { return Array(impl_->outputs().at(index).get()); } -Store Task::reduction(uint32_t index) const { return Store(&impl_->reductions().at(index)); } +Array Task::reduction(uint32_t index) const { return Array(impl_->reductions().at(index).get()); } size_t Task::num_inputs() const { return impl_->inputs().size(); } diff --git a/src/core/mapping/operation.h b/src/core/mapping/operation.h index 7f1084361f..a5346014a8 100644 --- a/src/core/mapping/operation.h +++ b/src/core/mapping/operation.h @@ -15,7 +15,7 @@ #include #include "core/data/scalar.h" -#include "core/mapping/store.h" +#include "core/mapping/array.h" /** * @file @@ -43,26 +43,26 @@ class Task { public: /** - * @brief Returns metadata for the task's input stores + * @brief Returns metadata for the task's input arrays * - * @return Vector of store metadata objects + * @return Vector of array metadata objects */ - std::vector inputs() const; + std::vector inputs() const; /** - * @brief Returns metadata for the task's output stores + * @brief Returns metadata for the task's output arrays * - * @return Vector of store metadata objects + * @return Vector of array metadata objects */ - std::vector outputs() const; + std::vector outputs() const; /** - * @brief Returns metadata for the task's reduction stores + * @brief Returns metadata for the task's reduction arrays * - * @return Vector of store metadata objects + * @return Vector of array metadata objects */ - std::vector reductions() const; + std::vector reductions() const; /** - * @brief Returns the vector of the task's by-value arguments. Unlike `mapping::Store` - * objects that have no access to data in the stores, the returned `Scalar` objects + * @brief Returns the vector of the task's by-value arguments. Unlike `mapping::Array` + * objects that have no access to data in the arrays, the returned `Scalar` objects * contain valid arguments to the task * * @return Vector of `Scalar` objects @@ -71,47 +71,47 @@ class Task { public: /** - * @brief Returns metadata for the task's input store + * @brief Returns metadata for the task's input array * - * @param index Index of the input store + * @param index Index of the input array * - * @return Store metadata object + * @return Array metadata object */ - Store input(uint32_t index) const; + Array input(uint32_t index) const; /** - * @brief Returns metadata for the task's output store + * @brief Returns metadata for the task's output array * - * @param index Index of the output store + * @param index Index of the output array * - * @return Store metadata object + * @return Array metadata object */ - Store output(uint32_t index) const; + Array output(uint32_t index) const; /** - * @brief Returns metadata for the task's reduction store + * @brief Returns metadata for the task's reduction array * - * @param index Index of the reduction store + * @param index Index of the reduction array * - * @return Store metadata object + * @return Array metadata object */ - Store reduction(uint32_t index) const; + Array reduction(uint32_t index) const; public: /** * @brief Returns the number of task's inputs * - * @return Number of stores + * @return Number of arrays */ size_t num_inputs() const; /** * @brief Returns the number of task's outputs * - * @return Number of stores + * @return Number of arrays */ size_t num_outputs() const; /** * @brief Returns the number of task's reductions * - * @return Number of stores + * @return Number of arrays */ size_t num_reductions() const; diff --git a/src/core/operation/detail/copy.cc b/src/core/operation/detail/copy.cc index df62ee81c3..2feb94cf29 100644 --- a/src/core/operation/detail/copy.cc +++ b/src/core/operation/detail/copy.cc @@ -77,7 +77,7 @@ void Copy::launch(Strategy* p_strategy) void Copy::add_to_solver(ConstraintSolver& solver) { - solver.add_constraint(constraint_.get()); + solver.add_constraint(std::move(constraint_)); solver.add_partition_symbol(target_.variable); solver.add_partition_symbol(source_.variable); } diff --git a/src/core/operation/detail/copy_launcher.cc b/src/core/operation/detail/copy_launcher.cc index a079d38923..6e873ca9da 100644 --- a/src/core/operation/detail/copy_launcher.cc +++ b/src/core/operation/detail/copy_launcher.cc @@ -21,7 +21,7 @@ namespace legate::detail { -struct CopyArg : public ArgWrapper { +struct CopyArg : public Serializable { public: CopyArg(uint32_t req_idx, LogicalStore* store, @@ -79,13 +79,10 @@ void CopyArg::pack(BufferBuilder& buffer) const CopyLauncher::CopyLauncher(const mapping::detail::Machine& machine, int64_t tag) : machine_(machine), tag_(tag) { - mapper_arg_ = new BufferBuilder(); - machine_.pack(*mapper_arg_); } CopyLauncher::~CopyLauncher() { - delete mapper_arg_; for (auto& arg : inputs_) delete arg; for (auto& arg : outputs_) delete arg; for (auto& arg : source_indirect_) delete arg; @@ -140,28 +137,48 @@ void CopyLauncher::add_target_indirect(detail::LogicalStore* store, void CopyLauncher::execute(const Legion::Domain& launch_domain) { - auto legion_copy_launcher = build_index_copy(launch_domain); - return Runtime::get_runtime()->dispatch(legion_copy_launcher.get()); + BufferBuilder mapper_arg; + pack_args(mapper_arg); + auto* runtime = Runtime::get_runtime(); + auto& provenance = runtime->provenance_manager()->get_provenance(); + Legion::IndexCopyLauncher index_copy(launch_domain, + Legion::Predicate::TRUE_PRED, + runtime->core_library()->get_mapper_id(), + tag_, + mapper_arg.to_legion_buffer(), + provenance.c_str()); + populate_copy(index_copy); + Runtime::get_runtime()->dispatch(index_copy); } void CopyLauncher::execute_single() { - auto legion_copy_launcher = build_single_copy(); - return Runtime::get_runtime()->dispatch(legion_copy_launcher.get()); + BufferBuilder mapper_arg; + pack_args(mapper_arg); + auto* runtime = Runtime::get_runtime(); + auto& provenance = runtime->provenance_manager()->get_provenance(); + Legion::CopyLauncher single_copy(Legion::Predicate::TRUE_PRED, + runtime->core_library()->get_mapper_id(), + tag_, + mapper_arg.to_legion_buffer(), + provenance.c_str()); + populate_copy(single_copy); + return Runtime::get_runtime()->dispatch(single_copy); } -void CopyLauncher::pack_sharding_functor_id() +void CopyLauncher::pack_sharding_functor_id(BufferBuilder& buffer) { - mapper_arg_->pack(Runtime::get_runtime()->get_sharding(machine_, key_proj_id_)); + buffer.pack(Runtime::get_runtime()->get_sharding(machine_, key_proj_id_)); } -void CopyLauncher::pack_args() +void CopyLauncher::pack_args(BufferBuilder& buffer) { - pack_sharding_functor_id(); + machine_.pack(buffer); + pack_sharding_functor_id(buffer); - auto pack_args = [&](const std::vector& args) { - mapper_arg_->pack(args.size()); - for (auto& arg : args) arg->pack(*mapper_arg_); + auto pack_args = [&buffer](const std::vector& args) { + buffer.pack(args.size()); + for (auto& arg : args) arg->pack(buffer); }; pack_args(inputs_); pack_args(outputs_); @@ -181,7 +198,7 @@ constexpr bool is_single = false; } // namespace template -void CopyLauncher::populate_copy(Launcher* launcher) +void CopyLauncher::populate_copy(Launcher& launcher) { auto populate_requirements = [&](auto& args, auto& requirements) { requirements.resize(args.size()); @@ -192,50 +209,16 @@ void CopyLauncher::populate_copy(Launcher* launcher) } }; - populate_requirements(inputs_, launcher->src_requirements); - populate_requirements(outputs_, launcher->dst_requirements); - populate_requirements(source_indirect_, launcher->src_indirect_requirements); - populate_requirements(target_indirect_, launcher->dst_indirect_requirements); + populate_requirements(inputs_, launcher.src_requirements); + populate_requirements(outputs_, launcher.dst_requirements); + populate_requirements(source_indirect_, launcher.src_indirect_requirements); + populate_requirements(target_indirect_, launcher.dst_indirect_requirements); - launcher->src_indirect_is_range.resize(source_indirect_.size(), false); - launcher->dst_indirect_is_range.resize(target_indirect_.size(), false); + launcher.src_indirect_is_range.resize(source_indirect_.size(), false); + launcher.dst_indirect_is_range.resize(target_indirect_.size(), false); - launcher->possible_src_indirect_out_of_range = source_indirect_out_of_range_; - launcher->possible_dst_indirect_out_of_range = target_indirect_out_of_range_; -} - -std::unique_ptr CopyLauncher::build_index_copy( - const Legion::Domain& launch_domain) -{ - pack_args(); - auto* runtime = Runtime::get_runtime(); - auto& provenance = runtime->provenance_manager()->get_provenance(); - auto index_copy = - std::make_unique(launch_domain, - Legion::Predicate::TRUE_PRED, - runtime->core_library()->get_mapper_id(), - tag_, - mapper_arg_->to_legion_buffer(), - provenance.c_str()); - - populate_copy(index_copy.get()); - return std::move(index_copy); -} - -std::unique_ptr CopyLauncher::build_single_copy() -{ - pack_args(); - auto* runtime = Runtime::get_runtime(); - auto& provenance = runtime->provenance_manager()->get_provenance(); - auto single_copy = - std::make_unique(Legion::Predicate::TRUE_PRED, - runtime->core_library()->get_mapper_id(), - tag_, - mapper_arg_->to_legion_buffer(), - provenance.c_str()); - - populate_copy(single_copy.get()); - return std::move(single_copy); + launcher.possible_src_indirect_out_of_range = source_indirect_out_of_range_; + launcher.possible_dst_indirect_out_of_range = target_indirect_out_of_range_; } } // namespace legate::detail diff --git a/src/core/operation/detail/copy_launcher.h b/src/core/operation/detail/copy_launcher.h index 26d1989825..5ef595b98f 100644 --- a/src/core/operation/detail/copy_launcher.h +++ b/src/core/operation/detail/copy_launcher.h @@ -24,10 +24,10 @@ class Scalar; namespace legate::detail { +class CopyArg; class LogicalStore; class ProjectionInfo; class OutputRequirementAnalyzer; -class CopyArg; class RequirementAnalyzer; class CopyLauncher { @@ -58,20 +58,17 @@ class CopyLauncher { void execute_single(); private: - void pack_args(); - void pack_sharding_functor_id(); - std::unique_ptr build_index_copy(const Legion::Domain& launch_domain); - std::unique_ptr build_single_copy(); + void pack_args(BufferBuilder& buffer); + void pack_sharding_functor_id(BufferBuilder& buffer); template - void populate_copy(Launcher* launcher); + void populate_copy(Launcher& launcher); private: - mapping::detail::Machine machine_; + const mapping::detail::Machine& machine_; int64_t tag_; Legion::ProjectionID key_proj_id_{0}; private: - BufferBuilder* mapper_arg_; std::vector inputs_{}; std::vector outputs_{}; std::vector source_indirect_{}; diff --git a/src/core/operation/detail/fill_launcher.cc b/src/core/operation/detail/fill_launcher.cc index e88f3877ca..b90f2f832c 100644 --- a/src/core/operation/detail/fill_launcher.cc +++ b/src/core/operation/detail/fill_launcher.cc @@ -22,42 +22,17 @@ namespace legate::detail { FillLauncher::FillLauncher(const mapping::detail::Machine& machine, int64_t tag) - : machine_(machine), tag_(tag), mapper_arg_(new BufferBuilder()) + : machine_(machine), tag_(tag) { } -FillLauncher::~FillLauncher() { delete mapper_arg_; } - void FillLauncher::launch(const Legion::Domain& launch_domain, LogicalStore* lhs, const ProjectionInfo& lhs_proj, LogicalStore* value) { - auto legion_fill_launcher = build_index_fill(launch_domain, lhs, lhs_proj, value); - return Runtime::get_runtime()->dispatch(legion_fill_launcher.get()); -} - -void FillLauncher::launch_single(LogicalStore* lhs, - const ProjectionInfo& lhs_proj, - LogicalStore* value) -{ - auto legion_fill_launcher = build_single_fill(lhs, lhs_proj, value); - return Runtime::get_runtime()->dispatch(legion_fill_launcher.get()); -} - -void FillLauncher::pack_mapper_arg(Legion::ProjectionID proj_id) -{ - machine_.pack(*mapper_arg_); - mapper_arg_->pack(Runtime::get_runtime()->get_sharding(machine_, proj_id)); -} - -std::unique_ptr FillLauncher::build_index_fill( - const Legion::Domain& launch_domain, - LogicalStore* lhs, - const ProjectionInfo& lhs_proj, - LogicalStore* value) -{ - pack_mapper_arg(lhs_proj.proj_id); + BufferBuilder mapper_arg; + pack_mapper_arg(mapper_arg, lhs_proj.proj_id); auto* runtime = Runtime::get_runtime(); auto& provenance = runtime->provenance_manager()->get_provenance(); auto lhs_region_field = lhs->get_region_field(); @@ -65,26 +40,27 @@ std::unique_ptr FillLauncher::build_index_fill( auto field_id = lhs_region_field->field_id(); auto future_value = value->get_future(); auto lhs_parent = runtime->find_parent_region(lhs_region); - auto index_fill = - std::make_unique(launch_domain, - lhs_proj.partition, - lhs_parent, - future_value, - lhs_proj.proj_id, - Legion::Predicate::TRUE_PRED, - runtime->core_library()->get_mapper_id(), - lhs_proj.tag, - mapper_arg_->to_legion_buffer(), - provenance.c_str()); + Legion::IndexFillLauncher index_fill(launch_domain, + lhs_proj.partition, + lhs_parent, + future_value, + lhs_proj.proj_id, + Legion::Predicate::TRUE_PRED, + runtime->core_library()->get_mapper_id(), + lhs_proj.tag, + mapper_arg.to_legion_buffer(), + provenance.c_str()); - index_fill->add_field(field_id); - return std::move(index_fill); + index_fill.add_field(field_id); + Runtime::get_runtime()->dispatch(index_fill); } -std::unique_ptr FillLauncher::build_single_fill( - LogicalStore* lhs, const ProjectionInfo& lhs_proj, LogicalStore* value) +void FillLauncher::launch_single(LogicalStore* lhs, + const ProjectionInfo& lhs_proj, + LogicalStore* value) { - pack_mapper_arg(lhs_proj.proj_id); + BufferBuilder mapper_arg; + pack_mapper_arg(mapper_arg, lhs_proj.proj_id); auto* runtime = Runtime::get_runtime(); auto& provenance = runtime->provenance_manager()->get_provenance(); auto lhs_region_field = lhs->get_region_field(); @@ -92,18 +68,23 @@ std::unique_ptr FillLauncher::build_single_fill( auto field_id = lhs_region_field->field_id(); auto future_value = value->get_future(); auto lhs_parent = runtime->find_parent_region(lhs_region); - auto single_fill = - std::make_unique(lhs_region, - lhs_parent, - future_value, - Legion::Predicate::TRUE_PRED, - runtime->core_library()->get_mapper_id(), - lhs_proj.tag, - mapper_arg_->to_legion_buffer(), - provenance.c_str()); + Legion::FillLauncher single_fill(lhs_region, + lhs_parent, + future_value, + Legion::Predicate::TRUE_PRED, + runtime->core_library()->get_mapper_id(), + lhs_proj.tag, + mapper_arg.to_legion_buffer(), + provenance.c_str()); + + single_fill.add_field(field_id); + Runtime::get_runtime()->dispatch(single_fill); +} - single_fill->add_field(field_id); - return std::move(single_fill); +void FillLauncher::pack_mapper_arg(BufferBuilder& buffer, Legion::ProjectionID proj_id) +{ + machine_.pack(buffer); + buffer.pack(Runtime::get_runtime()->get_sharding(machine_, proj_id)); } } // namespace legate::detail diff --git a/src/core/operation/detail/fill_launcher.h b/src/core/operation/detail/fill_launcher.h index b28a5f245d..9d7b70e716 100644 --- a/src/core/operation/detail/fill_launcher.h +++ b/src/core/operation/detail/fill_launcher.h @@ -28,7 +28,6 @@ class ProjectionInfo; class FillLauncher { public: FillLauncher(const mapping::detail::Machine& machine, int64_t tag = 0); - ~FillLauncher(); public: void launch(const Legion::Domain& launch_domain, @@ -38,21 +37,11 @@ class FillLauncher { void launch_single(LogicalStore* lhs, const ProjectionInfo& lhs_proj, LogicalStore* value); private: - void pack_mapper_arg(Legion::ProjectionID proj_id); - std::unique_ptr build_index_fill(const Legion::Domain& launch_domain, - LogicalStore* lhs, - const ProjectionInfo& lhs_proj, - LogicalStore* value); - std::unique_ptr build_single_fill(LogicalStore* lhs, - const ProjectionInfo& lhs_proj, - LogicalStore* value); + void pack_mapper_arg(BufferBuilder& buffer, Legion::ProjectionID proj_id); private: - mapping::detail::Machine machine_; + const mapping::detail::Machine& machine_; int64_t tag_; - - private: - BufferBuilder* mapper_arg_; }; } // namespace legate::detail diff --git a/src/core/operation/detail/gather.cc b/src/core/operation/detail/gather.cc index bcd417372f..3dc03f86d0 100644 --- a/src/core/operation/detail/gather.cc +++ b/src/core/operation/detail/gather.cc @@ -91,7 +91,7 @@ void Gather::launch(Strategy* p_strategy) void Gather::add_to_solver(ConstraintSolver& solver) { - solver.add_constraint(constraint_.get()); + solver.add_constraint(std::move(constraint_)); solver.add_partition_symbol(target_.variable); solver.add_partition_symbol(source_.variable); solver.add_partition_symbol(source_indirect_.variable); diff --git a/src/core/operation/detail/launcher_arg.cc b/src/core/operation/detail/launcher_arg.cc index c63b415bf9..c3106b9acd 100644 --- a/src/core/operation/detail/launcher_arg.cc +++ b/src/core/operation/detail/launcher_arg.cc @@ -12,59 +12,82 @@ #include "core/operation/detail/launcher_arg.h" +#include "core/data/detail/array_kind.h" #include "core/data/detail/logical_region_field.h" #include "core/operation/detail/req_analyzer.h" +#include "core/operation/detail/task_launcher.h" #include "core/type/detail/type_info.h" #include "core/utilities/detail/buffer_builder.h" namespace legate::detail { -void UntypedScalarArg::pack(BufferBuilder& buffer) const { scalar_.pack(buffer); } +void ScalarArg::pack(BufferBuilder& buffer) const { scalar_.pack(buffer); } -RegionFieldArg::RegionFieldArg(RequirementAnalyzer* analyzer, - LogicalStore* store, - Legion::FieldID field_id, +RegionFieldArg::RegionFieldArg(LogicalStore* store, Legion::PrivilegeMode privilege, std::unique_ptr proj_info) - : analyzer_(analyzer), - store_(store), - region_(store_->get_region_field()->region()), - field_id_(field_id), - privilege_(privilege), - proj_info_(std::move(proj_info)) + : store_(store), privilege_(privilege), proj_info_(std::move(proj_info)) { } -void RegionFieldArg::pack(BufferBuilder& buffer) const +void RegionFieldArg::pack(BufferBuilder& buffer, const StoreAnalyzer& analyzer) const { store_->pack(buffer); + auto region = store_->get_region_field()->region(); + auto field_id = store_->get_region_field()->field_id(); + buffer.pack(proj_info_->redop); - buffer.pack(region_.get_dim()); - buffer.pack(analyzer_->get_requirement_index(region_, privilege_, *proj_info_)); - buffer.pack(field_id_); + buffer.pack(region.get_dim()); + buffer.pack(analyzer.get_index(region, privilege_, *proj_info_)); + buffer.pack(field_id); } -OutputRegionArg::OutputRegionArg(OutputRequirementAnalyzer* analyzer, - LogicalStore* store, - Legion::FieldSpace field_space, - Legion::FieldID field_id) - : analyzer_(analyzer), store_(store), field_space_(field_space), field_id_(field_id) +void RegionFieldArg::analyze(StoreAnalyzer& analyzer) { + analyzer.insert(store_->get_region_field(), privilege_, *proj_info_); } -void OutputRegionArg::pack(BufferBuilder& buffer) const +std::optional RegionFieldArg::get_key_proj_id() const { - store_->pack(buffer); + return LEGATE_CORE_KEY_STORE_TAG == proj_info_->tag ? std::make_optional(proj_info_->proj_id) + : std::nullopt; +} + +void RegionFieldArg::perform_invalidations() const +{ + store_->get_region_field()->perform_invalidation_callbacks(); +} + +OutputRegionArg::OutputRegionArg(LogicalStore* store, Legion::FieldSpace field_space) + : store_(store), field_space_(field_space) +{ + // TODO: We should reuse field ids here + field_id_ = Runtime::get_runtime()->allocate_field(field_space_, store->type()->size()); +} - requirement_index_ = analyzer_->get_requirement_index(field_space_, field_id_); +void OutputRegionArg::pack(BufferBuilder& buffer, const StoreAnalyzer& analyzer) const +{ + store_->pack(buffer); buffer.pack(-1); buffer.pack(store_->dim()); + // Need to cache the requirement index for post-processing + requirement_index_ = analyzer.get_index(field_space_, field_id_); buffer.pack(requirement_index_); buffer.pack(field_id_); } +void OutputRegionArg::analyze(StoreAnalyzer& analyzer) +{ + analyzer.insert(store_->dim(), field_space_, field_id_); +} + +void OutputRegionArg::record_unbound_stores(std::vector& args) const +{ + args.push_back(this); +} + FutureStoreArg::FutureStoreArg(LogicalStore* store, bool read_only, bool has_storage, @@ -73,15 +96,145 @@ FutureStoreArg::FutureStoreArg(LogicalStore* store, { } -void FutureStoreArg::pack(BufferBuilder& buffer) const +void FutureStoreArg::pack(BufferBuilder& buffer, const StoreAnalyzer& analyzer) const { store_->pack(buffer); buffer.pack(redop_); buffer.pack(read_only_); - buffer.pack(has_storage_); + buffer.pack(has_storage_ ? analyzer.get_index(store_->get_future()) : -1); buffer.pack(store_->type()->size()); buffer.pack(store_->get_storage()->extents().data()); } +void FutureStoreArg::analyze(StoreAnalyzer& analyzer) +{ + if (!has_storage_) return; + analyzer.insert(store_->get_future()); +} + +BaseArrayArg::BaseArrayArg(std::unique_ptr data) : data_(std::move(data)) {} + +BaseArrayArg::BaseArrayArg(std::unique_ptr data, std::unique_ptr null_mask) + : data_(std::move(data)), null_mask_(std::move(null_mask)) +{ +} + +void BaseArrayArg::pack(BufferBuilder& buffer, const StoreAnalyzer& analyzer) const +{ + buffer.pack(static_cast(ArrayKind::BASE)); + data_->pack(buffer, analyzer); + + bool nullable = null_mask_ != nullptr; + buffer.pack(nullable); + if (nullable) { null_mask_->pack(buffer, analyzer); } +} + +void BaseArrayArg::analyze(StoreAnalyzer& analyzer) +{ + data_->analyze(analyzer); + if (null_mask_ != nullptr) null_mask_->analyze(analyzer); +} + +std::optional BaseArrayArg::get_key_proj_id() const +{ + return data_->get_key_proj_id(); +} + +void BaseArrayArg::record_unbound_stores(std::vector& args) const +{ + data_->record_unbound_stores(args); + if (null_mask_ != nullptr) null_mask_->record_unbound_stores(args); +} + +void BaseArrayArg::perform_invalidations() const +{ + // We don't need to invalidate any cached state for null masks + data_->perform_invalidations(); +} + +ListArrayArg::ListArrayArg(std::shared_ptr type, + std::unique_ptr descriptor, + std::unique_ptr vardata) + : type_(std::move(type)), descriptor_(std::move(descriptor)), vardata_(std::move(vardata)) +{ +} + +void ListArrayArg::pack(BufferBuilder& buffer, const StoreAnalyzer& analyzer) const +{ + buffer.pack(static_cast(ArrayKind::LIST)); + type_->pack(buffer); + descriptor_->pack(buffer, analyzer); + vardata_->pack(buffer, analyzer); +} + +void ListArrayArg::analyze(StoreAnalyzer& analyzer) +{ + descriptor_->analyze(analyzer); + vardata_->analyze(analyzer); +} + +std::optional ListArrayArg::get_key_proj_id() const +{ + return vardata_->get_key_proj_id(); +} + +void ListArrayArg::record_unbound_stores(std::vector& args) const +{ + descriptor_->record_unbound_stores(args); + vardata_->record_unbound_stores(args); +} + +void ListArrayArg::perform_invalidations() const +{ + descriptor_->perform_invalidations(); + vardata_->perform_invalidations(); +} + +StructArrayArg::StructArrayArg(std::shared_ptr type, + std::unique_ptr null_mask, + std::vector>&& fields) + : type_(std::move(type)), null_mask_(std::move(null_mask)), fields_(std::move(fields)) +{ +} + +void StructArrayArg::pack(BufferBuilder& buffer, const StoreAnalyzer& analyzer) const +{ + buffer.pack(static_cast(ArrayKind::STRUCT)); + type_->pack(buffer); + + bool nullable = null_mask_ != nullptr; + buffer.pack(nullable); + if (nullable) { null_mask_->pack(buffer, analyzer); } + + for (auto& field : fields_) { field->pack(buffer, analyzer); } +} + +void StructArrayArg::analyze(StoreAnalyzer& analyzer) +{ + if (null_mask_ != nullptr) { null_mask_->analyze(analyzer); } + for (auto& field : fields_) { field->analyze(analyzer); } +} + +std::optional StructArrayArg::get_key_proj_id() const +{ + for (auto& field : fields_) { + auto proj_id = field->get_key_proj_id(); + if (proj_id.has_value()) { return proj_id; } + } + return std::nullopt; +} + +void StructArrayArg::record_unbound_stores(std::vector& args) const +{ + if (null_mask_ != nullptr) { null_mask_->record_unbound_stores(args); } + for (auto& field : fields_) { field->record_unbound_stores(args); } +} + +void StructArrayArg::perform_invalidations() const +{ + if (null_mask_ != nullptr) { null_mask_->perform_invalidations(); } + for (auto& field : fields_) { field->perform_invalidations(); } +} + } // namespace legate::detail diff --git a/src/core/operation/detail/launcher_arg.h b/src/core/operation/detail/launcher_arg.h index be4faccf45..0e4f0d9c3b 100644 --- a/src/core/operation/detail/launcher_arg.h +++ b/src/core/operation/detail/launcher_arg.h @@ -12,43 +12,40 @@ #pragma once +#include + #include "core/data/detail/logical_store.h" #include "core/data/detail/scalar.h" #include "core/data/scalar.h" +#include "core/operation/detail/projection.h" namespace legate::detail { class BufferBuilder; -class OutputRequirementAnalyzer; -class ProjectionInfo; -class RequirementAnalyzer; +class OutputRegionArg; +class StoreAnalyzer; +class TaskLauncher; -struct ArgWrapper { - virtual ~ArgWrapper() {} +struct Serializable { + virtual ~Serializable() {} virtual void pack(BufferBuilder& buffer) const = 0; }; -template -struct ScalarArg : public ArgWrapper { - public: - ScalarArg(const T& value) : value_(value) {} - - public: - ~ScalarArg() {} - - public: - void pack(BufferBuilder& buffer) const override { buffer.pack(value_); } - - private: - T value_; +struct Analyzable { + virtual ~Analyzable() {} + virtual void pack(BufferBuilder& buffer, const StoreAnalyzer& analyzer) const = 0; + virtual void analyze(StoreAnalyzer& analyzer) = 0; + virtual std::optional get_key_proj_id() const { return std::nullopt; } + virtual void record_unbound_stores(std::vector& args) const {} + virtual void perform_invalidations() const {} }; -struct UntypedScalarArg : public ArgWrapper { +struct ScalarArg : public Serializable { public: - UntypedScalarArg(Scalar&& scalar) : scalar_(std::move(scalar)) {} + ScalarArg(Scalar&& scalar) : scalar_(std::move(scalar)) {} public: - ~UntypedScalarArg() {} + ~ScalarArg() {} public: void pack(BufferBuilder& buffer) const override; @@ -57,38 +54,35 @@ struct UntypedScalarArg : public ArgWrapper { Scalar scalar_; }; -struct RegionFieldArg : public ArgWrapper { +struct RegionFieldArg : public Analyzable { public: - RegionFieldArg(RequirementAnalyzer* analyzer, - LogicalStore* store, - Legion::FieldID field_id, + RegionFieldArg(LogicalStore* store, Legion::PrivilegeMode privilege, std::unique_ptr proj_info); public: - void pack(BufferBuilder& buffer) const override; + void pack(BufferBuilder& buffer, const StoreAnalyzer& analyzer) const override; + void analyze(StoreAnalyzer& analyzer) override; + std::optional get_key_proj_id() const override; + void perform_invalidations() const override; public: ~RegionFieldArg() {} private: - RequirementAnalyzer* analyzer_; LogicalStore* store_; - Legion::LogicalRegion region_; - Legion::FieldID field_id_; Legion::PrivilegeMode privilege_; std::unique_ptr proj_info_; }; -struct OutputRegionArg : public ArgWrapper { +struct OutputRegionArg : public Analyzable { public: - OutputRegionArg(OutputRequirementAnalyzer* analyzer, - LogicalStore* store, - Legion::FieldSpace field_space, - Legion::FieldID field_id); + OutputRegionArg(LogicalStore* store, Legion::FieldSpace field_space); public: - void pack(BufferBuilder& buffer) const override; + void pack(BufferBuilder& buffer, const StoreAnalyzer& analyzer) const override; + void analyze(StoreAnalyzer& analyzer) override; + void record_unbound_stores(std::vector& args) const override; public: ~OutputRegionArg() {} @@ -100,14 +94,13 @@ struct OutputRegionArg : public ArgWrapper { uint32_t requirement_index() const { return requirement_index_; } private: - OutputRequirementAnalyzer* analyzer_; LogicalStore* store_; Legion::FieldSpace field_space_; Legion::FieldID field_id_; mutable uint32_t requirement_index_{-1U}; }; -struct FutureStoreArg : public ArgWrapper { +struct FutureStoreArg : public Analyzable { public: FutureStoreArg(LogicalStore* store, bool read_only, @@ -118,7 +111,8 @@ struct FutureStoreArg : public ArgWrapper { ~FutureStoreArg() {} public: - void pack(BufferBuilder& buffer) const override; + void pack(BufferBuilder& buffer, const StoreAnalyzer& analyzer) const override; + void analyze(StoreAnalyzer& analyzer) override; private: LogicalStore* store_; @@ -127,4 +121,68 @@ struct FutureStoreArg : public ArgWrapper { Legion::ReductionOpID redop_; }; +struct BaseArrayArg : public Analyzable { + public: + BaseArrayArg(std::unique_ptr data); + BaseArrayArg(std::unique_ptr data, std::unique_ptr null_mask); + + public: + ~BaseArrayArg() {} + + public: + void pack(BufferBuilder& buffer, const StoreAnalyzer& analyzer) const override; + void analyze(StoreAnalyzer& analyzer) override; + std::optional get_key_proj_id() const override; + void record_unbound_stores(std::vector& args) const override; + void perform_invalidations() const override; + + private: + std::unique_ptr data_; + std::unique_ptr null_mask_{nullptr}; +}; + +struct ListArrayArg : public Analyzable { + public: + ListArrayArg(std::shared_ptr type, + std::unique_ptr descriptor, + std::unique_ptr vardata); + + public: + ~ListArrayArg() {} + + public: + void pack(BufferBuilder& buffer, const StoreAnalyzer& analyzer) const override; + void analyze(StoreAnalyzer& analyzer) override; + std::optional get_key_proj_id() const override; + void record_unbound_stores(std::vector& args) const override; + void perform_invalidations() const override; + + private: + std::shared_ptr type_; + std::unique_ptr descriptor_; + std::unique_ptr vardata_; +}; + +struct StructArrayArg : public Analyzable { + public: + StructArrayArg(std::shared_ptr type, + std::unique_ptr null_mask, + std::vector>&& fields); + + public: + ~StructArrayArg() {} + + public: + void pack(BufferBuilder& buffer, const StoreAnalyzer& analyzer) const override; + void analyze(StoreAnalyzer& analyzer) override; + std::optional get_key_proj_id() const override; + void record_unbound_stores(std::vector& args) const override; + void perform_invalidations() const override; + + private: + std::shared_ptr type_; + std::unique_ptr null_mask_; + std::vector> fields_; +}; + } // namespace legate::detail diff --git a/src/core/operation/detail/operation.cc b/src/core/operation/detail/operation.cc index 5cb0acb09b..982f59eb04 100644 --- a/src/core/operation/detail/operation.cc +++ b/src/core/operation/detail/operation.cc @@ -56,7 +56,6 @@ void Operation::record_partition(const Variable* variable, std::shared_ptr Operation::create_projection_info(const Strategy& strategy, diff --git a/src/core/operation/detail/operation.h b/src/core/operation/detail/operation.h index 60b8f8c905..41f6967dcf 100644 --- a/src/core/operation/detail/operation.h +++ b/src/core/operation/detail/operation.h @@ -59,15 +59,6 @@ class Operation { protected: uint64_t unique_id_; - - protected: - std::set> all_stores_{}; - std::vector inputs_{}; - std::vector outputs_{}; - std::vector reductions_{}; - std::vector reduction_ops_{}; - - protected: uint32_t next_part_id_{0}; std::vector> partition_symbols_{}; std::map> store_mappings_{}; diff --git a/src/core/operation/detail/reduce.cc b/src/core/operation/detail/reduce.cc index a494ce16c7..097acb4cd7 100644 --- a/src/core/operation/detail/reduce.cc +++ b/src/core/operation/detail/reduce.cc @@ -63,6 +63,8 @@ void Reduce::launch(Strategy* p_strategy) for (size_t i = 0; i < radix_; i++) proj_fns.push_back(proj::RadixProjectionFunctor(radix_, i)); } + auto to_array_arg = [](auto&& arg) { return std::make_unique(std::move(arg)); }; + std::shared_ptr new_output; bool done = false; while (!done) { @@ -72,13 +74,15 @@ void Reduce::launch(Strategy* p_strategy) // if there are more than 1 sub-task, we add several slices of the input // for each sub-task for (auto& proj_fn : proj_fns) { - launcher.add_input(input_.get(), - input_partition->create_projection_info(&launch_domain, proj_fn)); + auto proj_info = input_partition->create_projection_info(&launch_domain, proj_fn); + launcher.add_input(to_array_arg( + std::make_unique(input_.get(), READ_ONLY, std::move(proj_info)))); } } else { // otherwise we just add an entire input region to the task - auto proj = input_partition->create_projection_info(&launch_domain); - launcher.add_input(input_.get(), std::move(proj)); + auto proj_info = input_partition->create_projection_info(&launch_domain); + launcher.add_input(to_array_arg( + std::make_unique(input_.get(), READ_ONLY, std::move(proj_info)))); } // calculating #of sub-tasks in the reduction task @@ -89,15 +93,15 @@ void Reduce::launch(Strategy* p_strategy) auto runtime = detail::Runtime::get_runtime(); auto field_space = runtime->create_field_space(); - auto field_size = input_->type()->size(); - auto field_id = runtime->allocate_field(field_space, field_size); // if this is not the last iteration of the while loop, we generate // a new output region if (n_tasks != 1) { new_output = runtime->create_store(input_->type(), 1); - launcher.add_unbound_output(new_output.get(), field_space, field_id); + launcher.add_output( + to_array_arg(std::make_unique(new_output.get(), field_space))); } else { - launcher.add_unbound_output(output_.get(), field_space, field_id); + launcher.add_output( + to_array_arg(std::make_unique(output_.get(), field_space))); } launch_domain = Domain(DomainPoint(0), DomainPoint(n_tasks - 1)); diff --git a/src/core/operation/detail/req_analyzer.cc b/src/core/operation/detail/req_analyzer.cc index f08245a7b5..a96ade3908 100644 --- a/src/core/operation/detail/req_analyzer.cc +++ b/src/core/operation/detail/req_analyzer.cc @@ -23,11 +23,11 @@ void ProjectionSet::insert(Legion::PrivilegeMode new_privilege, const Projection { if (proj_infos.empty()) privilege = new_privilege; // conflicting privileges are promoted to a single read-write privilege - else if (privilege != new_privilege) + else if (!(privilege == new_privilege || privilege == NO_ACCESS || new_privilege == NO_ACCESS)) privilege = LEGION_READ_WRITE; proj_infos.emplace(proj_info); - if (privilege != LEGION_READ_ONLY && proj_infos.size() > 1) { + if (privilege != LEGION_READ_ONLY && privilege != NO_ACCESS && proj_infos.size() > 1) { log_legate.error("Interfering requirements are found"); LEGATE_ABORT; } @@ -79,12 +79,12 @@ constexpr bool is_single = false; } // namespace template -void FieldSet::populate_launcher(Launcher* task, const Legion::LogicalRegion& region) const +void FieldSet::populate_launcher(Launcher& task, const Legion::LogicalRegion& region) const { for (auto& [key, fields] : coalesced_) { auto& [privilege, proj_info] = key; - task->region_requirements.push_back(Legion::RegionRequirement()); - auto& requirement = task->region_requirements.back(); + task.region_requirements.push_back(Legion::RegionRequirement()); + auto& requirement = task.region_requirements.back(); proj_info.template populate_requirement>( requirement, region, fields, privilege); } @@ -94,8 +94,6 @@ void FieldSet::populate_launcher(Launcher* task, const Legion::LogicalRegion& re // RequirementAnalyzer ////////////////////// -RequirementAnalyzer::~RequirementAnalyzer() {} - void RequirementAnalyzer::insert(const Legion::LogicalRegion& region, Legion::FieldID field_id, Legion::PrivilegeMode privilege, @@ -127,18 +125,18 @@ void RequirementAnalyzer::analyze_requirements() } } -void RequirementAnalyzer::populate_launcher(Legion::IndexTaskLauncher* task) const +void RequirementAnalyzer::populate_launcher(Legion::IndexTaskLauncher& task) const { _populate_launcher(task); } -void RequirementAnalyzer::populate_launcher(Legion::TaskLauncher* task) const +void RequirementAnalyzer::populate_launcher(Legion::TaskLauncher& task) const { _populate_launcher(task); } template -void RequirementAnalyzer::_populate_launcher(Launcher* task) const +void RequirementAnalyzer::_populate_launcher(Launcher& task) const { for (auto& [region, entry] : field_sets_) entry.first.populate_launcher(task, region); } @@ -147,8 +145,6 @@ void RequirementAnalyzer::_populate_launcher(Launcher* task) const // OutputRequirementAnalyzer //////////////////////////// -OutputRequirementAnalyzer::~OutputRequirementAnalyzer() {} - void OutputRequirementAnalyzer::insert(int32_t dim, const Legion::FieldSpace& field_space, Legion::FieldID field_id) @@ -187,4 +183,36 @@ void OutputRequirementAnalyzer::populate_output_requirements( } } +///////////////// +// FutureAnalyzer +///////////////// + +void FutureAnalyzer::insert(const Legion::Future& future) { futures_.push_back(future); } + +int32_t FutureAnalyzer::get_future_index(const Legion::Future& future) const +{ + return future_indices_.at(future); +} + +void FutureAnalyzer::analyze_futures() +{ + int32_t index = 0; + for (auto& future : futures_) { future_indices_[future] = index++; } +} + +template +void FutureAnalyzer::_populate_launcher(Launcher& task) const +{ + for (auto& future : futures_) { task.add_future(future); } +} +void FutureAnalyzer::populate_launcher(Legion::IndexTaskLauncher& task) const +{ + _populate_launcher(task); +} + +void FutureAnalyzer::populate_launcher(Legion::TaskLauncher& task) const +{ + _populate_launcher(task); +} + } // namespace legate::detail diff --git a/src/core/operation/detail/req_analyzer.h b/src/core/operation/detail/req_analyzer.h index 110e7df158..53a33a2903 100644 --- a/src/core/operation/detail/req_analyzer.h +++ b/src/core/operation/detail/req_analyzer.h @@ -12,6 +12,7 @@ #pragma once +#include "core/data/detail/logical_region_field.h" #include "core/operation/detail/projection.h" #include "core/runtime/detail/runtime.h" @@ -41,7 +42,7 @@ class FieldSet { public: void coalesce(); template - void populate_launcher(Launcher* task, const Legion::LogicalRegion& region) const; + void populate_launcher(Launcher& task, const Legion::LogicalRegion& region) const; private: std::map> coalesced_; @@ -52,9 +53,6 @@ class FieldSet { }; class RequirementAnalyzer { - public: - ~RequirementAnalyzer(); - public: void insert(const Legion::LogicalRegion& region, Legion::FieldID field_id, @@ -67,21 +65,18 @@ class RequirementAnalyzer { public: void analyze_requirements(); - void populate_launcher(Legion::IndexTaskLauncher* task) const; - void populate_launcher(Legion::TaskLauncher* task) const; + void populate_launcher(Legion::IndexTaskLauncher& task) const; + void populate_launcher(Legion::TaskLauncher& task) const; private: template - void _populate_launcher(Launcher* task) const; + void _populate_launcher(Launcher& task) const; private: - std::map> field_sets_; + std::map> field_sets_{}; }; class OutputRequirementAnalyzer { - public: - ~OutputRequirementAnalyzer(); - public: void insert(int32_t dim, const Legion::FieldSpace& field_space, Legion::FieldID field_id); uint32_t get_requirement_index(const Legion::FieldSpace& field_space, @@ -97,8 +92,81 @@ class OutputRequirementAnalyzer { int32_t dim{-1}; uint32_t req_idx{0}; }; - std::map> field_groups_; - std::map req_infos_; + std::map> field_groups_{}; + std::map req_infos_{}; +}; + +class FutureAnalyzer { + public: + void insert(const Legion::Future& future); + int32_t get_future_index(const Legion::Future& future) const; + + public: + void analyze_futures(); + template + void _populate_launcher(Launcher& task) const; + void populate_launcher(Legion::IndexTaskLauncher& task) const; + void populate_launcher(Legion::TaskLauncher& task) const; + + private: + std::map future_indices_{}; + std::vector futures_{}; +}; + +struct StoreAnalyzer { + public: + void insert(const LogicalRegionField* region_field, + Legion::PrivilegeMode privilege, + const ProjectionInfo& proj_info) + { + req_analyzer_.insert(region_field->region(), region_field->field_id(), privilege, proj_info); + } + void insert(int32_t dim, const Legion::FieldSpace& field_space, Legion::FieldID field_id) + { + out_analyzer_.insert(dim, field_space, field_id); + } + void insert(const Legion::Future& future) { fut_analyzer_.insert(future); } + + public: + void analyze() + { + req_analyzer_.analyze_requirements(); + out_analyzer_.analyze_requirements(); + fut_analyzer_.analyze_futures(); + } + + public: + uint32_t get_index(const Legion::LogicalRegion& region, + Legion::PrivilegeMode privilege, + const ProjectionInfo& proj_info) const + { + return req_analyzer_.get_requirement_index(region, privilege, proj_info); + } + uint32_t get_index(const Legion::FieldSpace& field_space, Legion::FieldID field_id) const + { + return out_analyzer_.get_requirement_index(field_space, field_id); + } + int32_t get_index(const Legion::Future& future) const + { + return fut_analyzer_.get_future_index(future); + } + + public: + template + void populate(Launcher& launcher, std::vector& out_reqs) + { + req_analyzer_.populate_launcher(launcher); + out_analyzer_.populate_output_requirements(out_reqs); + fut_analyzer_.populate_launcher(launcher); + } + + public: + bool can_be_local_function_task() const { return req_analyzer_.empty() && out_analyzer_.empty(); } + + private: + RequirementAnalyzer req_analyzer_{}; + OutputRequirementAnalyzer out_analyzer_{}; + FutureAnalyzer fut_analyzer_{}; }; } // namespace legate::detail diff --git a/src/core/operation/detail/scatter.cc b/src/core/operation/detail/scatter.cc index 8144352369..251c1a98c1 100644 --- a/src/core/operation/detail/scatter.cc +++ b/src/core/operation/detail/scatter.cc @@ -91,7 +91,7 @@ void Scatter::launch(Strategy* p_strategy) void Scatter::add_to_solver(ConstraintSolver& solver) { - solver.add_constraint(constraint_.get()); + solver.add_constraint(std::move(constraint_)); solver.add_partition_symbol(target_.variable); solver.add_partition_symbol(target_indirect_.variable); solver.add_partition_symbol(source_.variable); diff --git a/src/core/operation/detail/scatter_gather.cc b/src/core/operation/detail/scatter_gather.cc index 9fb47efefe..a1e6223f68 100644 --- a/src/core/operation/detail/scatter_gather.cc +++ b/src/core/operation/detail/scatter_gather.cc @@ -112,7 +112,7 @@ void ScatterGather::launch(Strategy* p_strategy) void ScatterGather::add_to_solver(ConstraintSolver& solver) { - solver.add_constraint(constraint_.get()); + solver.add_constraint(std::move(constraint_)); solver.add_partition_symbol(target_.variable); solver.add_partition_symbol(target_indirect_.variable); solver.add_partition_symbol(source_.variable); diff --git a/src/core/operation/detail/task.cc b/src/core/operation/detail/task.cc index 71dbf204f3..52e05a1253 100644 --- a/src/core/operation/detail/task.cc +++ b/src/core/operation/detail/task.cc @@ -15,6 +15,7 @@ #include #include "core/data/scalar.h" +#include "core/operation/detail/launcher_arg.h" #include "core/operation/detail/projection.h" #include "core/operation/detail/task_launcher.h" #include "core/partitioning/detail/constraint_solver.h" @@ -59,49 +60,43 @@ void Task::add_communicator(const std::string& name) communicator_factories_.push_back(comm_mgr->find_factory(name)); } -void Task::launch(Strategy* p_strategy) +void Task::record_scalar_output(std::shared_ptr store) +{ + scalar_outputs_.push_back(std::move(store)); +} + +void Task::record_unbound_output(std::shared_ptr store) +{ + unbound_outputs_.push_back(std::move(store)); +} + +void Task::record_scalar_reduction(std::shared_ptr store, + Legion::ReductionOpID legion_redop_id) +{ + scalar_reductions_.push_back(std::make_pair(std::move(store), legion_redop_id)); +} + +void Task::launch_task(Strategy* p_strategy) { auto& strategy = *p_strategy; detail::TaskLauncher launcher(library_, machine_, provenance_, task_id_); const auto* launch_domain = strategy.launch_domain(this); - auto create_projection_info = [&strategy, &launch_domain](auto& store, auto& var) { - auto store_partition = store->create_partition(strategy[var]); - auto proj_info = store_partition->create_projection_info(launch_domain); - proj_info->tag = strategy.is_key_partition(var) ? LEGATE_CORE_KEY_STORE_TAG : 0; - return std::move(proj_info); - }; - - // Add input stores - for (auto& [store, var] : inputs_) launcher.add_input(store, create_projection_info(store, var)); - - // Add normal output stores - for (auto& [store, var] : outputs_) { - if (store->unbound()) continue; - launcher.add_output(store, create_projection_info(store, var)); - store->set_key_partition(machine(), strategy[var].get()); + for (auto& [arr, mapping] : inputs_) { + launcher.add_input(arr->to_launcher_arg(mapping, strategy, launch_domain, READ_ONLY, -1)); + for (auto& [store, symb] : mapping) { + // Key partitions for unbound stores will be populated by the launcher + if (store->unbound()) continue; + store->set_key_partition(machine_, strategy[symb].get()); + } } - - // Add reduction stores - uint32_t idx = 0; - for (auto& [store, var] : reductions_) { - auto store_partition = store->create_partition(strategy[var]); - auto proj = store_partition->create_projection_info(launch_domain); - bool read_write = store_partition->is_disjoint_for(launch_domain); - auto redop = reduction_ops_[idx++]; - proj->set_reduction_op(redop); - launcher.add_reduction(store, std::move(proj), read_write); + for (auto& [arr, mapping] : outputs_) { + launcher.add_output(arr->to_launcher_arg(mapping, strategy, launch_domain, WRITE_ONLY, -1)); } - - // Add unbound output stores - auto* runtime = detail::Runtime::get_runtime(); - for (auto& [store, var] : outputs_) { - if (!store->unbound()) continue; - auto field_space = strategy.find_field_space(var); - // TODO: We should reuse field ids here - auto field_size = store->type()->size(); - auto field_id = runtime->allocate_field(field_space, field_size); - launcher.add_unbound_output(store, field_space, field_id); + uint32_t idx = 0; + for (auto& [arr, mapping] : reductions_) { + launcher.add_reduction( + arr->to_launcher_arg(mapping, strategy, launch_domain, REDUCE, reduction_ops_[idx++])); } // Add by-value scalars @@ -143,10 +138,9 @@ void Task::demux_scalar_stores(const Legion::Future& result) return; else if (1 == total) { if (1 == num_scalar_outs) { - auto [store, _] = outputs_[scalar_outputs_.front()]; - store->set_future(result); + scalar_outputs_.front()->set_future(result); } else if (1 == num_scalar_reds) { - auto [store, _] = reductions_[scalar_reductions_.front()]; + auto& [store, _] = scalar_reductions_.front(); store->set_future(result); } else if (can_throw_exception_) { auto* runtime = detail::Runtime::get_runtime(); @@ -160,12 +154,10 @@ void Task::demux_scalar_stores(const Legion::Future& result) } else { auto* runtime = detail::Runtime::get_runtime(); uint32_t idx = num_unbound_outs; - for (const auto& out_idx : scalar_outputs_) { - auto [store, _] = outputs_[out_idx]; + for (auto& store : scalar_outputs_) { store->set_future(runtime->extract_scalar(result, idx++)); } - for (const auto& red_idx : scalar_reductions_) { - auto [store, _] = reductions_[red_idx]; + for (auto& [store, _] : scalar_reductions_) { store->set_future(runtime->extract_scalar(result, idx++)); } if (can_throw_exception_) @@ -187,9 +179,8 @@ void Task::demux_scalar_stores(const Legion::FutureMap& result, const Domain& la auto* runtime = detail::Runtime::get_runtime(); if (1 == total) { if (1 == num_scalar_reds) { - auto red_idx = scalar_reductions_.front(); - auto [store, _] = reductions_[red_idx]; - store->set_future(runtime->reduce_future_map(result, reduction_ops_[red_idx])); + auto& [store, redop] = scalar_reductions_.front(); + store->set_future(runtime->reduce_future_map(result, redop)); } else if (can_throw_exception_) { auto* runtime = detail::Runtime::get_runtime(); runtime->record_pending_exception(runtime->reduce_exception_future_map(result)); @@ -201,10 +192,9 @@ void Task::demux_scalar_stores(const Legion::FutureMap& result, const Domain& la #endif } else { uint32_t idx = num_unbound_outs; - for (const auto& red_idx : scalar_reductions_) { - auto [store, _] = reductions_[red_idx]; - auto values = runtime->extract_scalar(result, idx++, launch_domain); - store->set_future(runtime->reduce_future_map(values, reduction_ops_[red_idx])); + for (auto& [store, redop] : scalar_reductions_) { + auto values = runtime->extract_scalar(result, idx++, launch_domain); + store->set_future(runtime->reduce_future_map(values, redop)); } if (can_throw_exception_) { auto exn_fm = runtime->extract_scalar(result, idx, launch_domain); @@ -232,36 +222,65 @@ AutoTask::AutoTask(const Library* library, { } -void AutoTask::add_input(std::shared_ptr store, const Variable* partition_symbol) +const Variable* AutoTask::add_input(std::shared_ptr array) +{ + auto symb = find_or_declare_partition(array); + add_input(std::move(array), symb); + return symb; +} + +const Variable* AutoTask::add_output(std::shared_ptr array) +{ + auto symb = find_or_declare_partition(array); + add_output(std::move(array), symb); + return symb; +} + +const Variable* AutoTask::add_reduction(std::shared_ptr array, int32_t redop) { - add_store(inputs_, std::move(store), partition_symbol); + auto symb = find_or_declare_partition(array); + add_reduction(std::move(array), redop, symb); + return symb; } -void AutoTask::add_output(std::shared_ptr store, const Variable* partition_symbol) +void AutoTask::add_input(std::shared_ptr array, const Variable* partition_symbol) { - if (store->has_scalar_storage()) - scalar_outputs_.push_back(outputs_.size()); - else if (store->unbound()) - unbound_outputs_.push_back(outputs_.size()); - add_store(outputs_, std::move(store), partition_symbol); + auto& arg = inputs_.emplace_back(std::move(array)); + arg.array->generate_constraints(this, arg.mapping, partition_symbol); + for (auto& [store, symb] : arg.mapping) record_partition(symb, store); } -void AutoTask::add_reduction(std::shared_ptr store, +void AutoTask::add_output(std::shared_ptr array, const Variable* partition_symbol) +{ + array->record_scalar_or_unbound_outputs(this); + // TODO: We will later support structs with list/string fields + if (array->kind() == ArrayKind::LIST && array->unbound()) { + arrays_to_fixup_.push_back(array.get()); + } + auto& arg = outputs_.emplace_back(std::move(array)); + arg.array->generate_constraints(this, arg.mapping, partition_symbol); + for (auto& [store, symb] : arg.mapping) record_partition(symb, store); +} + +void AutoTask::add_reduction(std::shared_ptr array, int32_t redop, const Variable* partition_symbol) { - auto legion_redop_id = store->type()->find_reduction_operator(redop); - if (store->has_scalar_storage()) scalar_reductions_.push_back(reductions_.size()); - add_store(reductions_, std::move(store), partition_symbol); + if (array->type()->variable_size()) { + throw std::invalid_argument("List/string arrays cannot be used for reduction"); + } + auto legion_redop_id = array->type()->find_reduction_operator(redop); + array->record_scalar_reductions(this, legion_redop_id); reduction_ops_.push_back(legion_redop_id); + + auto& arg = reductions_.emplace_back(std::move(array)); + arg.array->generate_constraints(this, arg.mapping, partition_symbol); + for (auto& [store, symb] : arg.mapping) record_partition(symb, store); } -void AutoTask::add_store(std::vector& store_args, - std::shared_ptr store, - const Variable* partition_symbol) +const Variable* AutoTask::find_or_declare_partition(std::shared_ptr array) { - store_args.push_back(StoreArg{store.get(), partition_symbol}); - record_partition(partition_symbol, std::move(store)); + return Operation::find_or_declare_partition(array->primary_store()); } void AutoTask::add_constraint(std::unique_ptr constraint) @@ -271,10 +290,16 @@ void AutoTask::add_constraint(std::unique_ptr constraint) void AutoTask::add_to_solver(detail::ConstraintSolver& solver) { - for (auto& constraint : constraints_) solver.add_constraint(constraint.get()); - for (auto& [_, symb] : outputs_) solver.add_partition_symbol(symb, true); - for (auto& [_, symb] : reductions_) solver.add_partition_symbol(symb); - for (auto& [_, symb] : inputs_) solver.add_partition_symbol(symb); + for (auto& constraint : constraints_) solver.add_constraint(std::move(constraint)); + for (auto& [_, mapping] : outputs_) { + for (auto& [_, symb] : mapping) solver.add_partition_symbol(symb, true); + } + for (auto& [_, mapping] : inputs_) { + for (auto& [_, symb] : mapping) solver.add_partition_symbol(symb, false); + } + for (auto& [_, mapping] : reductions_) { + for (auto& [_, symb] : mapping) solver.add_partition_symbol(symb, false); + } } void AutoTask::validate() @@ -282,6 +307,25 @@ void AutoTask::validate() for (auto& constraint : constraints_) constraint->validate(); } +void AutoTask::launch(Strategy* p_strategy) +{ + launch_task(p_strategy); + if (!arrays_to_fixup_.empty()) fixup_ranges(*p_strategy); +} + +void AutoTask::fixup_ranges(Strategy& strategy) +{ + const auto* launch_domain = strategy.launch_domain(this); + if (nullptr == launch_domain) return; + + auto* core_lib = detail::Runtime::get_runtime()->core_library(); + detail::TaskLauncher launcher(core_lib, machine_, provenance_, LEGATE_CORE_FIXUP_RANGES); + for (auto* array : arrays_to_fixup_) { + launcher.add_output(array->to_launcher_arg_for_fixup(launch_domain, NO_ACCESS)); + } + launcher.execute(*launch_domain); +} + //////////////////////////////////////////////////// // legate::ManualTask //////////////////////////////////////////////////// @@ -296,7 +340,7 @@ ManualTask::ManualTask(const Library* library, : Task(library, task_id, unique_id, std::move(machine)), strategy_(std::make_unique()) { - strategy_->set_launch_shape(this, launch_shape); + if (!launch_shape.empty()) { strategy_->set_launch_shape(this, launch_shape); } } void ManualTask::add_input(std::shared_ptr store) @@ -311,10 +355,11 @@ void ManualTask::add_input(std::shared_ptr store_partitio void ManualTask::add_output(std::shared_ptr store) { - if (store->has_scalar_storage()) - scalar_outputs_.push_back(outputs_.size()); - else if (store->unbound()) - unbound_outputs_.push_back(outputs_.size()); + if (store->has_scalar_storage()) { + record_scalar_output(store); + } else if (store->unbound()) { + record_unbound_output(store); + } add_store(outputs_, std::move(store), create_no_partition()); } @@ -324,14 +369,16 @@ void ManualTask::add_output(std::shared_ptr store_partiti // TODO: We need to raise an exception for the user error in this case assert(!store_partition->store()->unbound()); #endif - if (store_partition->store()->has_scalar_storage()) scalar_outputs_.push_back(outputs_.size()); + if (store_partition->store()->has_scalar_storage()) { + record_scalar_output(store_partition->store()); + } add_store(outputs_, store_partition->store(), store_partition->partition()); } void ManualTask::add_reduction(std::shared_ptr store, int32_t redop) { auto legion_redop_id = store->type()->find_reduction_operator(redop); - if (store->has_scalar_storage()) scalar_reductions_.push_back(reductions_.size()); + if (store->has_scalar_storage()) { record_scalar_reduction(store, legion_redop_id); } add_store(reductions_, std::move(store), create_no_partition()); reduction_ops_.push_back(legion_redop_id); } @@ -340,29 +387,33 @@ void ManualTask::add_reduction(std::shared_ptr store_part int32_t redop) { auto legion_redop_id = store_partition->store()->type()->find_reduction_operator(redop); - if (store_partition->store()->has_scalar_storage()) - scalar_reductions_.push_back(reductions_.size()); + if (store_partition->store()->has_scalar_storage()) { + record_scalar_reduction(store_partition->store(), legion_redop_id); + } add_store(reductions_, store_partition->store(), store_partition->partition()); reduction_ops_.push_back(legion_redop_id); } -void ManualTask::add_store(std::vector& store_args, +void ManualTask::add_store(std::vector& store_args, std::shared_ptr store, std::shared_ptr partition) { auto partition_symbol = declare_partition(); - store_args.push_back(StoreArg{store.get(), partition_symbol}); + auto& arg = store_args.emplace_back(std::make_shared(store)); + arg.mapping.insert({store, partition_symbol}); if (store->unbound()) { auto field_space = detail::Runtime::get_runtime()->create_field_space(); strategy_->insert(partition_symbol, std::move(partition), field_space); - } else + } else { strategy_->insert(partition_symbol, std::move(partition)); - all_stores_.insert(std::move(store)); + } } void ManualTask::validate() {} -void ManualTask::launch(Strategy*) { Task::launch(strategy_.get()); } +void ManualTask::launch(Strategy*) { launch(); } + +void ManualTask::launch() { launch_task(strategy_.get()); } void ManualTask::add_to_solver(ConstraintSolver& solver) {} diff --git a/src/core/operation/detail/task.h b/src/core/operation/detail/task.h index d118d249c9..2c59653cb9 100644 --- a/src/core/operation/detail/task.h +++ b/src/core/operation/detail/task.h @@ -14,6 +14,7 @@ #include +#include "core/data/detail/logical_array.h" #include "core/data/detail/scalar.h" #include "core/operation/detail/operation.h" #include "core/partitioning/constraint.h" @@ -33,6 +34,13 @@ class Strategy; class Runtime; class Task : public Operation { + protected: + struct ArrayArg { + ArrayArg(std::shared_ptr _array) : array(std::move(_array)) {} + std::shared_ptr array; + std::map, const Variable*> mapping{}; + }; + protected: Task(const Library* library, int64_t task_id, @@ -51,7 +59,13 @@ class Task : public Operation { void add_communicator(const std::string& name); public: - virtual void launch(Strategy* strategy) override; + void record_scalar_output(std::shared_ptr store); + void record_unbound_output(std::shared_ptr store); + void record_scalar_reduction(std::shared_ptr store, + Legion::ReductionOpID legion_redop_id); + + protected: + void launch_task(Strategy* strategy); private: void demux_scalar_stores(const Legion::Future& result); @@ -67,9 +81,13 @@ class Task : public Operation { bool has_side_effect_{false}; bool can_throw_exception_{false}; std::vector scalars_{}; - std::vector unbound_outputs_{}; - std::vector scalar_outputs_{}; - std::vector scalar_reductions_{}; + std::vector inputs_{}; + std::vector outputs_{}; + std::vector reductions_{}; + std::vector reduction_ops_{}; + std::vector> unbound_outputs_{}; + std::vector> scalar_outputs_{}; + std::vector, Legion::ReductionOpID>> scalar_reductions_{}; std::vector communicator_factories_{}; }; @@ -85,16 +103,19 @@ class AutoTask : public Task { ~AutoTask() {} public: - void add_input(std::shared_ptr store, const Variable* partition_symbol); - void add_output(std::shared_ptr store, const Variable* partition_symbol); - void add_reduction(std::shared_ptr store, - Legion::ReductionOpID redop, + const Variable* add_input(std::shared_ptr array); + const Variable* add_output(std::shared_ptr array); + const Variable* add_reduction(std::shared_ptr array, int32_t redop); + + public: + void add_input(std::shared_ptr array, const Variable* partition_symbol); + void add_output(std::shared_ptr array, const Variable* partition_symbol); + void add_reduction(std::shared_ptr array, + int32_t redop, const Variable* partition_symbol); - private: - void add_store(std::vector& store_args, - std::shared_ptr store, - const Variable* partition_symbol); + public: + const Variable* find_or_declare_partition(std::shared_ptr array); public: void add_constraint(std::unique_ptr constraint); @@ -102,9 +123,14 @@ class AutoTask : public Task { public: void validate() override; + void launch(Strategy* strategy) override; + + private: + void fixup_ranges(Strategy& strategy); private: std::vector> constraints_{}; + std::vector arrays_to_fixup_{}; }; class ManualTask : public Task { @@ -129,13 +155,14 @@ class ManualTask : public Task { Legion::ReductionOpID redop); private: - void add_store(std::vector& store_args, + void add_store(std::vector& store_args, std::shared_ptr store, std::shared_ptr partition); public: void validate() override; void launch(Strategy* strategy) override; + void launch(); public: void add_to_solver(ConstraintSolver& solver) override; diff --git a/src/core/operation/detail/task_launcher.cc b/src/core/operation/detail/task_launcher.cc index af3de34ff1..8c365ea78f 100644 --- a/src/core/operation/detail/task_launcher.cc +++ b/src/core/operation/detail/task_launcher.cc @@ -11,6 +11,7 @@ */ #include "core/operation/detail/task_launcher.h" +#include #include "core/data/detail/logical_region_field.h" #include "core/data/detail/logical_store.h" #include "core/data/scalar.h" @@ -31,7 +32,6 @@ TaskLauncher::TaskLauncher(const Library* library, int64_t tag /*= 0*/) : library_(library), task_id_(task_id), tag_(tag), machine_(machine), provenance_("") { - initialize(); } TaskLauncher::TaskLauncher(const Library* library, @@ -41,75 +41,33 @@ TaskLauncher::TaskLauncher(const Library* library, int64_t tag /*= 0*/) : library_(library), task_id_(task_id), tag_(tag), machine_(machine), provenance_(provenance) { - initialize(); } -TaskLauncher::~TaskLauncher() -{ - for (auto& arg : inputs_) delete arg; - for (auto& arg : outputs_) delete arg; - for (auto& arg : reductions_) delete arg; - for (auto& arg : scalars_) delete arg; - - delete req_analyzer_; - delete buffer_; - delete mapper_arg_; -} - -void TaskLauncher::initialize() -{ - req_analyzer_ = new RequirementAnalyzer(); - out_analyzer_ = new OutputRequirementAnalyzer(); - buffer_ = new BufferBuilder(); - mapper_arg_ = new BufferBuilder(); - machine_.pack(*mapper_arg_); -} +TaskLauncher::~TaskLauncher() {} int64_t TaskLauncher::legion_task_id() const { return library_->get_task_id(task_id_); } int64_t TaskLauncher::legion_mapper_id() const { return library_->get_mapper_id(); } -void TaskLauncher::add_scalar(Scalar&& scalar) -{ - scalars_.push_back(new UntypedScalarArg(std::move(scalar))); -} - -void TaskLauncher::add_input(LogicalStore* store, std::unique_ptr proj_info) -{ - add_store(inputs_, store, std::move(proj_info), READ_ONLY); -} +void TaskLauncher::add_input(std::unique_ptr arg) { inputs_.push_back(std::move(arg)); } -void TaskLauncher::add_output(LogicalStore* store, std::unique_ptr proj_info) +void TaskLauncher::add_output(std::unique_ptr arg) { - add_store(outputs_, store, std::move(proj_info), WRITE_ONLY); + arg->record_unbound_stores(unbound_stores_); + outputs_.push_back(std::move(arg)); } -void TaskLauncher::add_reduction(LogicalStore* store, - std::unique_ptr proj_info, - bool read_write) +void TaskLauncher::add_reduction(std::unique_ptr arg) { - if (read_write) - add_store(reductions_, store, std::move(proj_info), READ_WRITE); - else - add_store(reductions_, store, std::move(proj_info), REDUCE); + reductions_.push_back(std::move(arg)); } -void TaskLauncher::add_unbound_output(LogicalStore* store, - Legion::FieldSpace field_space, - Legion::FieldID field_id) +void TaskLauncher::add_scalar(Scalar&& scalar) { - out_analyzer_->insert(store->dim(), field_space, field_id); - auto arg = new OutputRegionArg(out_analyzer_, store, field_space, field_id); - outputs_.push_back(arg); - unbound_stores_.push_back(arg); + scalars_.emplace_back(new ScalarArg(std::move(scalar))); } -void TaskLauncher::add_future(const Legion::Future& future) -{ - // FIXME: Futures that are directly added by this function are incompatible with those - // from scalar stores. We need to separate the two sets. - futures_.push_back(future); -} +void TaskLauncher::add_future(const Legion::Future& future) { futures_.push_back(future); } void TaskLauncher::add_future_map(const Legion::FutureMap& future_map) { @@ -121,151 +79,152 @@ void TaskLauncher::add_communicator(const Legion::FutureMap& communicator) communicators_.push_back(communicator); } -Legion::FutureMap TaskLauncher::execute(const Legion::Domain& launch_domain) -{ - auto legion_launcher = build_index_task(launch_domain); +namespace { - if (output_requirements_.empty()) return Runtime::get_runtime()->dispatch(legion_launcher.get()); - - auto result = Runtime::get_runtime()->dispatch(legion_launcher.get(), &output_requirements_); - post_process_unbound_stores(result, launch_domain); - return result; -} - -Legion::Future TaskLauncher::execute_single() +void analyze(StoreAnalyzer& analyzer, const std::vector>& args) { - auto legion_launcher = build_single_task(); - - if (output_requirements_.empty()) return Runtime::get_runtime()->dispatch(legion_launcher.get()); - auto result = Runtime::get_runtime()->dispatch(legion_launcher.get(), &output_requirements_); - post_process_unbound_stores(); - return result; + for (auto& arg : args) { arg->analyze(analyzer); } } -void TaskLauncher::add_store(std::vector& args, - LogicalStore* store, - std::unique_ptr proj_info, - Legion::PrivilegeMode privilege) +void pack_args(BufferBuilder& buffer, + StoreAnalyzer& analyzer, + const std::vector>& args) { - if (store->has_scalar_storage()) { - auto has_storage = privilege != WRITE_ONLY; - auto read_only = privilege == READ_ONLY; - if (has_storage) futures_.push_back(store->get_future()); - args.push_back(new FutureStoreArg(store, read_only, has_storage, proj_info->redop)); - } else { - auto region_field = store->get_region_field(); - auto region = region_field->region(); - auto field_id = region_field->field_id(); - - req_analyzer_->insert(region, field_id, privilege, *proj_info); - // Keep the projection functor id of the key store - if (LEGATE_CORE_KEY_STORE_TAG == proj_info->tag) key_proj_id_ = proj_info->proj_id; - args.push_back( - new RegionFieldArg(req_analyzer_, store, field_id, privilege, std::move(proj_info))); - } + buffer.pack(args.size()); + for (auto& arg : args) arg->pack(buffer, analyzer); } -void TaskLauncher::pack_args(const std::vector& args) +void pack_args(BufferBuilder& buffer, const std::vector>& args) { - buffer_->pack(args.size()); - for (auto& arg : args) arg->pack(*buffer_); + buffer.pack(args.size()); + for (auto& arg : args) arg->pack(buffer); } -void TaskLauncher::pack_sharding_functor_id() -{ - mapper_arg_->pack(Runtime::get_runtime()->get_sharding(machine_, key_proj_id_)); -} +} // namespace -std::unique_ptr TaskLauncher::build_single_task() +Legion::FutureMap TaskLauncher::execute(const Legion::Domain& launch_domain) { - // Coalesce region requirements before packing task arguments - // as the latter requires requirement indices to be finalized - req_analyzer_->analyze_requirements(); - out_analyzer_->analyze_requirements(); - - pack_args(inputs_); - pack_args(outputs_); - pack_args(reductions_); - pack_args(scalars_); - buffer_->pack(can_throw_exception_); - // insert_barrier - buffer_->pack(false); - // # communicators - buffer_->pack(0); - - pack_sharding_functor_id(); - auto* runtime = Runtime::get_runtime(); - - auto single_task = std::make_unique(legion_task_id(), - buffer_->to_legion_buffer(), - Legion::Predicate::TRUE_PRED, - legion_mapper_id(), - tag_, - mapper_arg_->to_legion_buffer(), - provenance_.c_str()); - for (auto& future : futures_) single_task->add_future(future); + StoreAnalyzer analyzer; + BufferBuilder task_arg; + BufferBuilder mapper_arg; + std::vector output_requirements; - req_analyzer_->populate_launcher(single_task.get()); - out_analyzer_->populate_output_requirements(output_requirements_); + analyze(analyzer, inputs_); + analyze(analyzer, outputs_); + analyze(analyzer, reductions_); + for (auto& future : futures_) analyzer.insert(future); - single_task->local_function_task = - !has_side_effect_ && req_analyzer_->empty() && out_analyzer_->empty(); - - return single_task; -} - -std::unique_ptr TaskLauncher::build_index_task( - const Legion::Domain& launch_domain) -{ // Coalesce region requirements before packing task arguments // as the latter requires requirement indices to be finalized - req_analyzer_->analyze_requirements(); - out_analyzer_->analyze_requirements(); - - pack_args(inputs_); - pack_args(outputs_); - pack_args(reductions_); - pack_args(scalars_); - buffer_->pack(can_throw_exception_); - buffer_->pack(insert_barrier_); - buffer_->pack(communicators_.size()); - - pack_sharding_functor_id(); - auto* runtime = Runtime::get_runtime(); + analyzer.analyze(); + + pack_args(task_arg, analyzer, inputs_); + pack_args(task_arg, analyzer, outputs_); + pack_args(task_arg, analyzer, reductions_); + pack_args(task_arg, scalars_); + task_arg.pack(can_throw_exception_); + task_arg.pack(insert_barrier_); + task_arg.pack(communicators_.size()); + + pack_mapper_arg(mapper_arg); + + Legion::IndexTaskLauncher index_task(legion_task_id(), + launch_domain, + task_arg.to_legion_buffer(), + Legion::ArgumentMap(), + Legion::Predicate::TRUE_PRED, + false /*must*/, + legion_mapper_id(), + tag_, + mapper_arg.to_legion_buffer(), + provenance_.c_str()); + + analyzer.populate(index_task, output_requirements); - auto index_task = std::make_unique(legion_task_id(), - launch_domain, - buffer_->to_legion_buffer(), - Legion::ArgumentMap(), - Legion::Predicate::TRUE_PRED, - false /*must*/, - legion_mapper_id(), - tag_, - mapper_arg_->to_legion_buffer(), - provenance_.c_str()); - for (auto& future : futures_) index_task->add_future(future); if (insert_barrier_) { + auto* runtime = Runtime::get_runtime(); size_t num_tasks = launch_domain.get_volume(); auto [arrival_barrier, wait_barrier] = runtime->create_barriers(num_tasks); - index_task->add_future(Legion::Future::from_value(arrival_barrier)); - index_task->add_future(Legion::Future::from_value(wait_barrier)); + index_task.add_future(Legion::Future::from_value(arrival_barrier)); + index_task.add_future(Legion::Future::from_value(wait_barrier)); runtime->destroy_barrier(arrival_barrier); runtime->destroy_barrier(wait_barrier); } - for (auto& communicator : communicators_) index_task->point_futures.push_back(communicator); - for (auto& future_map : future_maps_) index_task->point_futures.push_back(future_map); + for (auto& communicator : communicators_) index_task.point_futures.push_back(communicator); + for (auto& future_map : future_maps_) index_task.point_futures.push_back(future_map); + + index_task.concurrent = concurrent_ || !communicators_.empty(); + + auto result = Runtime::get_runtime()->dispatch(index_task, output_requirements); + post_process_unbound_stores(result, launch_domain, output_requirements); + for (auto& arg : outputs_) arg->perform_invalidations(); + return result; +} - req_analyzer_->populate_launcher(index_task.get()); - out_analyzer_->populate_output_requirements(output_requirements_); +Legion::Future TaskLauncher::execute_single() +{ + StoreAnalyzer analyzer; + BufferBuilder task_arg; + BufferBuilder mapper_arg; + std::vector output_requirements; - index_task->concurrent = concurrent_ || !communicators_.empty(); + analyze(analyzer, inputs_); + analyze(analyzer, outputs_); + analyze(analyzer, reductions_); + for (auto& future : futures_) analyzer.insert(future); + + // Coalesce region requirements before packing task arguments + // as the latter requires requirement indices to be finalized + analyzer.analyze(); - return index_task; + pack_args(task_arg, analyzer, inputs_); + pack_args(task_arg, analyzer, outputs_); + pack_args(task_arg, analyzer, reductions_); + pack_args(task_arg, scalars_); + task_arg.pack(can_throw_exception_); + // insert_barrier + task_arg.pack(false); + // # communicators + task_arg.pack(0); + + pack_mapper_arg(mapper_arg); + + Legion::TaskLauncher single_task(legion_task_id(), + task_arg.to_legion_buffer(), + Legion::Predicate::TRUE_PRED, + legion_mapper_id(), + tag_, + mapper_arg.to_legion_buffer(), + provenance_.c_str()); + analyzer.populate(single_task, output_requirements); + single_task.local_function_task = !has_side_effect_ && analyzer.can_be_local_function_task(); + + auto result = Runtime::get_runtime()->dispatch(single_task, output_requirements); + post_process_unbound_stores(output_requirements); + for (auto& arg : outputs_) arg->perform_invalidations(); + return result; } -void TaskLauncher::bind_region_fields_to_unbound_stores() {} +void TaskLauncher::pack_mapper_arg(BufferBuilder& buffer) +{ + machine_.pack(buffer); + + std::optional key_proj_id = std::nullopt; + auto find_key_proj_id = [&key_proj_id](auto& args) { + for (auto& arg : args) { + key_proj_id = arg->get_key_proj_id(); + if (key_proj_id) break; + } + }; + find_key_proj_id(inputs_); + if (!key_proj_id) find_key_proj_id(outputs_); + if (!key_proj_id) find_key_proj_id(reductions_); + if (!key_proj_id) { key_proj_id = std::optional(0); } + buffer.pack(Runtime::get_runtime()->get_sharding(machine_, *key_proj_id)); +} -void TaskLauncher::post_process_unbound_stores() +void TaskLauncher::post_process_unbound_stores( + const std::vector& output_requirements) { if (unbound_stores_.empty()) return; @@ -277,7 +236,7 @@ void TaskLauncher::post_process_unbound_stores() assert(arg->requirement_index() != -1U); #endif auto* store = arg->store(); - auto& req = output_requirements_[arg->requirement_index()]; + auto& req = output_requirements[arg->requirement_index()]; auto region_field = runtime->import_region_field(req.parent, arg->field_id(), store->type()->size()); store->set_region_field(std::move(region_field)); @@ -285,8 +244,10 @@ void TaskLauncher::post_process_unbound_stores() } } -void TaskLauncher::post_process_unbound_stores(const Legion::FutureMap& result, - const Legion::Domain& launch_domain) +void TaskLauncher::post_process_unbound_stores( + const Legion::FutureMap& result, + const Legion::Domain& launch_domain, + const std::vector& output_requirements) { if (unbound_stores_.empty()) return; @@ -313,12 +274,12 @@ void TaskLauncher::post_process_unbound_stores(const Legion::FutureMap& result, if (unbound_stores_.size() == 1) { auto* arg = unbound_stores_.front(); - const auto& req = output_requirements_[arg->requirement_index()]; + const auto& req = output_requirements[arg->requirement_index()]; post_process_unbound_store(arg, req, result, machine_); } else { uint32_t idx = 0; for (auto& arg : unbound_stores_) { - const auto& req = output_requirements_[arg->requirement_index()]; + const auto& req = output_requirements[arg->requirement_index()]; if (arg->store()->dim() == 1) post_process_unbound_store( arg, req, runtime->extract_scalar(result, idx, launch_domain), machine_); diff --git a/src/core/operation/detail/task_launcher.h b/src/core/operation/detail/task_launcher.h index 3872ade358..6933948e16 100644 --- a/src/core/operation/detail/task_launcher.h +++ b/src/core/operation/detail/task_launcher.h @@ -16,18 +16,17 @@ #include "legion.h" +#include "core/data/detail/scalar.h" #include "core/mapping/detail/machine.h" +#include "core/operation/detail/launcher_arg.h" +#include "core/operation/detail/req_analyzer.h" +#include "core/utilities/memory.h" namespace legate::detail { -class ArgWrapper; class Library; class LogicalStore; class ProjectionInfo; -class OutputRegionArg; -class OutputRequirementAnalyzer; -class RequirementAnalyzer; -class Scalar; class TaskLauncher { public: @@ -42,23 +41,15 @@ class TaskLauncher { int64_t tag = 0); ~TaskLauncher(); - private: - void initialize(); - public: int64_t legion_task_id() const; int64_t legion_mapper_id() const; public: + void add_input(std::unique_ptr arg); + void add_output(std::unique_ptr arg); + void add_reduction(std::unique_ptr arg); void add_scalar(Scalar&& scalar); - void add_input(LogicalStore* store, std::unique_ptr proj_info); - void add_output(LogicalStore* store, std::unique_ptr proj_info); - void add_reduction(LogicalStore* store, - std::unique_ptr proj_info, - bool read_write); - void add_unbound_output(LogicalStore* store, - Legion::FieldSpace field_space, - Legion::FieldID field_id); public: void add_future(const Legion::Future& future); @@ -71,33 +62,25 @@ class TaskLauncher { void set_insert_barrier(bool insert_barrier) { insert_barrier_ = insert_barrier; } void throws_exception(bool can_throw_exception) { can_throw_exception_ = can_throw_exception; } - private: - void add_store(std::vector& args, - LogicalStore* store, - std::unique_ptr proj_info, - Legion::PrivilegeMode privilege); - public: Legion::FutureMap execute(const Legion::Domain& launch_domain); Legion::Future execute_single(); private: - void pack_args(const std::vector& args); - void pack_sharding_functor_id(); - std::unique_ptr build_index_task(const Legion::Domain& launch_domain); - std::unique_ptr build_single_task(); - void bind_region_fields_to_unbound_stores(); - void post_process_unbound_stores(); - void post_process_unbound_stores(const Legion::FutureMap& result, - const Legion::Domain& launch_domain); + void pack_mapper_arg(BufferBuilder& buffer); + void post_process_unbound_stores( + const std::vector& output_requirements); + void post_process_unbound_stores( + const Legion::FutureMap& result, + const Legion::Domain& launch_domain, + const std::vector& output_requirements); private: const Library* library_; int64_t task_id_; int64_t tag_; - mapping::detail::Machine machine_; + const mapping::detail::Machine& machine_; std::string provenance_; - Legion::ProjectionID key_proj_id_{0}; private: bool has_side_effect_{true}; @@ -106,21 +89,14 @@ class TaskLauncher { bool can_throw_exception_{false}; private: - std::vector inputs_; - std::vector outputs_; - std::vector reductions_; - std::vector scalars_; + std::vector> inputs_; + std::vector> outputs_; + std::vector> reductions_; + std::vector> scalars_; std::vector futures_; - std::vector unbound_stores_; + std::vector unbound_stores_; std::vector future_maps_; std::vector communicators_; - - private: - RequirementAnalyzer* req_analyzer_; - OutputRequirementAnalyzer* out_analyzer_; - BufferBuilder* buffer_; - BufferBuilder* mapper_arg_; - std::vector output_requirements_; }; } // namespace legate::detail diff --git a/src/core/operation/task.cc b/src/core/operation/task.cc index 91d4c69541..9cf711a2dd 100644 --- a/src/core/operation/task.cc +++ b/src/core/operation/task.cc @@ -22,24 +22,44 @@ namespace legate { // legate::AutoTask //////////////////////////////////////////////////// -void AutoTask::add_input(LogicalStore store, Variable partition_symbol) +Variable AutoTask::add_input(LogicalArray array) { - impl_->add_input(store.impl(), partition_symbol.impl()); + return Variable(impl_->add_input(array.impl())); } -void AutoTask::add_output(LogicalStore store, Variable partition_symbol) +Variable AutoTask::add_output(LogicalArray array) { - impl_->add_output(store.impl(), partition_symbol.impl()); + return Variable(impl_->add_output(array.impl())); } -void AutoTask::add_reduction(LogicalStore store, ReductionOpKind redop, Variable partition_symbol) +Variable AutoTask::add_reduction(LogicalArray array, ReductionOpKind redop) { - impl_->add_reduction(store.impl(), static_cast(redop), partition_symbol.impl()); + return Variable(impl_->add_reduction(array.impl(), static_cast(redop))); } -void AutoTask::add_reduction(LogicalStore store, int32_t redop, Variable partition_symbol) +Variable AutoTask::add_reduction(LogicalArray array, int32_t redop) { - impl_->add_reduction(store.impl(), redop, partition_symbol.impl()); + return Variable(impl_->add_reduction(array.impl(), redop)); +} + +void AutoTask::add_input(LogicalArray array, Variable partition_symbol) +{ + impl_->add_input(array.impl(), partition_symbol.impl()); +} + +void AutoTask::add_output(LogicalArray array, Variable partition_symbol) +{ + impl_->add_output(array.impl(), partition_symbol.impl()); +} + +void AutoTask::add_reduction(LogicalArray array, ReductionOpKind redop, Variable partition_symbol) +{ + impl_->add_reduction(array.impl(), static_cast(redop), partition_symbol.impl()); +} + +void AutoTask::add_reduction(LogicalArray array, int32_t redop, Variable partition_symbol) +{ + impl_->add_reduction(array.impl(), redop, partition_symbol.impl()); } void AutoTask::add_scalar_arg(const Scalar& scalar) { impl_->add_scalar_arg(*scalar.impl_); } @@ -51,9 +71,9 @@ void AutoTask::add_constraint(Constraint&& constraint) impl_->add_constraint(std::unique_ptr(constraint.release())); } -Variable AutoTask::find_or_declare_partition(LogicalStore store) +Variable AutoTask::find_or_declare_partition(LogicalArray array) { - return Variable(impl_->find_or_declare_partition(store.impl())); + return Variable(impl_->find_or_declare_partition(array.impl())); } Variable AutoTask::declare_partition() { return Variable(impl_->declare_partition()); } diff --git a/src/core/operation/task.h b/src/core/operation/task.h index 9feb64b078..eec2ac4f17 100644 --- a/src/core/operation/task.h +++ b/src/core/operation/task.h @@ -14,6 +14,7 @@ #include +#include "core/data/logical_array.h" #include "core/data/logical_store.h" #include "core/data/scalar.h" #include "core/mapping/machine.h" @@ -38,47 +39,91 @@ namespace legate { class AutoTask { public: /** - * @brief Adds a store to the task as input + * @brief Adds an array to the task as input * - * Partitioning of the store is controlled by constraints on the partition symbol - * associated with the store + * Partitioning of the array is controlled by constraints on the partition symbol + * associated with the array * - * @param store A store to add to the task as input - * @param partition_symbol A partition symbol for the store + * @param array An array to add to the task as input + * @param partition_symbol A partition symbol for the array */ - void add_input(LogicalStore store, Variable partition_symbol); + Variable add_input(LogicalArray array); /** - * @brief Adds a store to the task as output + * @brief Adds an array to the task as output * - * Partitioning of the store is controlled by constraints on the partition symbol - * associated with the store + * Partitioning of the array is controlled by constraints on the partition symbol + * associated with the array * - * @param store A store to add to the task as output - * @param partition_symbol A partition symbol for the store + * @param array An array to add to the task as output + * @param partition_symbol A partition symbol for the array */ - void add_output(LogicalStore store, Variable partition_symbol); + Variable add_output(LogicalArray array); /** - * @brief Adds a store to the task for reductions + * @brief Adds an array to the task for reductions * - * Partitioning of the store is controlled by constraints on the partition symbol - * associated with the store + * Partitioning of the array is controlled by constraints on the partition symbol + * associated with the array * - * @param store A store to add to the task for reductions - * @param redop ID of the reduction operator to use. The store's type must support the operator. - * @param partition_symbol A partition symbol for the store + * @param array An array to add to the task for reductions + * @param redop ID of the reduction operator to use. The array's type must support the operator. + * @param partition_symbol A partition symbol for the array */ - void add_reduction(LogicalStore store, ReductionOpKind redop, Variable partition_symbol); + Variable add_reduction(LogicalArray array, ReductionOpKind redop); /** - * @brief Adds a store to the task for reductions + * @brief Adds an array to the task for reductions * - * Partitioning of the store is controlled by constraints on the partition symbol - * associated with the store + * Partitioning of the array is controlled by constraints on the partition symbol + * associated with the array * - * @param store A store to add to the task for reductions - * @param redop ID of the reduction operator to use. The store's type must support the operator. - * @param partition_symbol A partition symbol for the store + * @param array An array to add to the task for reductions + * @param redop ID of the reduction operator to use. The array's type must support the operator. + * @param partition_symbol A partition symbol for the array + */ + Variable add_reduction(LogicalArray array, int32_t redop); + + public: + /** + * @brief Adds an array to the task as input + * + * Partitioning of the array is controlled by constraints on the partition symbol + * associated with the array + * + * @param array An array to add to the task as input + * @param partition_symbol A partition symbol for the array + */ + void add_input(LogicalArray array, Variable partition_symbol); + /** + * @brief Adds an array to the task as output + * + * Partitioning of the array is controlled by constraints on the partition symbol + * associated with the array + * + * @param array An array to add to the task as output + * @param partition_symbol A partition symbol for the array + */ + void add_output(LogicalArray array, Variable partition_symbol); + /** + * @brief Adds an array to the task for reductions + * + * Partitioning of the array is controlled by constraints on the partition symbol + * associated with the array + * + * @param array An array to add to the task for reductions + * @param redop ID of the reduction operator to use. The array's type must support the operator. + * @param partition_symbol A partition symbol for the array + */ + void add_reduction(LogicalArray array, ReductionOpKind redop, Variable partition_symbol); + /** + * @brief Adds an array to the task for reductions + * + * Partitioning of the array is controlled by constraints on the partition symbol + * associated with the array + * + * @param array An array to add to the task for reductions + * @param redop ID of the reduction operator to use. The array's type must support the operator. + * @param partition_symbol A partition symbol for the array */ - void add_reduction(LogicalStore store, int32_t redop, Variable partition_symbol); + void add_reduction(LogicalArray array, int32_t redop, Variable partition_symbol); /** * @brief Adds a by-value scalar argument to the task * @@ -100,17 +145,17 @@ class AutoTask { public: /** - * @brief Finds or creates a partition symbol for the given store + * @brief Finds or creates a partition symbol for the given array * - * @param Store for which the partition symbol is queried + * @param array Array for which the partition symbol is queried * - * @return The existing symbol if there is one for the store, a fresh symbol otherwise + * @return The existing symbol if there is one for the array, a fresh symbol otherwise */ - Variable find_or_declare_partition(LogicalStore store); + Variable find_or_declare_partition(LogicalArray array); /** * @brief Declares partition symbol * - * @return A new symbol that can be used when passing a store to an operation + * @return A new symbol that can be used when passing an array to an operation */ Variable declare_partition(); /** diff --git a/src/core/partitioning/detail/constraint.cc b/src/core/partitioning/detail/constraint.cc index d2cd75b4b8..63e18ed935 100644 --- a/src/core/partitioning/detail/constraint.cc +++ b/src/core/partitioning/detail/constraint.cc @@ -62,6 +62,10 @@ void Alignment::validate() const if (*lhs_ == *rhs_) return; auto lhs_store = lhs_->operation()->find_store(lhs_); auto rhs_store = rhs_->operation()->find_store(rhs_); + if (lhs_store->unbound() != rhs_store->unbound()) { + throw std::invalid_argument("Alignment requires the stores to be all normal or all unbound"); + } + if (lhs_store->unbound()) return; if (lhs_store->extents() != rhs_store->extents()) throw std::invalid_argument("Alignment requires the stores to have the same shape, but found " + lhs_store->extents().to_string() + " and " + @@ -118,9 +122,13 @@ void ImageConstraint::validate() const auto func = var_function_->operation()->find_store(var_function_); auto range = var_range_->operation()->find_store(var_range_); - if (!(is_point_type(func->type(), range->dim()) || is_rect_type(func->type(), range->dim()))) + if (!(is_point_type(func->type(), range->dim()) || is_rect_type(func->type(), range->dim()))) { throw std::invalid_argument("Store from which the image partition is derived should have " + std::to_string(range->dim()) + "-D points or rects"); + } + if (range->transformed()) { + throw std::runtime_error("Image constraints on transformed stores are not supported yet"); + } } std::string ImageConstraint::to_string() const @@ -132,12 +140,14 @@ std::string ImageConstraint::to_string() const std::unique_ptr ImageConstraint::resolve(const detail::Strategy& strategy) const { - const auto* src = var_function(); - auto src_part = strategy[src]; - if (src_part->has_launch_domain()) + const auto* src = var_function(); + const auto* range = var_range(); + auto src_part = strategy[src]; + if (src_part->has_launch_domain()) { return create_image(src->operation()->find_store(src), src_part); - else + } else { return create_no_partition(); + } } std::unique_ptr align(const Variable* lhs, const Variable* rhs) diff --git a/src/core/partitioning/detail/constraint_solver.cc b/src/core/partitioning/detail/constraint_solver.cc index 7fcd7930e2..1286934f3c 100644 --- a/src/core/partitioning/detail/constraint_solver.cc +++ b/src/core/partitioning/detail/constraint_solver.cc @@ -87,9 +87,9 @@ void ConstraintSolver::add_partition_symbol(const Variable* partition_symbol, bo is_dependent_.insert({*partition_symbol, false}); } -void ConstraintSolver::add_constraint(const Constraint* constraint) +void ConstraintSolver::add_constraint(std::unique_ptr constraint) { - constraints_.push_back(constraint); + constraints_.push_back(std::move(constraint)); } void ConstraintSolver::solve_constraints() diff --git a/src/core/partitioning/detail/constraint_solver.h b/src/core/partitioning/detail/constraint_solver.h index 8338d21170..1978c50814 100644 --- a/src/core/partitioning/detail/constraint_solver.h +++ b/src/core/partitioning/detail/constraint_solver.h @@ -31,7 +31,7 @@ struct ConstraintSolver { public: void add_partition_symbol(const Variable* partition_symbol, bool is_output = false); - void add_constraint(const Constraint* constraint); + void add_constraint(std::unique_ptr constraint); public: void dump(); @@ -51,7 +51,7 @@ struct ConstraintSolver { private: ordered_set partition_symbols_{}; std::map is_output_{}; - std::vector constraints_{}; + std::vector> constraints_{}; private: class EquivClass; diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index 374a2e3d2c..70b3a07ce9 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -13,9 +13,12 @@ #include #include "core/data/detail/logical_store.h" +#include "core/operation/detail/task.h" +#include "core/partitioning/detail/constraint.h" #include "core/partitioning/partition.h" #include "core/runtime/detail/partition_manager.h" #include "core/runtime/detail/runtime.h" +#include "core/runtime/library.h" #include "core/type/detail/type_info.h" namespace legate { @@ -293,28 +296,39 @@ bool Image::satisfies_restrictions(const Restrictions& restrictions) const Legion::LogicalPartition Image::construct(Legion::LogicalRegion region, bool complete) const { + if (!has_launch_domain()) { return Legion::LogicalPartition::NO_PART; } auto* func_rf = func_->get_region_field(); auto func_region = func_rf->region(); auto func_partition = func_partition_->construct(func_region, func_partition_->is_complete_for(func_->get_storage())); - auto runtime = detail::Runtime::get_runtime(); - bool is_range = func_->type()->code == Type::Code::STRUCT; - auto color_space = runtime->find_or_create_index_space(to_domain(color_shape())); + auto runtime = detail::Runtime::get_runtime(); + auto part_mgr = runtime->partition_manager(); - auto index_partition = runtime->create_image_partition(region.get_index_space(), - color_space, - func_region, - func_partition, - func_rf->field_id(), - is_range); + auto target = region.get_index_space(); + auto index_partition = + part_mgr->find_image_partition(target, func_partition, func_rf->field_id()); + + if (Legion::IndexPartition::NO_PART == index_partition) { + bool is_range = func_->type()->code == Type::Code::STRUCT; + auto color_space = runtime->find_or_create_index_space(to_domain(color_shape())); + + auto field_id = func_rf->field_id(); + index_partition = runtime->create_image_partition( + target, color_space, func_region, func_partition, field_id, is_range); + part_mgr->record_image_partition(target, func_partition, field_id, index_partition); + func_rf->add_invalidation_callback([target, func_partition, field_id]() { + auto part_mgr = detail::Runtime::get_runtime()->partition_manager(); + part_mgr->invalidate_image_partition(target, func_partition, field_id); + }); + } return runtime->create_logical_partition(region, index_partition); } -bool Image::has_launch_domain() const { return true; } +bool Image::has_launch_domain() const { return func_partition_->has_launch_domain(); } -Domain Image::launch_domain() const { return to_domain(color_shape()); } +Domain Image::launch_domain() const { return func_partition_->launch_domain(); } std::unique_ptr Image::clone() const { return std::make_unique(*this); } diff --git a/src/core/runtime/detail/library.cc b/src/core/runtime/detail/library.cc index 9a6b467fb3..0a442e69df 100644 --- a/src/core/runtime/detail/library.cc +++ b/src/core/runtime/detail/library.cc @@ -156,7 +156,7 @@ void Library::register_task(int64_t local_task_id, std::unique_ptr tas tasks_.emplace(std::make_pair(local_task_id, std::move(task_info))); } -const TaskInfo* Library::find_task(int64_t local_task_id) const noexcept(false) +const TaskInfo* Library::find_task(int64_t local_task_id) const { auto finder = tasks_.find(local_task_id); if (tasks_.end() == finder) { diff --git a/src/core/runtime/detail/library.h b/src/core/runtime/detail/library.h index 6750576861..8dcc3b4ff7 100644 --- a/src/core/runtime/detail/library.h +++ b/src/core/runtime/detail/library.h @@ -108,7 +108,7 @@ class Library { public: void register_task(int64_t local_task_id, std::unique_ptr task_info); - const TaskInfo* find_task(int64_t local_task_id) const noexcept(false); + const TaskInfo* find_task(int64_t local_task_id) const; private: void perform_callback(Legion::RegistrationWithArgsCallbackFnptr callback, diff --git a/src/core/runtime/detail/partition_manager.cc b/src/core/runtime/detail/partition_manager.cc index 9bc6077d94..41e3055471 100644 --- a/src/core/runtime/detail/partition_manager.cc +++ b/src/core/runtime/detail/partition_manager.cc @@ -231,6 +231,20 @@ Legion::IndexPartition PartitionManager::find_index_partition(const Legion::Inde return _find_index_partition(weighted_cache_, index_space, weighted); } +Legion::IndexPartition PartitionManager::find_image_partition( + const Legion::IndexSpace& index_space, + const Legion::LogicalPartition& func_partition, + Legion::FieldID field_id) const +{ + ImageCacheKey key(index_space, func_partition, field_id); + auto finder = image_cache_.find(key); + if (finder != image_cache_.end()) { + return finder->second; + } else { + return Legion::IndexPartition::NO_PART; + } +} + void PartitionManager::record_index_partition(const Legion::IndexSpace& index_space, const Tiling& tiling, const Legion::IndexPartition& index_partition) @@ -245,4 +259,23 @@ void PartitionManager::record_index_partition(const Legion::IndexSpace& index_sp weighted_cache_[std::make_pair(index_space, weighted)] = index_partition; } +void PartitionManager::record_image_partition(const Legion::IndexSpace& index_space, + const Legion::LogicalPartition& func_partition, + Legion::FieldID field_id, + const Legion::IndexPartition& index_partition) +{ + image_cache_[std::tie(index_space, func_partition, field_id)] = index_partition; +} + +void PartitionManager::invalidate_image_partition(const Legion::IndexSpace& index_space, + const Legion::LogicalPartition& func_partition, + Legion::FieldID field_id) +{ + auto finder = image_cache_.find(std::tie(index_space, func_partition, field_id)); +#ifdef DEBUG_LEGATE + assert(finder != image_cache_.end()); +#endif + image_cache_.erase(finder); +} + } // namespace legate::detail diff --git a/src/core/runtime/detail/partition_manager.h b/src/core/runtime/detail/partition_manager.h index 12b9355103..9182128800 100644 --- a/src/core/runtime/detail/partition_manager.h +++ b/src/core/runtime/detail/partition_manager.h @@ -49,12 +49,26 @@ class PartitionManager { const Tiling& tiling) const; Legion::IndexPartition find_index_partition(const Legion::IndexSpace& index_space, const Weighted& weighted) const; + Legion::IndexPartition find_image_partition(const Legion::IndexSpace& index_space, + const Legion::LogicalPartition& func_partition, + Legion::FieldID field_id) const; + + public: void record_index_partition(const Legion::IndexSpace& index_space, const Tiling& tiling, const Legion::IndexPartition& index_partition); void record_index_partition(const Legion::IndexSpace& index_space, const Weighted& weighted, const Legion::IndexPartition& index_partition); + void record_image_partition(const Legion::IndexSpace& index_space, + const Legion::LogicalPartition& func_partition, + Legion::FieldID field_id, + const Legion::IndexPartition& index_partition); + + public: + void invalidate_image_partition(const Legion::IndexSpace& index_space, + const Legion::LogicalPartition& func_partition, + Legion::FieldID field_id); private: int64_t min_shard_volume_; @@ -65,6 +79,8 @@ class PartitionManager { std::map tiling_cache_; using WeightedCacheKey = std::pair; std::map weighted_cache_; + using ImageCacheKey = std::tuple; + std::map image_cache_; }; } // namespace legate::detail diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index c39ffa826e..68bc714d2b 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -13,6 +13,8 @@ #include "core/runtime/detail/runtime.h" #include "core/comm/comm.h" +#include "core/data/detail/array_tasks.h" +#include "core/data/detail/logical_array.h" #include "core/data/detail/logical_region_field.h" #include "core/data/detail/logical_store.h" #include "core/mapping/detail/core_mapper.h" @@ -161,7 +163,7 @@ void Runtime::initialize(Legion::Context legion_context) comm::register_builtin_communicator_factories(core_library_); } -mapping::detail::Machine Runtime::slice_machine_for_task(Library* library, int64_t task_id) +mapping::detail::Machine Runtime::slice_machine_for_task(const Library* library, int64_t task_id) { auto* task_info = library->find_task(task_id); @@ -183,14 +185,14 @@ mapping::detail::Machine Runtime::slice_machine_for_task(Library* library, int64 } // This function should be moved to the library context -std::unique_ptr Runtime::create_task(Library* library, int64_t task_id) +std::unique_ptr Runtime::create_task(const Library* library, int64_t task_id) { auto machine = slice_machine_for_task(library, task_id); auto task = new AutoTask(library, task_id, next_unique_id_++, std::move(machine)); return std::unique_ptr(task); } -std::unique_ptr Runtime::create_task(Library* library, +std::unique_ptr Runtime::create_task(const Library* library, int64_t task_id, const Shape& launch_shape) { @@ -303,8 +305,114 @@ void Runtime::schedule(std::vector> operations) for (auto& op : operations) op->launch(strategy.get()); } -std::shared_ptr Runtime::create_store(std::shared_ptr type, int32_t dim) +std::shared_ptr Runtime::create_array(std::shared_ptr type, + uint32_t dim, + bool nullable) +{ + if (Type::Code::STRUCT == type->code) { + return create_struct_array(std::move(type), dim, nullable); + } else if (type->variable_size()) { + if (dim != 1) { throw std::invalid_argument("List/string arrays can only be 1D"); } + auto elem_type = + Type::Code::STRING == type->code ? int8() : type->as_list_type().element_type(); + auto descriptor = create_base_array(rect_type(1), dim, nullable); + auto vardata = create_array(std::move(elem_type), 1, false); + return std::make_shared( + std::move(type), std::move(descriptor), std::move(vardata)); + } else { + return create_base_array(std::move(type), dim, nullable); + } +} + +std::shared_ptr Runtime::create_array(const Shape& extents, + std::shared_ptr type, + bool nullable, + bool optimize_scalar) +{ + if (Type::Code::STRUCT == type->code) { + return create_struct_array(extents, std::move(type), nullable, optimize_scalar); + } else if (type->variable_size()) { + if (extents.size() != 1) { throw std::invalid_argument("List/string arrays can only be 1D"); } + auto elem_type = + Type::Code::STRING == type->code ? int8() : type->as_list_type().element_type(); + auto descriptor = create_base_array(extents, rect_type(1), nullable, optimize_scalar); + auto vardata = create_array(std::move(elem_type), 1, false); + return std::make_shared( + std::move(type), std::move(descriptor), std::move(vardata)); + } else { + return create_base_array(extents, std::move(type), nullable, optimize_scalar); + } +} + +std::shared_ptr Runtime::create_array_like(std::shared_ptr array, + std::shared_ptr type) +{ + if (Type::Code::STRUCT == type->code || type->variable_size()) { + throw std::runtime_error( + "create_array_like doesn't support variable size types or struct types"); + } + + if (array->unbound()) { + return create_array(std::move(type), array->dim(), array->nullable()); + } else { + bool optimize_scalar = array->data()->has_scalar_storage(); + return create_array(array->extents(), std::move(type), array->nullable(), optimize_scalar); + } +} + +std::shared_ptr Runtime::create_struct_array(std::shared_ptr type, + uint32_t dim, + bool nullable) +{ + const auto& st_type = type->as_struct_type(); + auto null_mask = nullable ? create_store(bool_(), dim) : nullptr; + + std::vector> fields; + for (auto& field_type : st_type.field_types()) { + fields.push_back(create_array(field_type, dim, false)); + } + return std::make_shared( + std::move(type), std::move(null_mask), std::move(fields)); +} + +std::shared_ptr Runtime::create_struct_array(const Shape& extents, + std::shared_ptr type, + bool nullable, + bool optimize_scalar) { + const auto& st_type = type->as_struct_type(); + auto null_mask = nullable ? create_store(extents, bool_(), optimize_scalar) : nullptr; + + std::vector> fields; + for (auto& field_type : st_type.field_types()) { + fields.push_back(create_array(extents, field_type, false, optimize_scalar)); + } + return std::make_shared( + std::move(type), std::move(null_mask), std::move(fields)); +} + +std::shared_ptr Runtime::create_base_array(std::shared_ptr type, + uint32_t dim, + bool nullable) +{ + auto data = create_store(std::move(type), dim); + auto null_mask = nullable ? create_store(bool_(), dim) : nullptr; + return std::make_shared(std::move(data), std::move(null_mask)); +} + +std::shared_ptr Runtime::create_base_array(const Shape& extents, + std::shared_ptr type, + bool nullable, + bool optimize_scalar) +{ + auto data = create_store(extents, std::move(type), optimize_scalar); + auto null_mask = nullable ? create_store(extents, bool_(), optimize_scalar) : nullptr; + return std::make_shared(std::move(data), std::move(null_mask)); +} + +std::shared_ptr Runtime::create_store(std::shared_ptr type, uint32_t dim) +{ + check_dimensionality(dim); auto storage = std::make_shared(dim, std::move(type)); return std::make_shared(std::move(storage)); } @@ -313,6 +421,7 @@ std::shared_ptr Runtime::create_store(const Shape& extents, std::shared_ptr type, bool optimize_scalar /*=false*/) { + check_dimensionality(extents.size()); auto storage = std::make_shared(extents, std::move(type), optimize_scalar); return std::make_shared(std::move(storage)); } @@ -325,6 +434,15 @@ std::shared_ptr Runtime::create_store(const Scalar& scalar) return std::make_shared(std::move(storage)); } +void Runtime::check_dimensionality(uint32_t dim) +{ + if (dim > LEGATE_MAX_DIM) { + throw std::out_of_range("The maximum number of dimensions is " + + std::to_string(LEGION_MAX_DIM) + ", but a " + std::to_string(dim) + + "-D store is requested"); + } +} + uint32_t Runtime::max_pending_exceptions() const { return max_pending_exceptions_; } void Runtime::set_max_pending_exceptions(uint32_t max_pending_exceptions) @@ -518,6 +636,11 @@ Legion::IndexPartition Runtime::create_image_partition( Legion::FieldID func_field_id, bool is_range) { +#ifdef DEBUG_LEGATE + log_legate.debug() << "Create image partition {index_space: " << index_space + << ", func_partition: " << func_partition + << ", func_field_id: " << func_field_id << ", is_range: " << is_range << "}"; +#endif if (is_range) return legion_runtime_->create_partition_by_image_range(legion_context_, index_space, @@ -657,42 +780,42 @@ Legion::Future Runtime::get_tunable(Legion::MapperID mapper_id, int64_t tunable_ return legion_runtime_->select_tunable_value(legion_context_, launcher); } -Legion::Future Runtime::dispatch(Legion::TaskLauncher* launcher, - std::vector* output_requirements) +Legion::Future Runtime::dispatch(Legion::TaskLauncher& launcher, + std::vector& output_requirements) { assert(nullptr != legion_context_); - return legion_runtime_->execute_task(legion_context_, *launcher, output_requirements); + return legion_runtime_->execute_task(legion_context_, launcher, &output_requirements); } -Legion::FutureMap Runtime::dispatch(Legion::IndexTaskLauncher* launcher, - std::vector* output_requirements) +Legion::FutureMap Runtime::dispatch(Legion::IndexTaskLauncher& launcher, + std::vector& output_requirements) { assert(nullptr != legion_context_); - return legion_runtime_->execute_index_space(legion_context_, *launcher, output_requirements); + return legion_runtime_->execute_index_space(legion_context_, launcher, &output_requirements); } -void Runtime::dispatch(Legion::CopyLauncher* launcher) +void Runtime::dispatch(Legion::CopyLauncher& launcher) { assert(nullptr != legion_context_); - return legion_runtime_->issue_copy_operation(legion_context_, *launcher); + return legion_runtime_->issue_copy_operation(legion_context_, launcher); } -void Runtime::dispatch(Legion::IndexCopyLauncher* launcher) +void Runtime::dispatch(Legion::IndexCopyLauncher& launcher) { assert(nullptr != legion_context_); - return legion_runtime_->issue_copy_operation(legion_context_, *launcher); + return legion_runtime_->issue_copy_operation(legion_context_, launcher); } -void Runtime::dispatch(Legion::FillLauncher* launcher) +void Runtime::dispatch(Legion::FillLauncher& launcher) { assert(nullptr != legion_context_); - return legion_runtime_->fill_fields(legion_context_, *launcher); + return legion_runtime_->fill_fields(legion_context_, launcher); } -void Runtime::dispatch(Legion::IndexFillLauncher* launcher) +void Runtime::dispatch(Legion::IndexFillLauncher& launcher) { assert(nullptr != legion_context_); - return legion_runtime_->fill_fields(legion_context_, *launcher); + return legion_runtime_->fill_fields(legion_context_, launcher); } Legion::Future Runtime::extract_scalar(const Legion::Future& result, uint32_t idx) const @@ -1012,6 +1135,7 @@ void register_legate_core_tasks(Legion::Machine machine, #endif core_lib->register_task(LEGATE_CORE_EXTRACT_SCALAR_TASK_ID, std::move(task_info)); + register_array_tasks(core_lib); comm::register_tasks(runtime, core_lib); } diff --git a/src/core/runtime/detail/runtime.h b/src/core/runtime/detail/runtime.h index 6bbbd62d10..e53628147e 100644 --- a/src/core/runtime/detail/runtime.h +++ b/src/core/runtime/detail/runtime.h @@ -36,12 +36,15 @@ namespace legate::detail { class AutoTask; +class BaseLogicalArray; class Copy; class Library; +class LogicalArray; class LogicalRegionField; class LogicalStore; class ManualTask; class Operation; +class StructLogicalArray; template class ConsensusMatchResult { @@ -94,9 +97,9 @@ class Runtime { void initialize(Legion::Context legion_context); public: - mapping::detail::Machine slice_machine_for_task(Library* library, int64_t task_id); - std::unique_ptr create_task(Library* library, int64_t task_id); - std::unique_ptr create_task(Library* library, + mapping::detail::Machine slice_machine_for_task(const Library* library, int64_t task_id); + std::unique_ptr create_task(const Library* library, int64_t task_id); + std::unique_ptr create_task(const Library* library, int64_t task_id, const Shape& launch_shape); void issue_copy(std::shared_ptr target, @@ -125,12 +128,44 @@ class Runtime { void submit(std::unique_ptr op); public: - std::shared_ptr create_store(std::shared_ptr type, int32_t dim = 1); + std::shared_ptr create_array(std::shared_ptr type, + uint32_t dim, + bool nullable); + std::shared_ptr create_array(const Shape& extents, + std::shared_ptr type, + bool nullable, + bool optimize_scalar); + std::shared_ptr create_array_like(std::shared_ptr array, + std::shared_ptr type); + + private: + std::shared_ptr create_struct_array(std::shared_ptr type, + uint32_t dim, + bool nullable); + std::shared_ptr create_struct_array(const Shape& extents, + std::shared_ptr type, + bool nullable, + bool optimize_scalar); + + private: + std::shared_ptr create_base_array(std::shared_ptr type, + uint32_t dim, + bool nullable); + std::shared_ptr create_base_array(const Shape& extents, + std::shared_ptr type, + bool nullable, + bool optimize_scalar); + + public: + std::shared_ptr create_store(std::shared_ptr type, uint32_t); std::shared_ptr create_store(const Shape& extents, std::shared_ptr type, bool optimize_scalar = false); std::shared_ptr create_store(const Scalar& scalar); + private: + void check_dimensionality(uint32_t dim); + public: uint32_t max_pending_exceptions() const; void set_max_pending_exceptions(uint32_t max_pending_exceptions); @@ -210,14 +245,14 @@ class Runtime { } public: - Legion::Future dispatch(Legion::TaskLauncher* launcher, - std::vector* output_requirements = nullptr); - Legion::FutureMap dispatch(Legion::IndexTaskLauncher* launcher, - std::vector* output_requirements = nullptr); - void dispatch(Legion::CopyLauncher* launcher); - void dispatch(Legion::IndexCopyLauncher* launcher); - void dispatch(Legion::FillLauncher* launcher); - void dispatch(Legion::IndexFillLauncher* launcher); + Legion::Future dispatch(Legion::TaskLauncher& launcher, + std::vector& output_requirements); + Legion::FutureMap dispatch(Legion::IndexTaskLauncher& launcher, + std::vector& output_requirements); + void dispatch(Legion::CopyLauncher& launcher); + void dispatch(Legion::IndexCopyLauncher& launcher); + void dispatch(Legion::FillLauncher& launcher); + void dispatch(Legion::IndexFillLauncher& launcher); public: Legion::Future extract_scalar(const Legion::Future& result, uint32_t idx) const; diff --git a/src/core/runtime/library.h b/src/core/runtime/library.h index 8033d11df2..41e9dddafa 100644 --- a/src/core/runtime/library.h +++ b/src/core/runtime/library.h @@ -146,11 +146,8 @@ class Library { public: void register_task(int64_t local_task_id, std::unique_ptr task_info); - private: - friend class Runtime; - Library(detail::Library* impl); - public: + Library(detail::Library* impl); Library(const Library&) = default; Library(Library&&) = default; diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index d29102a536..fdec470529 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -245,7 +245,26 @@ void Runtime::submit(AutoTask&& task) { impl_->submit(std::move(task.impl_)); } void Runtime::submit(ManualTask&& task) { impl_->submit(std::move(task.impl_)); } -LogicalStore Runtime::create_store(const Type& type, int32_t dim) +LogicalArray Runtime::create_array(const Type& type, uint32_t dim, bool nullable) +{ + return LogicalArray(impl_->create_array(type.impl(), dim, nullable)); +} + +LogicalArray Runtime::create_array(const Shape& extents, + const Type& type, + bool nullable, + bool optimize_scalar) +{ + return LogicalArray(impl_->create_array(extents, type.impl(), nullable, optimize_scalar)); +} + +LogicalArray Runtime::create_array_like(const LogicalArray& to_mirror, std::optional type) +{ + auto ty = type ? type.value().impl() : to_mirror.type().impl(); + return LogicalArray(impl_->create_array_like(to_mirror.impl(), std::move(ty))); +} + +LogicalStore Runtime::create_store(const Type& type, uint32_t dim) { return LogicalStore(impl_->create_store(type.impl(), dim)); } diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 34ecd71485..955b315120 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -15,6 +15,7 @@ #include #include +#include "core/data/logical_array.h" #include "core/data/logical_store.h" #include "core/data/shape.h" #include "core/data/store.h" @@ -351,6 +352,43 @@ class Runtime { */ void submit(ManualTask&& task); + public: + /** + * @brief Creates an unbound array + * + * @param extents Shape of the array + * @param type Element type + * @param optimize_scalar When true, the runtime internally uses futures optimized for storing + * scalars + * + * @return Logical array + */ + LogicalArray create_array(const Type& type, uint32_t dim = 1, bool nullable = false); + /** + * @brief Creates a normal array + * + * @param extents Shape of the array + * @param type Element type + * @param optimize_scalar When true, the runtime internally uses futures optimized for storing + * scalars + * + * @return Logical array + */ + LogicalArray create_array(const Shape& extents, + const Type& type, + bool nullable = false, + bool optimize_scalar = false); + /** + * @brief Creates an array isomorphic to the given array + * + * @param type Optional type for the resulting array. Must be compatible with the input array's + * type + * + * @return Logical array isomorphic to the input + */ + LogicalArray create_array_like(const LogicalArray& to_mirror, + std::optional type = std::nullopt); + public: /** * @brief Creates an unbound store @@ -360,7 +398,7 @@ class Runtime { * * @return Logical store */ - LogicalStore create_store(const Type& type, int32_t dim = 1); + LogicalStore create_store(const Type& type, uint32_t dim = 1); /** * @brief Creates a normal store * diff --git a/src/core/task/detail/task_context.cc b/src/core/task/detail/task_context.cc index 3df8d6eb48..5000832e63 100644 --- a/src/core/task/detail/task_context.cc +++ b/src/core/task/detail/task_context.cc @@ -31,21 +31,27 @@ TaskContext::TaskContext(const Legion::Task* task, } TaskDeserializer dez(task, regions); - inputs_ = dez.unpack>(); - outputs_ = dez.unpack>(); - reductions_ = dez.unpack>(); - scalars_ = dez.unpack>(); + inputs_ = dez.unpack_arrays(); + outputs_ = dez.unpack_arrays(); + reductions_ = dez.unpack_arrays(); + scalars_ = dez.unpack_scalars(); // Make copies of stores that we need to postprocess, as clients might move the stores away for (auto& output : outputs_) { - if (output.is_unbound_store()) { - unbound_stores_.push_back(output); - } else if (output.is_future()) { - scalar_stores_.push_back(output); + auto stores = output->stores(); + for (auto& store : stores) { + if (store->is_unbound_store()) { + unbound_stores_.push_back(std::move(store)); + } else if (store->is_future()) { + scalar_stores_.push_back(std::move(store)); + } } } for (auto& reduction : reductions_) { - if (reduction.is_future()) { scalar_stores_.push_back(reduction); } + auto stores = reduction->stores(); + for (auto& store : stores) { + if (store->is_future()) { scalar_stores_.push_back(std::move(store)); } + } } can_raise_exception_ = dez.unpack(); @@ -65,9 +71,9 @@ TaskContext::TaskContext(const Legion::Task* task, // when the number of subregions isn't a multiple of the chosen radix. // To simplify the programming mode, we filter out those "invalid" stores out. if (task_->tag == LEGATE_CORE_TREE_REDUCE_TAG) { - std::vector inputs; + std::vector> inputs; for (auto& input : inputs_) - if (input.valid()) inputs.push_back(std::move(input)); + if (input->valid()) inputs.push_back(std::move(input)); inputs_.swap(inputs); } @@ -84,18 +90,19 @@ TaskContext::TaskContext(const Legion::Task* task, // If the task is running on a GPU and there is at least one scalar store for reduction, // we need to wait for all the host-to-device copies for initialization to finish if (Processor::get_executing_processor().kind() == Processor::Kind::TOC_PROC) - for (auto& reduction : reductions_) - if (reduction.is_future()) { + for (auto& reduction : reductions_) { + auto reduction_store = reduction->data(); + if (reduction_store->is_future()) { CHECK_CUDA(cudaDeviceSynchronize()); break; } + } #endif } void TaskContext::make_all_unbound_stores_empty() { - for (auto& output : outputs_) - if (output.is_unbound_store()) output.bind_empty_data(); + for (auto& store : unbound_stores_) { store->bind_empty_data(); } } ReturnValues TaskContext::pack_return_values() const @@ -123,8 +130,8 @@ std::vector TaskContext::get_return_values() const { std::vector return_values; - for (auto& store : unbound_stores_) { return_values.push_back(store.impl()->pack_weight()); } - for (auto& store : scalar_stores_) { return_values.push_back(store.impl()->pack()); } + for (auto& store : unbound_stores_) { return_values.push_back(store->pack_weight()); } + for (auto& store : scalar_stores_) { return_values.push_back(store->pack()); } // If this is a reduction task, we do sanity checks on the invariants // the Python code relies on. diff --git a/src/core/task/detail/task_context.h b/src/core/task/detail/task_context.h index 0464d01099..74a46a88d0 100644 --- a/src/core/task/detail/task_context.h +++ b/src/core/task/detail/task_context.h @@ -13,8 +13,8 @@ #pragma once #include "core/comm/communicator.h" +#include "core/data/detail/array.h" #include "core/data/scalar.h" -#include "core/data/store.h" #include "core/mapping/detail/machine.h" #include "core/task/detail/return.h" @@ -25,10 +25,10 @@ class TaskContext { TaskContext(const Legion::Task* task, const std::vector& regions); public: - std::vector& inputs() { return inputs_; } - std::vector& outputs() { return outputs_; } - std::vector& reductions() { return reductions_; } - std::vector& scalars() { return scalars_; } + std::vector>& inputs() { return inputs_; } + std::vector>& outputs() { return outputs_; } + std::vector>& reductions() { return reductions_; } + const std::vector& scalars() { return scalars_; } std::vector& communicators() { return comms_; } public: @@ -58,9 +58,9 @@ class TaskContext { const std::vector& regions_; private: - std::vector inputs_, outputs_, reductions_; - std::vector unbound_stores_; - std::vector scalar_stores_; + std::vector> inputs_, outputs_, reductions_; + std::vector> unbound_stores_; + std::vector> scalar_stores_; std::vector scalars_; std::vector comms_; bool can_raise_exception_; diff --git a/src/core/task/task_context.cc b/src/core/task/task_context.cc index 85180a4d5f..c4a769995f 100644 --- a/src/core/task/task_context.cc +++ b/src/core/task/task_context.cc @@ -15,15 +15,43 @@ namespace legate { -std::vector& TaskContext::inputs() { return impl_->inputs(); } +namespace { -std::vector& TaskContext::outputs() { return impl_->outputs(); } +std::vector to_arrays(const std::vector>& array_impls) +{ + std::vector result; + for (const auto& array_impl : array_impls) { result.emplace_back(array_impl); } + return std::move(result); +} -std::vector& TaskContext::reductions() { return impl_->reductions(); } +} // namespace -std::vector& TaskContext::scalars() { return impl_->scalars(); } +Array TaskContext::input(uint32_t index) const { return Array(impl_->inputs().at(index)); } -std::vector& TaskContext::communicators() { return impl_->communicators(); } +std::vector TaskContext::inputs() const { return to_arrays(impl_->inputs()); } + +Array TaskContext::output(uint32_t index) const { return Array(impl_->outputs().at(index)); } + +std::vector TaskContext::outputs() const { return to_arrays(impl_->outputs()); } + +Array TaskContext::reduction(uint32_t index) const { return Array(impl_->reductions().at(index)); } + +std::vector TaskContext::reductions() const { return to_arrays(impl_->reductions()); } + +const Scalar& TaskContext::scalar(uint32_t index) const { return impl_->scalars().at(index); } + +const std::vector& TaskContext::scalars() const { return impl_->scalars(); } + +std::vector TaskContext::communicators() const +{ + return impl_->communicators(); +} + +size_t TaskContext::num_inputs() const { return impl_->inputs().size(); } + +size_t TaskContext::num_outputs() const { return impl_->outputs().size(); } + +size_t TaskContext::num_reductions() const { return impl_->reductions().size(); } bool TaskContext::is_single_task() const { return impl_->is_single_task(); } diff --git a/src/core/task/task_context.h b/src/core/task/task_context.h index cc06d00256..6bc4aea6e4 100644 --- a/src/core/task/task_context.h +++ b/src/core/task/task_context.h @@ -13,8 +13,8 @@ #pragma once #include "core/comm/communicator.h" +#include "core/data/array.h" #include "core/data/scalar.h" -#include "core/data/store.h" #include "core/mapping/machine.h" /** @@ -28,9 +28,6 @@ class TaskContext; namespace legate { -class Store; -class Scalar; - /** * @ingroup task * @brief A task context that contains task arguments and communicators @@ -38,29 +35,61 @@ class Scalar; class TaskContext { public: /** - * @brief Returns input stores of the task + * @brief Returns an input array of the task + * + * @param index Index of the array + * + * @return Array + */ + Array input(uint32_t index) const; + /** + * @brief Returns all input arrays of the task * - * @return Vector of input stores + * @return Vector of arrays */ - std::vector& inputs(); + std::vector inputs() const; /** - * @brief Returns output stores of the task + * @brief Returns an output array of the task + * + * @param index Index of the array * - * @return Vector of output stores + * @return Array */ - std::vector& outputs(); + Array output(uint32_t index) const; /** - * @brief Returns reduction stores of the task + * @brief Returns all output arrays of the task * - * @return Vector of reduction stores + * @return Vector of arrays */ - std::vector& reductions(); + std::vector outputs() const; + /** + * @brief Returns a reduction array of the task + * + * @param index Index of the array + * + * @return Array + */ + Array reduction(uint32_t index) const; + /** + * @brief Returns all reduction arrays of the task + * + * @return Vector of arrays + */ + std::vector reductions() const; /** * @brief Returns by-value arguments of the task * + * @param index Index of the array + * * @return Vector of scalar objects */ - std::vector& scalars(); + const Scalar& scalar(uint32_t index) const; + /** + * @brief Returns by-value arguments of the task + * + * @return Vector of scalars + */ + const std::vector& scalars() const; /** * @brief Returns communicators of the task * @@ -70,7 +99,27 @@ class TaskContext { * * @return Vector of communicator objects */ - std::vector& communicators(); + std::vector communicators() const; + + public: + /** + * @brief Returns the number of task's inputs + * + * @return Number of arrays + */ + size_t num_inputs() const; + /** + * @brief Returns the number of task's outputs + * + * @return Number of arrays + */ + size_t num_outputs() const; + /** + * @brief Returns the number of task's reductions + * + * @return Number of arrays + */ + size_t num_reductions() const; public: /** diff --git a/src/core/type/detail/type_info.cc b/src/core/type/detail/type_info.cc index d3bdd0be52..7049adacab 100644 --- a/src/core/type/detail/type_info.cc +++ b/src/core/type/detail/type_info.cc @@ -62,6 +62,12 @@ const char* _VARIABLE_SIZE_ERROR_MESSAGE = "Variable-size element type cannot be Type::Type(Code c) : code(c) {} +uint32_t Type::size() const +{ + throw std::invalid_argument("Size of a variable size type is undefined"); + return 0; +} + const FixedArrayType& Type::as_fixed_array_type() const { throw std::invalid_argument("Type is not a fixed array type"); @@ -74,6 +80,12 @@ const StructType& Type::as_struct_type() const return *static_cast(nullptr); } +const ListType& Type::as_list_type() const +{ + throw std::invalid_argument("Type is not a list type"); + return *static_cast(nullptr); +} + void Type::record_reduction_operator(int32_t op_kind, int32_t global_op_id) const { detail::Runtime::get_runtime()->record_reduction_operator(uid(), op_kind, global_op_id); @@ -106,9 +118,7 @@ bool PrimitiveType::equal(const Type& other) const { return code == other.code; ExtensionType::ExtensionType(int32_t uid, Type::Code code) : Type(code), uid_(uid) {} -FixedArrayType::FixedArrayType(int32_t uid, - std::shared_ptr element_type, - uint32_t N) noexcept(false) +FixedArrayType::FixedArrayType(int32_t uid, std::shared_ptr element_type, uint32_t N) : ExtensionType(uid, Type::Code::FIXED_ARRAY), element_type_(std::move(element_type)), N_(N), @@ -148,15 +158,21 @@ bool FixedArrayType::equal(const Type& other) const #endif } -StructType::StructType(int32_t uid, - std::vector>&& field_types, - bool align) noexcept(false) +StructType::StructType(int32_t uid, std::vector>&& field_types, bool align) : ExtensionType(uid, Type::Code::STRUCT), aligned_(align), alignment_(1), size_(0), field_types_(std::move(field_types)) { + if (std::any_of( + field_types_.begin(), field_types_.end(), [](auto& ty) { return ty->variable_size(); })) { + throw std::runtime_error("Struct types can't have a variable size field"); + } + if (field_types_.empty()) { + throw std::invalid_argument("Struct types must have at least one field"); + } + offsets_.reserve(field_types_.size()); if (aligned_) { static constexpr auto align_offset = [](uint32_t offset, uint32_t align) { @@ -247,14 +263,51 @@ std::shared_ptr primitive_type(Type::Code code) return std::make_shared(code); } +ListType::ListType(int32_t uid, std::shared_ptr element_type) + : ExtensionType(uid, Type::Code::LIST), element_type_(std::move(element_type)) +{ + if (element_type_->variable_size()) { + throw std::runtime_error("Nested variable size types are not implemented yet"); + } +} + +std::string ListType::to_string() const +{ + std::stringstream ss; + ss << "list(" << element_type_->to_string() << ")"; + return std::move(ss).str(); +} + +void ListType::pack(BufferBuilder& buffer) const +{ + buffer.pack(static_cast(code)); + buffer.pack(uid_); + element_type_->pack(buffer); +} + +const ListType& ListType::as_list_type() const { return *this; } + +bool ListType::equal(const Type& other) const +{ + if (code != other.code) return false; + auto& casted = static_cast(other); + +#ifdef DEBUG_LEGATE + // Do a structural check in debug mode + return uid_ == casted.uid_ && element_type_ == casted.element_type_; +#else + // Each type is uniquely identified by the uid, so it's sufficient to compare between uids + return uid_ == casted.uid_; +#endif +} + std::shared_ptr string_type() { static auto type = std::make_shared(); return type; } -std::shared_ptr fixed_array_type(std::shared_ptr element_type, - uint32_t N) noexcept(false) +std::shared_ptr fixed_array_type(std::shared_ptr element_type, uint32_t N) { // We use UIDs of the following format for "common" fixed array types // 1B 1B @@ -268,18 +321,165 @@ std::shared_ptr fixed_array_type(std::shared_ptr element_type, return std::make_shared(uid, std::move(element_type), N); } -std::shared_ptr struct_type(const std::vector>& field_types, - bool align) noexcept(false) +std::shared_ptr struct_type(const std::vector>& field_types, bool align) { return std::make_shared( Runtime::get_runtime()->get_type_uid(), std::vector>(field_types), align); } -std::shared_ptr struct_type(std::vector>&& field_types, - bool align) noexcept(false) +std::shared_ptr struct_type(std::vector>&& field_types, bool align) { return std::make_shared( Runtime::get_runtime()->get_type_uid(), std::move(field_types), align); } +std::shared_ptr list_type(std::shared_ptr element_type) +{ + return std::make_shared(Runtime::get_runtime()->get_type_uid(), + std::move(element_type)); +} + +std::shared_ptr bool_() +{ + static auto result = detail::primitive_type(Type::Code::BOOL); + return result; +} + +std::shared_ptr int8() +{ + static auto result = detail::primitive_type(Type::Code::INT8); + return result; +} + +std::shared_ptr int16() +{ + static auto result = detail::primitive_type(Type::Code::INT16); + return result; +} + +std::shared_ptr int32() +{ + static auto result = detail::primitive_type(Type::Code::INT32); + return result; +} + +std::shared_ptr int64() +{ + static auto result = detail::primitive_type(Type::Code::INT64); + return result; +} + +std::shared_ptr uint8() +{ + static auto result = detail::primitive_type(Type::Code::UINT8); + return result; +} + +std::shared_ptr uint16() +{ + static auto result = detail::primitive_type(Type::Code::UINT16); + return result; +} + +std::shared_ptr uint32() +{ + static auto result = detail::primitive_type(Type::Code::UINT32); + return result; +} + +std::shared_ptr uint64() +{ + static auto result = detail::primitive_type(Type::Code::UINT64); + return result; +} + +std::shared_ptr float16() +{ + static auto result = detail::primitive_type(Type::Code::FLOAT16); + return result; +} + +std::shared_ptr float32() +{ + static auto result = detail::primitive_type(Type::Code::FLOAT32); + return result; +} + +std::shared_ptr float64() +{ + static auto result = detail::primitive_type(Type::Code::FLOAT64); + return result; +} + +std::shared_ptr complex64() +{ + static auto result = detail::primitive_type(Type::Code::COMPLEX64); + return result; +} + +std::shared_ptr complex128() +{ + static auto result = detail::primitive_type(Type::Code::COMPLEX128); + return result; +} + +namespace { + +constexpr int32_t POINT_UID_BASE = static_cast(Type::Code::INVALID); +constexpr int32_t RECT_UID_BASE = POINT_UID_BASE + LEGATE_MAX_DIM + 1; + +} // namespace + +std::shared_ptr point_type(int32_t ndim) +{ + static std::shared_ptr cache[LEGATE_MAX_DIM + 1]; + + if (ndim <= 0 || ndim > LEGATE_MAX_DIM) + throw std::out_of_range(std::to_string(ndim) + " is not a supported number of dimensions"); + if (nullptr == cache[ndim]) { + cache[ndim] = std::make_shared(POINT_UID_BASE + ndim, int64(), ndim); + } + return cache[ndim]; +} + +std::shared_ptr rect_type(int32_t ndim) +{ + static std::shared_ptr cache[LEGATE_MAX_DIM + 1]; + + if (ndim <= 0 || ndim > LEGATE_MAX_DIM) + throw std::out_of_range(std::to_string(ndim) + " is not a supported number of dimensions"); + + if (nullptr == cache[ndim]) { + auto pt_type = point_type(ndim); + std::vector> field_types{pt_type, pt_type}; + cache[ndim] = std::make_shared( + RECT_UID_BASE + ndim, std::move(field_types), true /*align*/); + } + return cache[ndim]; +} + +bool is_point_type(const std::shared_ptr& type, int32_t ndim) +{ + switch (type->code) { + case Type::Code::INT64: { + return 1 == ndim; + } + case Type::Code::FIXED_ARRAY: { + return type->as_fixed_array_type().num_elements() == ndim; + } + default: { + return false; + } + } + return false; +} + +bool is_rect_type(const std::shared_ptr& type, int32_t ndim) +{ + if (type->code != Type::Code::STRUCT) return false; + const auto& st_type = type->as_struct_type(); + return st_type.num_fields() == 2 && is_point_type(st_type.field_type(0), ndim) && + is_point_type(st_type.field_type(1), ndim); +} + } // namespace legate::detail diff --git a/src/core/type/detail/type_info.h b/src/core/type/detail/type_info.h index ec3a41810b..c5a7f6c54a 100644 --- a/src/core/type/detail/type_info.h +++ b/src/core/type/detail/type_info.h @@ -18,6 +18,7 @@ namespace legate::detail { class BufferBuilder; class FixedArrayType; +class ListType; class StructType; class Type { @@ -29,7 +30,7 @@ class Type { public: virtual ~Type() {} - virtual uint32_t size() const = 0; + virtual uint32_t size() const; virtual uint32_t alignment() const = 0; virtual int32_t uid() const = 0; virtual bool variable_size() const = 0; @@ -38,6 +39,7 @@ class Type { virtual void pack(BufferBuilder& buffer) const = 0; virtual const FixedArrayType& as_fixed_array_type() const; virtual const StructType& as_struct_type() const; + virtual const ListType& as_list_type() const; virtual bool equal(const Type& other) const = 0; public: @@ -72,7 +74,6 @@ class StringType : public Type { public: StringType(); bool variable_size() const override { return true; } - uint32_t size() const override { return 0; } uint32_t alignment() const override { return 0; } int32_t uid() const override; std::string to_string() const override; @@ -95,7 +96,7 @@ class ExtensionType : public Type { class FixedArrayType : public ExtensionType { public: - FixedArrayType(int32_t uid, std::shared_ptr element_type, uint32_t N) noexcept(false); + FixedArrayType(int32_t uid, std::shared_ptr element_type, uint32_t N); uint32_t size() const override { return size_; } uint32_t alignment() const override { return element_type_->alignment(); } bool variable_size() const override { return false; } @@ -117,9 +118,7 @@ class FixedArrayType : public ExtensionType { class StructType : public ExtensionType { public: - StructType(int32_t uid, - std::vector>&& field_types, - bool align = false) noexcept(false); + StructType(int32_t uid, std::vector>&& field_types, bool align = false); uint32_t size() const override { return size_; } uint32_t alignment() const override { return alignment_; } bool variable_size() const override { return false; } @@ -129,6 +128,7 @@ class StructType : public ExtensionType { uint32_t num_fields() const { return field_types_.size(); } std::shared_ptr field_type(uint32_t field_idx) const; + const std::vector>& field_types() const { return field_types_; } bool aligned() const { return aligned_; } private: @@ -142,17 +142,54 @@ class StructType : public ExtensionType { std::vector offsets_{}; }; +class ListType : public ExtensionType { + public: + ListType(int32_t uid, std::shared_ptr element_type); + uint32_t alignment() const override { return 0; } + bool variable_size() const override { return true; } + std::string to_string() const override; + void pack(BufferBuilder& buffer) const override; + const ListType& as_list_type() const override; + + std::shared_ptr element_type() const { return element_type_; } + + private: + bool equal(const Type& other) const override; + + private: + const std::shared_ptr element_type_; +}; + std::shared_ptr primitive_type(Type::Code code); std::shared_ptr string_type(); -std::shared_ptr fixed_array_type(std::shared_ptr element_type, - uint32_t N) noexcept(false); +std::shared_ptr fixed_array_type(std::shared_ptr element_type, uint32_t N); std::shared_ptr struct_type(const std::vector>& field_types, - bool align) noexcept(false); - -std::shared_ptr struct_type(std::vector>&& field_types, - bool align) noexcept(false); + bool align); + +std::shared_ptr struct_type(std::vector>&& field_types, bool align); + +std::shared_ptr list_type(std::shared_ptr element_type); + +std::shared_ptr bool_(); +std::shared_ptr int8(); +std::shared_ptr int16(); +std::shared_ptr int32(); +std::shared_ptr int64(); +std::shared_ptr uint8(); +std::shared_ptr uint16(); +std::shared_ptr uint32(); +std::shared_ptr uint64(); +std::shared_ptr float16(); +std::shared_ptr float32(); +std::shared_ptr float64(); +std::shared_ptr complex64(); +std::shared_ptr complex128(); +std::shared_ptr point_type(int32_t ndim); +std::shared_ptr rect_type(int32_t ndim); +bool is_point_type(const std::shared_ptr& type, int32_t ndim); +bool is_rect_type(const std::shared_ptr& type, int32_t ndim); } // namespace legate::detail diff --git a/src/core/type/type_info.cc b/src/core/type/type_info.cc index 42082c8a48..c91a0c673e 100644 --- a/src/core/type/type_info.cc +++ b/src/core/type/type_info.cc @@ -44,6 +44,12 @@ StructType Type::as_struct_type() const return StructType(impl_); } +ListType Type::as_list_type() const +{ + if (impl_->code != Code::LIST) { throw std::invalid_argument("Type is not a list type"); } + return ListType(impl_); +} + void Type::record_reduction_operator(int32_t op_kind, int32_t global_op_id) const { impl_->record_reduction_operator(op_kind, global_op_id); @@ -98,22 +104,31 @@ bool StructType::aligned() const StructType::StructType(std::shared_ptr type) : Type(std::move(type)) {} +Type ListType::element_type() const +{ + return Type(static_cast(impl_.get())->element_type()); +} + +ListType::ListType(std::shared_ptr type) : Type(std::move(type)) {} + Type primitive_type(Type::Code code) { return Type(detail::primitive_type(code)); } Type string_type() { return Type(detail::string_type()); } -Type fixed_array_type(const Type& element_type, uint32_t N) noexcept(false) +Type fixed_array_type(const Type& element_type, uint32_t N) { return Type(detail::fixed_array_type(element_type.impl(), N)); } -Type struct_type(const std::vector& field_types, bool align) noexcept(false) +Type struct_type(const std::vector& field_types, bool align) { std::vector> detail_field_types; for (const auto& field_type : field_types) { detail_field_types.push_back(field_type.impl()); } return Type(detail::struct_type(std::move(detail_field_types), align)); } +Type list_type(const Type& element_type) { return Type(detail::list_type(element_type.impl())); } + std::ostream& operator<<(std::ostream& ostream, const Type::Code& code) { ostream << static_cast(code); @@ -126,140 +141,46 @@ std::ostream& operator<<(std::ostream& ostream, const Type& type) return ostream; } -Type bool_() -{ - static auto result = primitive_type(Type::Code::BOOL); - return result; -} +Type bool_() { return Type(detail::bool_()); } -Type int8() -{ - static auto result = primitive_type(Type::Code::INT8); - return result; -} - -Type int16() -{ - static auto result = primitive_type(Type::Code::INT16); - return result; -} - -Type int32() -{ - static auto result = primitive_type(Type::Code::INT32); - return result; -} - -Type int64() -{ - static auto result = primitive_type(Type::Code::INT64); - return result; -} - -Type uint8() -{ - static auto result = primitive_type(Type::Code::UINT8); - return result; -} +Type int8() { return Type(detail::int8()); } -Type uint16() -{ - static auto result = primitive_type(Type::Code::UINT16); - return result; -} +Type int16() { return Type(detail::int16()); } -Type uint32() -{ - static auto result = primitive_type(Type::Code::UINT32); - return result; -} +Type int32() { return Type(detail::int32()); } -Type uint64() -{ - static auto result = primitive_type(Type::Code::UINT64); - return result; -} +Type int64() { return Type(detail::int64()); } -Type float16() -{ - static auto result = primitive_type(Type::Code::FLOAT16); - return result; -} +Type uint8() { return Type(detail::uint8()); } -Type float32() -{ - static auto result = primitive_type(Type::Code::FLOAT32); - return result; -} +Type uint16() { return Type(detail::uint16()); } -Type float64() -{ - static auto result = primitive_type(Type::Code::FLOAT64); - return result; -} +Type uint32() { return Type(detail::uint32()); } -Type complex64() -{ - static auto result = primitive_type(Type::Code::COMPLEX64); - return result; -} +Type uint64() { return Type(detail::uint64()); } -Type complex128() -{ - static auto result = primitive_type(Type::Code::COMPLEX128); - return result; -} +Type float16() { return Type(detail::float16()); } -namespace { +Type float32() { return Type(detail::float32()); } -constexpr int32_t POINT_UID_BASE = static_cast(Type::Code::INVALID); -constexpr int32_t RECT_UID_BASE = POINT_UID_BASE + LEGATE_MAX_DIM + 1; +Type float64() { return Type(detail::float64()); } -} // namespace +Type complex64() { return Type(detail::complex64()); } -Type point_type(int32_t ndim) -{ - static Type cache[LEGATE_MAX_DIM + 1]; +Type complex128() { return Type(detail::complex128()); } - if (ndim <= 0 || ndim > LEGATE_MAX_DIM) - throw std::out_of_range(std::to_string(ndim) + " is not a supported number of dimensions"); - if (cache[ndim].impl() == nullptr) { - cache[ndim] = - Type(std::make_shared(POINT_UID_BASE + ndim, int64().impl(), ndim)); - } - return cache[ndim]; -} +Type point_type(int32_t ndim) { return Type(detail::point_type(ndim)); } -Type rect_type(int32_t ndim) -{ - std::vector> field_types{point_type(ndim).impl(), - point_type(ndim).impl()}; - return Type(std::make_shared( - RECT_UID_BASE + ndim, std::move(field_types), true /*align*/)); -} +Type rect_type(int32_t ndim) { return Type(detail::rect_type(ndim)); } bool is_point_type(const Type& type, int32_t ndim) { - switch (type.code()) { - case Type::Code::INT64: { - return 1 == ndim; - } - case Type::Code::FIXED_ARRAY: { - return type.as_fixed_array_type().num_elements() == ndim; - } - default: { - return false; - } - } - return false; + return detail::is_point_type(type.impl(), ndim); } bool is_rect_type(const Type& type, int32_t ndim) { - if (type.code() != Type::Code::STRUCT) return false; - auto st_type = type.as_struct_type(); - return st_type.num_fields() == 2 && is_point_type(st_type.field_type(0), ndim) && - is_point_type(st_type.field_type(1), ndim); + return detail::is_rect_type(type.impl(), ndim); } } // namespace legate diff --git a/src/core/type/type_info.h b/src/core/type/type_info.h index 18f8e4cdf2..528ce722b2 100644 --- a/src/core/type/type_info.h +++ b/src/core/type/type_info.h @@ -27,6 +27,7 @@ class Type; namespace legate { class FixedArrayType; +class ListType; class StructType; /** @@ -73,6 +74,7 @@ class Type { FIXED_ARRAY = FIXED_ARRAY_LT, /*!< Fixed-size array type */ STRUCT = STRUCT_LT, /*!< Struct type */ STRING = STRING_LT, /*!< String type */ + LIST = LIST_LT, /*!< List type */ INVALID = INVALID_LT, /*!< Invalid type */ }; @@ -137,6 +139,14 @@ class Type { * @return Type object */ StructType as_struct_type() const; + /** + * @brief Dynamically casts the type into a struct type. + * + * If the type is not a struct type, an exception will be raised. + * + * @return Type object + */ + ListType as_list_type() const; /** * @brief Records a reduction operator. * @@ -256,6 +266,24 @@ class StructType : public Type { StructType(std::shared_ptr type); }; +/** + * @ingroup types + * @brief A class for list types + */ +class ListType : public Type { + public: + /** + * @brief Returns the element type + * + * @return Element type + */ + Type element_type() const; + + private: + friend class Type; + ListType(std::shared_ptr type); +}; + /** * @ingroup types * @brief Creates a metadata object for a primitive type @@ -283,7 +311,7 @@ Type string_type(); * * @return Type object */ -Type fixed_array_type(const Type& element_type, uint32_t N) noexcept(false); +Type fixed_array_type(const Type& element_type, uint32_t N); /** * @ingroup types @@ -294,7 +322,17 @@ Type fixed_array_type(const Type& element_type, uint32_t N) noexcept(false); * * @return Type object */ -Type struct_type(const std::vector& field_types, bool align = false) noexcept(false); +Type struct_type(const std::vector& field_types, bool align = false); + +/** + * @ingroup types + * @brief Creates a metadata object for a list type + * + * @param element_type Type of the list elements + * + * @return Type object + */ +Type list_type(const Type& element_type); /** * @ingroup types @@ -307,7 +345,7 @@ Type struct_type(const std::vector& field_types, bool align = false) noexc */ template std::enable_if_t...>, Type> struct_type( - bool align, Types... field_types) noexcept(false) + bool align, Types... field_types) { std::vector vec_field_types; auto copy_field_type = [&vec_field_types](const auto& field_type) { diff --git a/src/core/utilities/deserializer.cc b/src/core/utilities/deserializer.cc index 224f46ec96..6874a77f79 100644 --- a/src/core/utilities/deserializer.cc +++ b/src/core/utilities/deserializer.cc @@ -36,28 +36,80 @@ TaskDeserializer::TaskDeserializer(const Legion::Task* task, first_task_ = !task->is_index_space || (task->index_point == task->index_domain.lo()); } -std::shared_ptr TaskDeserializer::_unpack_store() +std::vector> TaskDeserializer::unpack_arrays() +{ + std::vector> arrays; + auto size = unpack(); + arrays.reserve(size); + for (uint32_t idx = 0; idx < size; ++idx) { arrays.emplace_back(unpack_array()); } + return std::move(arrays); +} + +std::shared_ptr TaskDeserializer::unpack_array() +{ + auto kind = static_cast(unpack()); + + switch (kind) { + case detail::ArrayKind::BASE: return unpack_base_array(); + case detail::ArrayKind::LIST: return unpack_list_array(); + case detail::ArrayKind::STRUCT: return unpack_struct_array(); + } + assert(false); + return nullptr; +} + +std::shared_ptr TaskDeserializer::unpack_base_array() +{ + auto data = unpack_store(); + auto nullable = unpack(); + auto null_mask = nullable ? unpack_store() : nullptr; + return std::make_shared(std::move(data), std::move(null_mask)); +} + +std::shared_ptr TaskDeserializer::unpack_list_array() +{ + auto type = unpack_type(); + unpack(); // Unpack kind + auto descriptor = unpack_base_array(); + auto vardata = unpack_array(); + return std::make_shared( + std::move(type), std::move(descriptor), std::move(vardata)); +} + +std::shared_ptr TaskDeserializer::unpack_struct_array() +{ + auto type = unpack_type(); +#ifdef DEBUG_LEGATE + assert(type->code == Type::Code::STRUCT); +#endif + std::vector> fields; + const auto& st_type = type->as_struct_type(); + auto nullable = unpack(); + auto null_mask = nullable ? unpack_store() : nullptr; + for (uint32_t idx = 0; idx < st_type.num_fields(); ++idx) { fields.push_back(unpack_array()); } + return std::make_shared( + std::move(type), std::move(null_mask), std::move(fields)); +} + +std::shared_ptr TaskDeserializer::unpack_store() { auto is_future = unpack(); auto is_output_region = unpack(); auto dim = unpack(); auto type = unpack_type(); - - auto transform = unpack_transform(); + auto transform = unpack_transform(); + auto redop_id = unpack(); if (is_future) { - auto redop_id = unpack(); - auto fut = unpack(); + auto fut = unpack(); if (redop_id != -1 && !first_task_) fut.initialize_with_identity(redop_id); return std::make_shared( dim, std::move(type), redop_id, fut, std::move(transform)); } else if (!is_output_region) { - auto redop_id = unpack(); - auto rf = unpack(); + auto rf = unpack(); return std::make_shared( dim, std::move(type), redop_id, std::move(rf), std::move(transform)); } else { - auto redop_id = unpack(); assert(redop_id == -1); auto out = unpack(); return std::make_shared( @@ -67,24 +119,13 @@ std::shared_ptr TaskDeserializer::_unpack_store() void TaskDeserializer::_unpack(detail::FutureWrapper& value) { - auto read_only = unpack(); - auto has_storage = unpack(); - auto field_size = unpack(); - - auto point = unpack>(); - Domain domain; - domain.dim = static_cast(point.size()); - for (int32_t idx = 0; idx < domain.dim; ++idx) { - domain.rect_data[idx] = 0; - domain.rect_data[idx + domain.dim] = point[idx] - 1; - } - - Legion::Future future; - if (has_storage) { - future = futures_[0]; - futures_ = futures_.subspan(1); - } + auto read_only = unpack(); + auto future_index = unpack(); + auto field_size = unpack(); + auto domain = unpack(); + auto has_storage = future_index >= 0; + Legion::Future future = has_storage ? futures_[future_index] : Legion::Future(); value = detail::FutureWrapper(read_only, field_size, domain, future, has_storage && first_task_); } @@ -133,16 +174,67 @@ MapperDataDeserializer::MapperDataDeserializer(const Legion::Mappable* mappable) TaskDeserializer::TaskDeserializer(const Legion::Task* task, Legion::Mapping::MapperRuntime* runtime, Legion::Mapping::MapperContext context) - : BaseDeserializer(task->args, task->arglen), - task_(task), - runtime_(runtime), - context_(context), - future_index_(0) + : BaseDeserializer(task->args, task->arglen), task_(task), runtime_(runtime), context_(context) { first_task_ = false; } -void TaskDeserializer::_unpack(detail::Store& store) +std::vector> TaskDeserializer::unpack_arrays() +{ + std::vector> arrays; + auto size = unpack(); + arrays.reserve(size); + for (uint32_t idx = 0; idx < size; ++idx) { arrays.emplace_back(unpack_array()); } + return std::move(arrays); +} + +std::shared_ptr TaskDeserializer::unpack_array() +{ + auto kind = static_cast(unpack()); + + switch (kind) { + case legate::detail::ArrayKind::BASE: return unpack_base_array(); + case legate::detail::ArrayKind::LIST: return unpack_list_array(); + case legate::detail::ArrayKind::STRUCT: return unpack_struct_array(); + } + assert(false); + return nullptr; +} + +std::shared_ptr TaskDeserializer::unpack_base_array() +{ + auto data = unpack_store(); + auto nullable = unpack(); + auto null_mask = nullable ? unpack_store() : nullptr; + return std::make_shared(std::move(data), std::move(null_mask)); +} + +std::shared_ptr TaskDeserializer::unpack_list_array() +{ + auto type = unpack_type(); + unpack(); // Unpack kind + auto descriptor = unpack_base_array(); + auto vardata = unpack_array(); + return std::make_shared( + std::move(type), std::move(descriptor), std::move(vardata)); +} + +std::shared_ptr TaskDeserializer::unpack_struct_array() +{ + auto type = unpack_type(); +#ifdef DEBUG_LEGATE + assert(type->code == Type::Code::STRUCT); +#endif + std::vector> fields; + const auto& st_type = type->as_struct_type(); + auto nullable = unpack(); + auto null_mask = nullable ? unpack_store() : nullptr; + for (uint32_t idx = 0; idx < st_type.num_fields(); ++idx) { fields.push_back(unpack_array()); } + return std::make_shared( + std::move(type), std::move(null_mask), std::move(fields)); +} + +std::shared_ptr TaskDeserializer::unpack_store() { auto is_future = unpack(); auto is_output_region = unpack(); @@ -155,38 +247,23 @@ void TaskDeserializer::_unpack(detail::Store& store) // We still need to parse the reduction op unpack(); auto fut = unpack(); - store = detail::Store(dim, std::move(type), fut, std::move(transform)); - } else { - auto redop_id = unpack(); - detail::RegionField rf; - _unpack(rf, is_output_region); - store = detail::Store(runtime_, - context_, - dim, - std::move(type), - redop_id, - rf, - is_output_region, - std::move(transform)); + return std::make_shared(dim, std::move(type), fut, std::move(transform)); } + auto redop_id = unpack(); + detail::RegionField rf; + _unpack(rf, is_output_region); + return std::make_shared( + runtime_, context_, dim, std::move(type), redop_id, rf, is_output_region, std::move(transform)); } void TaskDeserializer::_unpack(detail::FutureWrapper& value) { // We still need to deserialize these fields to get to the domain unpack(); - unpack(); + auto future_index = unpack(); unpack(); - - auto point = unpack>(); - Domain domain; - domain.dim = static_cast(point.size()); - for (int32_t idx = 0; idx < domain.dim; ++idx) { - domain.rect_data[idx] = 0; - domain.rect_data[idx + domain.dim] = point[idx] - 1; - } - - value = detail::FutureWrapper(future_index_++, domain); + auto domain = unpack(); + value = detail::FutureWrapper(future_index, domain); } void TaskDeserializer::_unpack(detail::RegionField& value, bool is_output_region) diff --git a/src/core/utilities/deserializer.h b/src/core/utilities/deserializer.h index 4f4f285625..9ebe7d6964 100644 --- a/src/core/utilities/deserializer.h +++ b/src/core/utilities/deserializer.h @@ -17,11 +17,13 @@ #include "legion.h" #include "core/comm/communicator.h" +#include "core/data/detail/array.h" #include "core/data/detail/scalar.h" #include "core/data/detail/store.h" #include "core/data/detail/transform.h" #include "core/data/scalar.h" #include "core/data/store.h" +#include "core/mapping/detail/array.h" #include "core/mapping/detail/machine.h" #include "core/mapping/detail/store.h" #include "core/type/detail/type_info.h" @@ -62,12 +64,6 @@ class BaseDeserializer { values.reserve(size); for (uint32_t idx = 0; idx < size; ++idx) values.emplace_back(unpack()); } - void _unpack(std::vector& values) - { - auto size = unpack(); - values.reserve(size); - for (uint32_t idx = 0; idx < size; ++idx) values.emplace_back(_unpack_scalar()); - } template void _unpack(std::pair& values) { @@ -76,10 +72,12 @@ class BaseDeserializer { } public: - Scalar _unpack_scalar(); + std::vector unpack_scalars(); + std::unique_ptr unpack_scalar(); void _unpack(mapping::TaskTarget& value); void _unpack(mapping::ProcessorRange& value); void _unpack(mapping::detail::Machine& value); + void _unpack(Domain& domain); public: Span current_args() const { return args_; } @@ -101,15 +99,14 @@ class TaskDeserializer : public BaseDeserializer { using BaseDeserializer::_unpack; public: - void _unpack(std::vector& values) - { - auto size = unpack(); - values.reserve(size); - for (uint32_t idx = 0; idx < size; ++idx) values.emplace_back(_unpack_store()); - } + std::vector> unpack_arrays(); + std::shared_ptr unpack_array(); + std::shared_ptr unpack_base_array(); + std::shared_ptr unpack_list_array(); + std::shared_ptr unpack_struct_array(); + std::shared_ptr unpack_store(); public: - std::shared_ptr _unpack_store(); void _unpack(detail::FutureWrapper& value); void _unpack(detail::RegionField& value); void _unpack(detail::UnboundRegionField& value); @@ -144,6 +141,15 @@ class TaskDeserializer : public BaseDeserializer { using BaseDeserializer::_unpack; public: + std::vector> unpack_arrays(); + std::shared_ptr unpack_array(); + std::shared_ptr unpack_base_array(); + std::shared_ptr unpack_list_array(); + std::shared_ptr unpack_struct_array(); + std::shared_ptr unpack_store(); + + public: + void _unpack(detail::Array& array); void _unpack(detail::Store& store); void _unpack(detail::FutureWrapper& value); void _unpack(detail::RegionField& value, bool is_output_region); @@ -152,7 +158,6 @@ class TaskDeserializer : public BaseDeserializer { const Legion::Task* task_; Legion::Mapping::MapperRuntime* runtime_; Legion::Mapping::MapperContext context_; - uint32_t future_index_; }; class CopyDeserializer : public BaseDeserializer { diff --git a/src/core/utilities/deserializer.inl b/src/core/utilities/deserializer.inl index c6b7117933..83a16a153e 100644 --- a/src/core/utilities/deserializer.inl +++ b/src/core/utilities/deserializer.inl @@ -24,14 +24,24 @@ BaseDeserializer::BaseDeserializer(const void* args, size_t arglen } template -Scalar BaseDeserializer::_unpack_scalar() +std::vector BaseDeserializer::unpack_scalars() +{ + std::vector values; + auto size = unpack(); + values.reserve(size); + for (uint32_t idx = 0; idx < size; ++idx) { values.emplace_back(unpack_scalar()); } + return values; +} + +template +std::unique_ptr BaseDeserializer::unpack_scalar() { // this unpack_type call must be in a separate line from the following one because they both // read and update the buffer location. auto type = unpack_type(); auto result = std::make_unique(type, args_.ptr(), false /*copy*/); args_ = args_.subspan(result->size()); - return Scalar(std::move(result)); + return std::move(result); } template @@ -60,6 +70,17 @@ void BaseDeserializer::_unpack(mapping::detail::Machine& value) } } +template +void BaseDeserializer::_unpack(Domain& domain) +{ + domain.dim = unpack(); + for (int32_t idx = 0; idx < domain.dim; ++idx) { + auto coord = unpack(); + domain.rect_data[idx] = 0; + domain.rect_data[idx + domain.dim] = coord - 1; + } +} + template std::shared_ptr BaseDeserializer::unpack_transform() { @@ -130,6 +151,11 @@ std::shared_ptr BaseDeserializer::unpack_type() return std::make_shared(uid, std::move(field_types), align); } + case Type::Code::LIST: { + auto uid = unpack(); + auto type = unpack_type(); + return std::make_shared(uid, std::move(type)); + } case Type::Code::BOOL: case Type::Code::INT8: case Type::Code::INT16: diff --git a/src/core/utilities/detail/buffer_builder.h b/src/core/utilities/detail/buffer_builder.h index 1754c1399b..67c37918ef 100644 --- a/src/core/utilities/detail/buffer_builder.h +++ b/src/core/utilities/detail/buffer_builder.h @@ -70,6 +70,12 @@ class BufferBuilder { */ Legion::UntypedBuffer to_legion_buffer() const; + public: + BufferBuilder(const BufferBuilder&) = default; + BufferBuilder& operator=(const BufferBuilder&) = default; + BufferBuilder(BufferBuilder&&) = default; + BufferBuilder& operator=(BufferBuilder&&) = default; + private: std::vector buffer_; }; diff --git a/src/core/utilities/typedefs.h b/src/core/utilities/typedefs.h index 9cd53f1f66..ac387623ac 100644 --- a/src/core/utilities/typedefs.h +++ b/src/core/utilities/typedefs.h @@ -28,7 +28,7 @@ namespace legate { * @brief Function signature for task variants. Each task variant must be a function of this type. */ class TaskContext; -using VariantImpl = void (*)(TaskContext&); +using VariantImpl = void (*)(TaskContext); // C enum typedefs using LegateVariantCode = legate_core_variant_t; diff --git a/tests/cpp/integration/alignment_constraints.cc b/tests/cpp/integration/alignment_constraints.cc index 9007e2a501..a5e2728af6 100644 --- a/tests/cpp/integration/alignment_constraints.cc +++ b/tests/cpp/integration/alignment_constraints.cc @@ -28,16 +28,16 @@ enum TaskIDs { // Dummy task to make the runtime think the store is initialized struct Initializer : public legate::LegateTask { static const int32_t TASK_ID = INIT; - static void cpu_variant(legate::TaskContext& context) {} + static void cpu_variant(legate::TaskContext context) {} }; template struct AlignmentTester : public legate::LegateTask> { static const int32_t TASK_ID = ALIGNMENT_TESTER + DIM; - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto& outputs = context.outputs(); - auto shape = outputs.at(0).shape(); + auto outputs = context.outputs(); + auto shape = outputs.at(0).shape(); for (auto& output : outputs) EXPECT_EQ(shape, output.shape()); } }; @@ -45,7 +45,7 @@ struct AlignmentTester : public legate::LegateTask> { template struct AlignmentBroadcastTester : public legate::LegateTask> { static const int32_t TASK_ID = ALIGNMENT_BROADCAST_TESTER + DIM; - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { auto shape1 = context.outputs().at(0).shape(); auto shape2 = context.outputs().at(1).shape(); @@ -60,7 +60,7 @@ struct AlignmentBroadcastTester : public legate::LegateTask struct TransformedTester : public legate::LegateTask> { static const int32_t TASK_ID = TRANSFORMED_TESTER + DIM; - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { auto shape1 = context.inputs().at(0).shape(); auto shape2 = context.inputs().at(1).shape(); @@ -95,13 +95,9 @@ void test_alignment() auto store3 = runtime->create_store(extents, legate::int64()); auto task = runtime->create_task(context, ALIGNMENT_TESTER + extents.size()); - auto part1 = task.declare_partition(); - auto part2 = task.declare_partition(); - auto part3 = task.declare_partition(); - - task.add_output(store1, part1); - task.add_output(store2, part2); - task.add_output(store3, part3); + auto part1 = task.add_output(store1); + auto part2 = task.add_output(store2); + auto part3 = task.add_output(store3); task.add_constraint(legate::align(part1, part2)); task.add_constraint(legate::align(part2, part3)); @@ -124,11 +120,8 @@ void test_alignment_and_broadcast() auto store2 = runtime->create_store(extents, legate::int64()); auto task = runtime->create_task(context, ALIGNMENT_BROADCAST_TESTER + extents.size()); - auto part1 = task.declare_partition(); - auto part2 = task.declare_partition(); - - task.add_output(store1, part1); - task.add_output(store2, part2); + auto part1 = task.add_output(store1); + auto part2 = task.add_output(store2); task.add_scalar_arg(legate::Scalar(extents[0])); @@ -149,8 +142,7 @@ void initialize(legate::LogicalStore store) auto context = runtime->find_library(library_name); auto task = runtime->create_task(context, INIT); - auto part = task.declare_partition(); - task.add_output(store, part); + task.add_output(store); runtime->submit(std::move(task)); } @@ -162,11 +154,8 @@ void test_alignment_transformed() auto launch_tester = [&](auto store1, auto store2) { auto task = runtime->create_task(context, TRANSFORMED_TESTER + store1.dim()); - auto part1 = task.declare_partition(); - auto part2 = task.declare_partition(); - - task.add_input(store1, part1); - task.add_input(store2, part2); + auto part1 = task.add_input(store1); + auto part2 = task.add_input(store2); task.add_constraint(legate::align(part1, part2)); @@ -204,10 +193,8 @@ void test_invalid_alignment() auto store2 = runtime->create_store({9}, legate::int64()); auto task = runtime->create_task(context, TRANSFORMED_TESTER + store1.dim()); - auto part1 = task.declare_partition(); - auto part2 = task.declare_partition(); - task.add_output(store1, part1); - task.add_output(store2, part2); + auto part1 = task.add_output(store1); + auto part2 = task.add_output(store2); task.add_constraint(legate::align(part1, part2)); EXPECT_THROW(runtime->submit(std::move(task)), std::invalid_argument); } diff --git a/tests/cpp/integration/broadcast_constraints.cc b/tests/cpp/integration/broadcast_constraints.cc index 209a0ffaa2..b2f97987a4 100644 --- a/tests/cpp/integration/broadcast_constraints.cc +++ b/tests/cpp/integration/broadcast_constraints.cc @@ -25,12 +25,12 @@ constexpr int32_t TESTER = 0; constexpr int32_t INITIALIZER = 1; struct TesterTask : public legate::LegateTask { - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto extent = context.scalars().at(0).value(); - auto dims = context.scalars().at(1).values(); - auto is_read = context.scalars().at(2).value(); - auto shape = is_read ? context.inputs().at(0).shape<3>() : context.outputs().at(0).shape<3>(); + auto extent = context.scalar(0).value(); + auto dims = context.scalar(1).values(); + auto is_read = context.scalar(2).value(); + auto shape = is_read ? context.input(0).shape<3>() : context.output(0).shape<3>(); for (auto dim : dims) { EXPECT_EQ(shape.lo[dim], 0); @@ -40,7 +40,7 @@ struct TesterTask : public legate::LegateTask { }; struct Initializer : public legate::LegateTask { - static void cpu_variant(legate::TaskContext& context) {} + static void cpu_variant(legate::TaskContext context) {} }; void prepare() @@ -61,8 +61,7 @@ void test_normal_store() for (auto dim : dims) extents[dim] = EXT_LARGE; auto store = runtime->create_store(extents, legate::int64()); auto task = runtime->create_task(context, TESTER); - auto part = task.declare_partition(); - task.add_output(store, part); + auto part = task.add_output(store); task.add_scalar_arg(legate::Scalar(EXT_LARGE)); task.add_scalar_arg(legate::Scalar(dims)); task.add_scalar_arg(legate::Scalar(false)); @@ -86,8 +85,7 @@ void test_promoted_store() auto initialize = [&](auto store) { auto task = runtime->create_task(context, INITIALIZER); - auto part = task.declare_partition(); - task.add_output(store, part); + task.add_output(store); runtime->submit(std::move(task)); }; @@ -98,8 +96,7 @@ void test_promoted_store() initialize(store); auto task = runtime->create_task(context, TESTER); - auto part = task.declare_partition(); - task.add_input(store.promote(2, EXT_LARGE), part); + auto part = task.add_input(store.promote(2, EXT_LARGE)); task.add_scalar_arg(legate::Scalar(EXT_LARGE)); task.add_scalar_arg(legate::Scalar(std::vector{dim})); task.add_scalar_arg(legate::Scalar(true)); @@ -118,8 +115,7 @@ void test_invalid_broadcast() auto task = runtime->create_task(context, INITIALIZER); auto store = runtime->create_store({10}, legate::int64()); - auto part = task.declare_partition(); - task.add_output(store, part); + auto part = task.add_output(store); task.add_constraint(legate::broadcast(part, {1})); EXPECT_THROW(runtime->submit(std::move(task)), std::invalid_argument); } diff --git a/tests/cpp/integration/copy_gather.cc b/tests/cpp/integration/copy_gather.cc index 5e009f83f4..8ae40f5d22 100644 --- a/tests/cpp/integration/copy_gather.cc +++ b/tests/cpp/integration/copy_gather.cc @@ -26,13 +26,13 @@ template struct CheckGatherTask : public legate::LegateTask> { struct CheckGatherTaskBody { template - void operator()(legate::TaskContext& context) + void operator()(legate::TaskContext context) { using VAL = legate::legate_type_of; - auto& src_store = context.inputs().at(0); - auto& tgt_store = context.inputs().at(1); - auto& ind_store = context.inputs().at(2); + auto src_store = context.input(0).data(); + auto tgt_store = context.input(1).data(); + auto ind_store = context.input(2).data(); auto ind_shape = ind_store.shape(); if (ind_shape.empty()) return; @@ -50,9 +50,9 @@ struct CheckGatherTask : public legate::LegateTask> { struct CheckGatherScatterTaskBody { template - void operator()(legate::TaskContext& context) + void operator()(legate::TaskContext context) { using VAL = legate::legate_type_of; - auto& src_store = context.inputs().at(0); - auto& tgt_store = context.inputs().at(1); - auto& src_ind_store = context.inputs().at(2); - auto& tgt_ind_store = context.inputs().at(3); - auto init = context.scalars().at(0).value(); + auto src_store = context.input(0).data(); + auto tgt_store = context.input(1).data(); + auto src_ind_store = context.input(2).data(); + auto tgt_ind_store = context.input(3).data(); + auto init = context.scalar(0).value(); auto src_shape = src_store.shape(); auto tgt_shape = tgt_store.shape(); @@ -68,9 +68,9 @@ struct CheckGatherScatterTask static const int32_t TASK_ID = CHECK_GATHER_SCATTER_TASK + SRC_DIM * TEST_MAX_DIM * TEST_MAX_DIM + IND_DIM * TEST_MAX_DIM + TGT_DIM; - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto type_code = context.inputs().at(0).type().code(); + auto type_code = context.input(0).type().code(); type_dispatch_for_test(type_code, CheckGatherScatterTaskBody{}, context); } }; diff --git a/tests/cpp/integration/copy_normal.cc b/tests/cpp/integration/copy_normal.cc index 4ecdfe9cbe..ced41207bd 100644 --- a/tests/cpp/integration/copy_normal.cc +++ b/tests/cpp/integration/copy_normal.cc @@ -41,11 +41,11 @@ struct CheckCopyTask : public legate::LegateTask> { }; static const int32_t TASK_ID = CHECK_COPY_TASK + DIM; - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto& source = context.inputs().at(0); - auto& target = context.inputs().at(1); - auto shape = source.shape(); + auto source = context.input(0).data(); + auto target = context.input(1).data(); + auto shape = source.shape(); if (shape.empty()) return; @@ -61,7 +61,7 @@ struct CheckCopyReductionTask : public legate::LegateTask::value, int> = 0> void operator()(legate::Store& source, legate::Store& target, - legate::Scalar& seed, + const legate::Scalar& seed, legate::Rect& shape) { using VAL = legate::legate_type_of; @@ -76,7 +76,7 @@ struct CheckCopyReductionTask : public legate::LegateTask::value, int> = 0> void operator()(legate::Store& source, legate::Store& target, - legate::Scalar& seed, + const legate::Scalar& seed, legate::Rect& shape) { assert(false); @@ -84,12 +84,12 @@ struct CheckCopyReductionTask : public legate::LegateTask(); + legate::Store source = context.input(0); + legate::Store target = context.input(1); + auto& seed = context.scalar(0); + auto shape = target.shape(); if (shape.empty()) return; diff --git a/tests/cpp/integration/copy_scatter.cc b/tests/cpp/integration/copy_scatter.cc index 0843fda228..6ea7974bbb 100644 --- a/tests/cpp/integration/copy_scatter.cc +++ b/tests/cpp/integration/copy_scatter.cc @@ -26,14 +26,14 @@ template struct CheckScatterTask : public legate::LegateTask> { struct CheckScatterTaskBody { template - void operator()(legate::TaskContext& context) + void operator()(legate::TaskContext context) { using VAL = legate::legate_type_of; - auto& src_store = context.inputs().at(0); - auto& tgt_store = context.inputs().at(1); - auto& ind_store = context.inputs().at(2); - auto init = context.scalars().at(0).value(); + auto src_store = context.input(0).data(); + auto tgt_store = context.input(1).data(); + auto ind_store = context.input(2).data(); + auto init = context.scalar(0).value(); auto ind_shape = ind_store.shape(); if (ind_shape.empty()) return; @@ -64,9 +64,9 @@ struct CheckScatterTask : public legate::LegateTask> { }; static const int32_t TASK_ID = FILL_TASK + DIM; - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto& output = context.outputs().at(0); - auto shape = output.shape(); - auto& seed = context.scalars().at(0); + auto output = context.output(0).data(); + auto shape = output.shape(); + auto seed = context.scalar(0); if (shape.empty()) return; @@ -88,11 +88,11 @@ legate::Point delinearize(size_t index, template struct FillIndirectTask : public legate::LegateTask> { static const int32_t TASK_ID = FILL_INDIRECT_TASK + IND_DIM * TEST_MAX_DIM + DATA_DIM; - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto& output = context.outputs().at(0); + auto output = context.output(0).data(); auto ind_shape = output.shape(); - auto data_shape = context.scalars().at(0).value>(); + auto data_shape = context.scalar(0).value>(); size_t data_vol = data_shape.volume(); size_t ind_vol = ind_shape.volume(); diff --git a/tests/cpp/integration/cpu_communicator.cc b/tests/cpp/integration/cpu_communicator.cc index 5f17b6f36b..bfe9d84ce2 100644 --- a/tests/cpp/integration/cpu_communicator.cc +++ b/tests/cpp/integration/cpu_communicator.cc @@ -26,7 +26,7 @@ enum TaskIDs { constexpr size_t SIZE = 10; struct CPUCommunicatorTester : public legate::LegateTask { - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { EXPECT_TRUE((context.is_single_task() && context.communicators().empty()) || context.communicators().size() == 1); diff --git a/tests/cpp/integration/exception.cc b/tests/cpp/integration/exception.cc index 692d9fd4e6..b556177f4b 100644 --- a/tests/cpp/integration/exception.cc +++ b/tests/cpp/integration/exception.cc @@ -28,11 +28,11 @@ const char* EXN_MSG = "Exception Test"; constexpr size_t SIZE = 10; struct ExceptionTask : public legate::LegateTask { - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { EXPECT_TRUE(context.can_raise_exception()); - auto raise = context.scalars().at(0).value(); - auto index = context.scalars().at(1).value(); + auto raise = context.scalar(0).value(); + auto index = context.scalar(1).value(); // Make sure only some of the point tasks raise an exception if (raise && (context.is_single_task() || context.get_task_index()[0] == 0)) { if (context.is_single_task()) diff --git a/tests/cpp/integration/fill.cc b/tests/cpp/integration/fill.cc index e409143215..4c47e3f2e7 100644 --- a/tests/cpp/integration/fill.cc +++ b/tests/cpp/integration/fill.cc @@ -28,13 +28,13 @@ enum TaskIDs { template struct CheckTask : public legate::LegateTask> { static const int32_t TASK_ID = CHECK_TASK + DIM; - static void cpu_variant(legate::TaskContext& context); + static void cpu_variant(legate::TaskContext context); }; template struct CheckSliceTask : public legate::LegateTask> { static const int32_t TASK_ID = CHECK_SLICE_TASK + DIM; - static void cpu_variant(legate::TaskContext& context); + static void cpu_variant(legate::TaskContext context); }; void register_tasks() @@ -50,11 +50,11 @@ void register_tasks() } template -/*static*/ void CheckTask::cpu_variant(legate::TaskContext& context) +/*static*/ void CheckTask::cpu_variant(legate::TaskContext context) { - auto& input = context.inputs().at(0); + auto input = context.input(0).data(); auto shape = input.shape(); - int64_t value = context.scalars().at(0).value(); + int64_t value = context.scalar(0).value(); if (shape.empty()) return; @@ -63,13 +63,13 @@ template } template -/*static*/ void CheckSliceTask::cpu_variant(legate::TaskContext& context) +/*static*/ void CheckSliceTask::cpu_variant(legate::TaskContext context) { - auto& input = context.inputs().at(0); + auto input = context.input(0).data(); auto shape = input.shape(); - int64_t value_in_slice = context.scalars().at(0).value(); - int64_t value_outside_slice = context.scalars().at(1).value(); - int64_t offset = context.scalars().at(2).value(); + int64_t value_in_slice = context.scalar(0).value(); + int64_t value_outside_slice = context.scalar(1).value(); + int64_t offset = context.scalar(2).value(); if (shape.empty()) return; diff --git a/tests/cpp/integration/image_constraints.cc b/tests/cpp/integration/image_constraints.cc index 29c2fe6543..cb2e831de7 100644 --- a/tests/cpp/integration/image_constraints.cc +++ b/tests/cpp/integration/image_constraints.cc @@ -43,7 +43,7 @@ template struct InitializeFunction : public legate::LegateTask> { struct InitializeRects { template - void operator()(legate::Store& output, const legate::Scalar& extents_scalar) + void operator()(legate::Store& output, const legate::Scalar& extents_scalar, bool ascending) { auto shape = output.shape(); auto extents = extents_scalar.value>(); @@ -58,21 +58,23 @@ struct InitializeFunction : public legate::LegateTask= extents[dim]) return false; return true; }; - size_t idx = 0; + int64_t idx = ascending ? 0 : vol - 1; + int64_t diff = ascending ? 1 : -1; for (legate::PointInRectIterator it(shape); it.valid(); ++it) { - auto lo = delinearize(idx++ * tgt_vol / vol, extents); + auto lo = delinearize(idx * tgt_vol / vol, extents); auto hi = lo + legate::Point::ONES(); if (in_bounds(hi)) acc[*it] = legate::Rect(lo, hi); else acc[*it] = legate::Rect(lo, lo); + idx += diff; } } }; struct InitializePoints { template - void operator()(legate::Store& output, const legate::Scalar& extents_scalar) + void operator()(legate::Store& output, const legate::Scalar& extents_scalar, bool ascending) { auto shape = output.shape(); auto extents = extents_scalar.value>(); @@ -82,26 +84,31 @@ struct InitializeFunction : public legate::LegateTask it(shape); it.valid(); ++it) { - auto p = delinearize(idx++ * tgt_vol / vol, extents); + auto p = delinearize(idx * tgt_vol / vol, extents); acc[*it] = p; + idx += diff; } } }; static const int32_t TASK_ID = INIT_FUNC + static_cast(RECT) * TEST_MAX_DIM + DIM; - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto& output = context.outputs().at(0); - auto& extents = context.scalars().at(0); + auto output = context.output(0).data(); + auto extents = context.scalar(0); + auto ascending = context.scalar(1).value(); if constexpr (RECT) { const auto& rect_type = output.type().as_struct_type(); const auto& point_type = rect_type.field_type(0).as_fixed_array_type(); - legate::dim_dispatch(point_type.num_elements(), InitializeRects{}, output, extents); + legate::dim_dispatch( + point_type.num_elements(), InitializeRects{}, output, extents, ascending); } else { const auto& point_type = output.type().as_fixed_array_type(); - legate::dim_dispatch(point_type.num_elements(), InitializePoints{}, output, extents); + legate::dim_dispatch( + point_type.num_elements(), InitializePoints{}, output, extents, ascending); } } }; @@ -139,12 +146,11 @@ struct ImageTester : public legate::LegateTask> { } } }; - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto& func = context.inputs().at(0); - - auto range = context.inputs().at(1).domain(); - EXPECT_FALSE(!context.is_single_task() && range.get_volume() > 1 && range.dense()); + auto func = context.input(0).data(); + auto range = context.input(1).domain(); + EXPECT_TRUE(context.is_single_task() || range.get_volume() <= 1 || !range.dense()); if constexpr (RECT) { const auto& rect_type = func.type().as_struct_type(); @@ -154,6 +160,14 @@ struct ImageTester : public legate::LegateTask> { const auto& point_type = func.type().as_fixed_array_type(); legate::dim_dispatch(point_type.num_elements(), CheckPoints{}, func, range); } + + if (context.get_task_index() == context.get_launch_domain().lo()) { + logger.debug() << "== Image received in task 0 =="; + for (legate::Domain::DomainPointIterator it(range); it; ++it) { + logger.debug() << " " << *it; + } + logger.debug() << "=============================="; + } } }; @@ -175,7 +189,9 @@ void prepare() ImageTester<3, false>::register_variants(context); } -void initialize_function(legate::LogicalStore func, const std::vector range_extents) +void initialize_function(legate::LogicalStore func, + const std::vector range_extents, + bool ascending) { auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(library_name); @@ -186,6 +202,7 @@ void initialize_function(legate::LogicalStore func, const std::vector ra auto part = task.declare_partition(); task.add_output(func, part); task.add_scalar_arg(range_extents); + task.add_scalar_arg(ascending); task.add_constraint(legate::broadcast(part, legate::from_range(func.dim()))); runtime->submit(std::move(task)); @@ -226,9 +243,15 @@ void test_image(const ImageTestSpec& spec) auto func = runtime->create_store(spec.domain_extents, std::move(image_type)); auto range = runtime->create_store(spec.range_extents, legate::int64()); - initialize_function(func, spec.range_extents); + initialize_function(func, spec.range_extents, true); runtime->issue_fill(range, legate::Scalar(int64_t(1234))); check_image(func, range); + runtime->issue_execution_fence(); + check_image(func, range); + initialize_function(func, spec.range_extents, false); + check_image(func, range); + runtime->issue_execution_fence(); + check_image(func, range); } void test_invalid() diff --git a/tests/cpp/integration/inline_map.cc b/tests/cpp/integration/inline_map.cc index f22170ff69..d6bac7555a 100644 --- a/tests/cpp/integration/inline_map.cc +++ b/tests/cpp/integration/inline_map.cc @@ -25,11 +25,11 @@ enum TaskOpCode { struct AdderTask : public legate::LegateTask { static const int32_t TASK_ID = ADDER; - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto& output = context.outputs()[0]; - auto shape = output.shape<1>(); - auto acc = output.read_write_accessor(shape); + auto output = context.output(0).data(); + auto shape = output.shape<1>(); + auto acc = output.read_write_accessor(shape); for (legate::PointInRectIterator<1> it(shape); it.valid(); ++it) acc[*it] += 1; } }; diff --git a/tests/cpp/integration/inout.cc b/tests/cpp/integration/inout.cc index b9a6783230..991f411171 100644 --- a/tests/cpp/integration/inout.cc +++ b/tests/cpp/integration/inout.cc @@ -22,12 +22,12 @@ void test_inout() auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(task::simple::library_name); - auto store = runtime->create_store({10, 10}, legate::int64()); - runtime->issue_fill(store, legate::Scalar(int64_t(0))); + auto array = runtime->create_array({10, 10}, legate::int64()); + runtime->issue_fill(array.data(), legate::Scalar(int64_t(0))); auto task = runtime->create_task(context, task::simple::HELLO); - task.add_input(store, task.find_or_declare_partition(store)); - task.add_output(store, task.find_or_declare_partition(store)); + task.add_input(array, task.find_or_declare_partition(array)); + task.add_output(array, task.find_or_declare_partition(array)); runtime->submit(std::move(task)); } diff --git a/tests/cpp/integration/machine_scope.cc b/tests/cpp/integration/machine_scope.cc index 6e77c86071..7414ab4d8b 100644 --- a/tests/cpp/integration/machine_scope.cc +++ b/tests/cpp/integration/machine_scope.cc @@ -24,7 +24,7 @@ enum TaskIDs { CPU_VARIANT = 1, }; -void validate(legate::TaskContext& context) +void validate(legate::TaskContext context) { if (context.is_single_task()) return; @@ -36,18 +36,18 @@ void validate(legate::TaskContext& context) struct MultiVariantTask : public legate::LegateTask { static const int32_t TASK_ID = MULTI_VARIANT; - static void cpu_variant(legate::TaskContext& context) { validate(context); } + static void cpu_variant(legate::TaskContext context) { validate(context); } #ifdef LEGATE_USE_OPENMP - static void omp_variant(legate::TaskContext& context) { validate(context); } + static void omp_variant(legate::TaskContext context) { validate(context); } #endif #ifdef LEGATE_USE_CUDA - static void gpu_variant(legate::TaskContext& context) { validate(context); } + static void gpu_variant(legate::TaskContext context) { validate(context); } #endif }; struct CpuVariantOnlyTask : public legate::LegateTask { static const int32_t TASK_ID = CPU_VARIANT; - static void cpu_variant(legate::TaskContext& context) { validate(context); } + static void cpu_variant(legate::TaskContext context) { validate(context); } }; void register_tasks() diff --git a/tests/cpp/integration/nccl.cu b/tests/cpp/integration/nccl.cu index 9bb4444b98..8a7dfd813f 100644 --- a/tests/cpp/integration/nccl.cu +++ b/tests/cpp/integration/nccl.cu @@ -29,7 +29,7 @@ enum TaskIDs { constexpr size_t SIZE = 10; struct NCCLTester : public legate::LegateTask { - static void gpu_variant(legate::TaskContext& context) + static void gpu_variant(legate::TaskContext context) { EXPECT_TRUE((context.is_single_task() && context.communicators().empty()) || context.communicators().size() == 1); diff --git a/tests/cpp/integration/provenance.cc b/tests/cpp/integration/provenance.cc index 0f436d006b..4fcd2ab84c 100644 --- a/tests/cpp/integration/provenance.cc +++ b/tests/cpp/integration/provenance.cc @@ -25,7 +25,7 @@ enum TaskIDs { struct ProvenanceTask : public legate::LegateTask { static const int32_t TASK_ID = PROVENANCE; - static void cpu_variant(legate::TaskContext& context); + static void cpu_variant(legate::TaskContext context); }; void register_tasks() @@ -35,9 +35,9 @@ void register_tasks() ProvenanceTask::register_variants(library); } -/*static*/ void ProvenanceTask::cpu_variant(legate::TaskContext& context) +/*static*/ void ProvenanceTask::cpu_variant(legate::TaskContext context) { - std::string scalar = context.scalars()[0].value(); + std::string scalar = context.scalar(0).value(); auto provenance = context.get_provenance(); EXPECT_TRUE(provenance.find(scalar) != std::string::npos); } diff --git a/tests/cpp/integration/tasks/task_region_manager.h b/tests/cpp/integration/tasks/task_region_manager.h index 53f77279e6..c6a3b6d1d9 100644 --- a/tests/cpp/integration/tasks/task_region_manager.h +++ b/tests/cpp/integration/tasks/task_region_manager.h @@ -29,7 +29,7 @@ void register_tasks(); struct TesterTask : public legate::LegateTask { static const int32_t TASK_ID = 0; - static void cpu_variant(legate::TaskContext& context) {} + static void cpu_variant(legate::TaskContext context) {} }; } // namespace region_manager diff --git a/tests/cpp/integration/tasks/task_simple.cc b/tests/cpp/integration/tasks/task_simple.cc index cb147acd4f..e35bb66f49 100644 --- a/tests/cpp/integration/tasks/task_simple.cc +++ b/tests/cpp/integration/tasks/task_simple.cc @@ -27,10 +27,10 @@ void register_tasks() ReducerTask::register_variants(context); } -/*static*/ void HelloTask::cpu_variant(legate::TaskContext& context) +/*static*/ void HelloTask::cpu_variant(legate::TaskContext context) { - auto& output = context.outputs()[0]; - auto shape = output.shape<2>(); + auto output = context.output(0).data(); + auto shape = output.shape<2>(); if (shape.empty()) return; @@ -39,10 +39,10 @@ void register_tasks() acc[*it] = (*it)[0] + (*it)[1] * 1000; } -/*static*/ void WriterTask::cpu_variant(legate::TaskContext& context) +/*static*/ void WriterTask::cpu_variant(legate::TaskContext context) { - auto& output1 = context.outputs()[0]; - auto& output2 = context.outputs()[1]; + auto output1 = context.output(0).data(); + auto output2 = context.output(1).data(); auto acc1 = output1.write_accessor(); auto acc2 = output2.write_accessor(); @@ -51,10 +51,10 @@ void register_tasks() acc2[{0, 0, 0}] = 20; } -/*static*/ void ReducerTask::cpu_variant(legate::TaskContext& context) +/*static*/ void ReducerTask::cpu_variant(legate::TaskContext context) { - auto& red1 = context.reductions()[0]; - auto& red2 = context.reductions()[1]; + auto red1 = context.reduction(0).data(); + auto red2 = context.reduction(1).data(); auto acc1 = red1.reduce_accessor, true, 2>(); auto acc2 = red2.reduce_accessor, true, 3>(); diff --git a/tests/cpp/integration/tasks/task_simple.h b/tests/cpp/integration/tasks/task_simple.h index 7a0cb26b37..d7870c7271 100644 --- a/tests/cpp/integration/tasks/task_simple.h +++ b/tests/cpp/integration/tasks/task_simple.h @@ -33,17 +33,17 @@ void register_tasks(); struct HelloTask : public legate::LegateTask { static const int32_t TASK_ID = HELLO; - static void cpu_variant(legate::TaskContext& context); + static void cpu_variant(legate::TaskContext context); }; struct WriterTask : public legate::LegateTask { static const int32_t TASK_ID = WRITER; - static void cpu_variant(legate::TaskContext& context); + static void cpu_variant(legate::TaskContext context); }; struct ReducerTask : public legate::LegateTask { static const int32_t TASK_ID = REDUCER; - static void cpu_variant(legate::TaskContext& context); + static void cpu_variant(legate::TaskContext context); }; } // namespace simple diff --git a/tests/cpp/integration/tree_reduce.cc b/tests/cpp/integration/tree_reduce.cc index ad1653d616..d9e8301975 100644 --- a/tests/cpp/integration/tree_reduce.cc +++ b/tests/cpp/integration/tree_reduce.cc @@ -33,26 +33,26 @@ enum TaskIDs { struct ProduceNormalTask : public legate::LegateTask { static const int32_t TASK_ID = TASK_PRODUCE_NORMAL; - static void cpu_variant(legate::TaskContext& context) {} + static void cpu_variant(legate::TaskContext context) {} }; struct ProduceUnboundTask : public legate::LegateTask { static const int32_t TASK_ID = TASK_PRODUCE_UNBOUND; - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto& output = context.outputs().at(0); - auto size = context.get_task_index()[0] + 1; - auto buffer = output.create_output_buffer(legate::Point<1>(size), true /*bind*/); + auto output = context.output(0).data(); + auto size = context.get_task_index()[0] + 1; + auto buffer = output.create_output_buffer(legate::Point<1>(size), true /*bind*/); for (int64_t idx = 0; idx < size; ++idx) buffer[idx] = size; } }; struct ReduceNormalTask : public legate::LegateTask { static const int32_t TASK_ID = TASK_REDUCE_NORMAL; - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto& inputs = context.inputs(); - auto& output = context.outputs().at(0); + auto inputs = context.inputs(); + auto output = context.output(0).data(); for (auto& input : inputs) { auto shape = input.shape<1>(); EXPECT_TRUE(shape.empty() || shape.volume() == TILE_SIZE); @@ -63,10 +63,10 @@ struct ReduceNormalTask : public legate::LegateTask { struct ReduceUnboundTask : public legate::LegateTask { static const int32_t TASK_ID = TASK_REDUCE_UNBOUND; - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto& inputs = context.inputs(); - auto& output = context.outputs().at(0); + auto inputs = context.inputs(); + auto output = context.output(0).data(); uint32_t expected = 1; for (auto& input : inputs) { auto shape = input.shape<1>(); diff --git a/tests/cpp/integration/tree_reduce_unique.cc b/tests/cpp/integration/tree_reduce_unique.cc index ed44a73700..48260e7781 100644 --- a/tests/cpp/integration/tree_reduce_unique.cc +++ b/tests/cpp/integration/tree_reduce_unique.cc @@ -28,25 +28,25 @@ enum TaskIDs { TASK_FILL = 1, TASK_UNIQUE, TASK_UNIQUE_REDUCE, TASK_CHECK }; struct FillTask : public legate::LegateTask { static const int32_t TASK_ID = TASK_FILL; - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto& output = context.outputs().at(0); - auto rect = output.shape<1>(); - auto volume = rect.volume(); - auto out = output.write_accessor(rect); + auto output = context.output(0).data(); + auto rect = output.shape<1>(); + auto volume = rect.volume(); + auto out = output.write_accessor(rect); for (size_t idx = 0; idx < volume; ++idx) { out[idx] = idx / 2; } } }; struct UniqueTask : public legate::LegateTask { static const int32_t TASK_ID = TASK_UNIQUE; - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto& input = context.inputs().at(0); - auto& output = context.outputs().at(0); - auto rect = input.shape<1>(); - auto volume = rect.volume(); - auto in = input.read_accessor(rect); + auto input = context.input(0).data(); + auto output = context.output(0).data(); + auto rect = input.shape<1>(); + auto volume = rect.volume(); + auto in = input.read_accessor(rect); std::set dedup_set; for (size_t idx = 0; idx < volume; ++idx) dedup_set.insert(in[idx]); @@ -58,35 +58,34 @@ struct UniqueTask : public legate::LegateTask { struct UniqueReduceTask : public legate::LegateTask { static const int32_t TASK_ID = TASK_UNIQUE_REDUCE; - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto& output = context.outputs().at(0); - std::vector, Legion::Rect<1>>> inputs; + auto output = context.output(0).data(); + std::vector, legate::Rect<1>>> inputs; for (auto& input_arr : context.inputs()) { auto shape = input_arr.shape<1>(); - auto acc = input_arr.read_accessor(shape); + auto acc = input_arr.data().read_accessor(shape); inputs.push_back(std::make_pair(acc, shape)); } std::set dedup_set; for (auto& pair : inputs) { auto& input = pair.first; auto& shape = pair.second; - for (Legion::coord_t idx = shape.lo[0]; idx <= shape.hi[0]; ++idx) - dedup_set.insert(input[idx]); + for (auto idx = shape.lo[0]; idx <= shape.hi[0]; ++idx) dedup_set.insert(input[idx]); } size_t size = dedup_set.size(); size_t pos = 0; - auto result = output.create_output_buffer(Legion::Point<1>(size), true); + auto result = output.create_output_buffer(legate::Point<1>(size), true); for (auto e : dedup_set) result[pos++] = e; } }; struct CheckTask : public legate::LegateTask { static const int32_t TASK_ID = TASK_CHECK; - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { - auto& input = context.inputs().at(0); + auto input = context.input(0).data(); auto rect = input.shape<1>(); auto volume = rect.volume(); auto in = input.read_accessor(rect); diff --git a/tests/cpp/integration/weighted.cc b/tests/cpp/integration/weighted.cc index c6acdaf3a8..5c6f5b1d30 100644 --- a/tests/cpp/integration/weighted.cc +++ b/tests/cpp/integration/weighted.cc @@ -26,24 +26,24 @@ enum TaskIDs { constexpr uint32_t NUM_TASKS = 4; struct Initializer : public legate::LegateTask { - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { auto task_idx = context.get_task_index()[0]; - auto& outputs = context.outputs(); + auto outputs = context.outputs(); for (uint32_t idx = 0; idx < outputs.size(); ++idx) { - auto& output = outputs.at(idx); + auto output = outputs.at(idx).data(); output.create_output_buffer(legate::Point<1>(task_idx + 10 * (idx + 1)), true); } } }; struct Tester : public legate::LegateTask { - static void cpu_variant(legate::TaskContext& context) + static void cpu_variant(legate::TaskContext context) { EXPECT_FALSE(context.is_single_task()); EXPECT_EQ(context.get_launch_domain().get_volume(), NUM_TASKS); auto task_idx = context.get_task_index()[0]; - auto& outputs = context.outputs(); + auto outputs = context.outputs(); for (uint32_t idx = 0; idx < outputs.size(); ++idx) { auto volume = outputs.at(idx).shape<1>().volume(); EXPECT_EQ(volume, task_idx + 10 * (idx + 1)); @@ -78,11 +78,9 @@ void check(legate::Runtime* runtime, auto task = runtime->create_task(library, CHECK); for (auto& input : inputs) { - auto part_in = task.declare_partition(); + auto part_in = task.add_input(input); auto output = runtime->create_store(input.extents(), input.type()); - auto part_out = task.declare_partition(); - task.add_input(input, part_in); - task.add_output(output, part_out); + auto part_out = task.add_output(output); task.add_constraint(legate::align(part_in, part_out)); } diff --git a/tests/cpp/run.sh b/tests/cpp/run.sh index 6e012b101e..d5f8f86f6e 100755 --- a/tests/cpp/run.sh +++ b/tests/cpp/run.sh @@ -3,7 +3,7 @@ if [ $# -eq 0 ] then REALM_BACKTRACE=1 LEGATE_TEST=1 python run.py -elif [ $# -eq 1 ] && [ "$1" = "ctest" ] +elif [ $# -ge 1 ] && [ "$1" = "ctest" ] then echo "Using ctest" cd build diff --git a/tests/cpp/unit/logical_array_creation.cc b/tests/cpp/unit/logical_array_creation.cc new file mode 100644 index 0000000000..ec609d5890 --- /dev/null +++ b/tests/cpp/unit/logical_array_creation.cc @@ -0,0 +1,234 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include + +#include "legate.h" + +namespace array_test { + +void test_primitive_array(bool nullable) +{ + auto runtime = legate::Runtime::get_runtime(); + // Bound + { + auto array = runtime->create_array({4, 4}, legate::int64(), nullable); + EXPECT_FALSE(array.unbound()); + EXPECT_EQ(array.dim(), 2); + EXPECT_EQ(array.extents(), (std::vector{4, 4})); + EXPECT_EQ(array.volume(), 16); + EXPECT_EQ(array.type(), legate::int64()); + EXPECT_EQ(array.nullable(), nullable); + EXPECT_EQ(array.num_children(), 0); + + auto store = array.data(); + EXPECT_FALSE(store.unbound()); + EXPECT_EQ(store.dim(), 2); + EXPECT_EQ(store.extents(), (std::vector{4, 4})); + EXPECT_EQ(store.volume(), 16); + EXPECT_EQ(store.type(), legate::int64()); + + if (!nullable) { EXPECT_THROW(array.null_mask(), std::invalid_argument); } + EXPECT_THROW(array.child(0), std::invalid_argument); + } + // Unbound + { + auto array = runtime->create_array(legate::int64(), 3, nullable); + EXPECT_TRUE(array.unbound()); + EXPECT_EQ(array.dim(), 3); + EXPECT_THROW(array.extents(), std::invalid_argument); + EXPECT_THROW(array.volume(), std::invalid_argument); + EXPECT_EQ(array.type(), legate::int64()); + EXPECT_EQ(array.nullable(), nullable); + EXPECT_EQ(array.num_children(), 0); + + auto store = array.data(); + EXPECT_TRUE(store.unbound()); + EXPECT_EQ(store.dim(), 3); + EXPECT_THROW(store.extents(), std::invalid_argument); + EXPECT_THROW(store.volume(), std::invalid_argument); + EXPECT_EQ(store.type(), legate::int64()); + + if (!nullable) { EXPECT_THROW(array.null_mask(), std::invalid_argument); } + EXPECT_THROW(array.child(0), std::invalid_argument); + } +} + +void test_list_array(bool nullable) +{ + auto runtime = legate::Runtime::get_runtime(); + auto arr_type = legate::list_type(legate::int64()).as_list_type(); + // Bound descriptor + { + auto array = runtime->create_array({7}, arr_type, nullable); + // List arrays are unbound even with the fixed extents + EXPECT_TRUE(array.unbound()); + EXPECT_EQ(array.dim(), 1); + EXPECT_EQ(array.extents(), (std::vector{7})); + EXPECT_EQ(array.volume(), 7); + EXPECT_EQ(array.type(), arr_type); + EXPECT_EQ(array.nullable(), nullable); + EXPECT_EQ(array.num_children(), 2); + + EXPECT_THROW(array.data(), std::invalid_argument); + if (!nullable) { EXPECT_THROW(array.null_mask(), std::invalid_argument); } + + auto list_array = array.as_list_array(); + // Sub-arrays of list arrays can be retrieved only when they are initialized first + EXPECT_THROW(list_array.descriptor(), std::invalid_argument); + EXPECT_THROW(list_array.vardata(), std::invalid_argument); + EXPECT_THROW(list_array.child(2), std::invalid_argument); + } + // Unbound + { + auto array = runtime->create_array(arr_type, 1, nullable); + EXPECT_TRUE(array.unbound()); + EXPECT_EQ(array.dim(), 1); + EXPECT_EQ(array.type(), arr_type); + EXPECT_EQ(array.nullable(), nullable); + EXPECT_EQ(array.num_children(), 2); + + EXPECT_THROW(array.data(), std::invalid_argument); + if (!nullable) { EXPECT_THROW(array.null_mask(), std::invalid_argument); } + + auto list_array = array.as_list_array(); + // Sub-arrays of list arrays can be retrieved only when they are initialized first + EXPECT_THROW(list_array.descriptor(), std::invalid_argument); + EXPECT_THROW(list_array.vardata(), std::invalid_argument); + EXPECT_THROW(list_array.child(2), std::invalid_argument); + } +} + +void test_struct_array(bool nullable) +{ + auto runtime = legate::Runtime::get_runtime(); + auto st_type = legate::struct_type(true, legate::uint16(), legate::int64(), legate::float32()) + .as_struct_type(); + auto num_fields = st_type.num_fields(); + // Bound + { + auto array = runtime->create_array({4, 4}, st_type, nullable); + EXPECT_FALSE(array.unbound()); + EXPECT_EQ(array.dim(), 2); + EXPECT_EQ(array.extents(), (std::vector{4, 4})); + EXPECT_EQ(array.volume(), 16); + EXPECT_EQ(array.type(), st_type); + EXPECT_EQ(array.nullable(), nullable); + EXPECT_EQ(array.num_children(), st_type.num_fields()); + + if (!nullable) { EXPECT_THROW(array.null_mask(), std::invalid_argument); } + EXPECT_THROW(array.child(num_fields), std::out_of_range); + for (uint32_t idx = 0; idx < num_fields; ++idx) { + auto field_type = st_type.field_type(idx); + auto field_subarray = array.child(idx); + + EXPECT_EQ(field_subarray.unbound(), array.unbound()); + EXPECT_EQ(field_subarray.dim(), array.dim()); + EXPECT_EQ(field_subarray.extents(), array.extents()); + EXPECT_EQ(field_subarray.volume(), array.volume()); + EXPECT_EQ(field_subarray.type(), field_type); + // There'd be only one null mask for the whole struct array + EXPECT_EQ(field_subarray.nullable(), false); + EXPECT_EQ(field_subarray.num_children(), 0); + } + } + + // Unbound + { + auto array = runtime->create_array(st_type, 3, nullable); + EXPECT_TRUE(array.unbound()); + EXPECT_EQ(array.dim(), 3); + EXPECT_EQ(array.type(), st_type); + EXPECT_EQ(array.nullable(), nullable); + EXPECT_EQ(array.num_children(), st_type.num_fields()); + + if (!nullable) { EXPECT_THROW(array.null_mask(), std::invalid_argument); } + EXPECT_THROW(array.child(0), std::invalid_argument); + } +} + +void test_isomorphic(bool nullable) +{ + auto runtime = legate::Runtime::get_runtime(); + // Bound + { + auto source = runtime->create_array({4, 4}, legate::int64(), nullable); + auto target1 = runtime->create_array_like(source); + EXPECT_EQ(source.dim(), target1.dim()); + EXPECT_EQ(source.type(), target1.type()); + EXPECT_EQ(source.extents(), target1.extents()); + EXPECT_EQ(source.volume(), target1.volume()); + EXPECT_EQ(source.unbound(), target1.unbound()); + EXPECT_EQ(source.nullable(), target1.nullable()); + + auto target2 = runtime->create_array_like(source, legate::float64()); + EXPECT_EQ(source.dim(), target2.dim()); + EXPECT_EQ(target2.type(), legate::float64()); + EXPECT_EQ(source.extents(), target2.extents()); + EXPECT_EQ(source.volume(), target2.volume()); + EXPECT_EQ(source.unbound(), target2.unbound()); + EXPECT_EQ(source.nullable(), target2.nullable()); + } + // Unbound, Same type + { + auto source = runtime->create_array(legate::int64(), 3, nullable); + auto target1 = runtime->create_array_like(source); + EXPECT_EQ(source.dim(), target1.dim()); + EXPECT_EQ(source.type(), target1.type()); + EXPECT_THROW(target1.extents(), std::invalid_argument); + EXPECT_THROW(target1.volume(), std::invalid_argument); + EXPECT_EQ(source.unbound(), target1.unbound()); + EXPECT_EQ(source.nullable(), target1.nullable()); + + auto target2 = runtime->create_array_like(source, legate::float64()); + EXPECT_EQ(source.dim(), target2.dim()); + EXPECT_EQ(target2.type(), target2.type()); + EXPECT_THROW(target2.extents(), std::invalid_argument); + EXPECT_THROW(target2.volume(), std::invalid_argument); + EXPECT_EQ(source.unbound(), target2.unbound()); + EXPECT_EQ(source.nullable(), target2.nullable()); + } +} + +void test_invalid() +{ + auto runtime = legate::Runtime::get_runtime(); + + // Multi-dimensional list/string arrays are not allowed + EXPECT_THROW(runtime->create_array({1, 2, 3}, legate::string_type()), std::invalid_argument); + EXPECT_THROW(runtime->create_array({1, 2}, legate::list_type(legate::int64())), + std::invalid_argument); + EXPECT_THROW(runtime->create_array(legate::string_type(), 2), std::invalid_argument); + EXPECT_THROW(runtime->create_array(legate::list_type(legate::int64()), 3), std::invalid_argument); +} + +TEST(LogicalArray, CreatePrimitiveNonNullable) { test_primitive_array(false); } + +TEST(LogicalArray, CreatePrimitiveNullable) { test_primitive_array(true); } + +TEST(LogicalArray, CreateListNonNullable) { test_list_array(false); } + +TEST(LogicalArray, CreateListNullable) { test_list_array(true); } + +TEST(LogicalArray, CreateStructNonNullable) { test_struct_array(false); } + +TEST(LogicalArray, CreateStructNullable) { test_struct_array(true); } + +TEST(LogicalArray, CreateLike) +{ + test_isomorphic(false); + test_isomorphic(true); +} + +TEST(LogicalArray, CreateInvalid) { test_invalid(); } + +} // namespace array_test diff --git a/tests/cpp/unit/logical_store.cc b/tests/cpp/unit/logical_store.cc index 1d89e70645..cd9a823544 100644 --- a/tests/cpp/unit/logical_store.cc +++ b/tests/cpp/unit/logical_store.cc @@ -11,9 +11,6 @@ */ #include -#include -#include -#include #include "legate.h" diff --git a/tests/cpp/unit/registration.cc b/tests/cpp/unit/registration.cc index 653f311bce..e22150c781 100644 --- a/tests/cpp/unit/registration.cc +++ b/tests/cpp/unit/registration.cc @@ -19,13 +19,13 @@ namespace test_registration { template struct CPUVariantTask : public legate::LegateTask> { static const int32_t TASK_ID = ID; - static void cpu_variant(legate::TaskContext& context) {} + static void cpu_variant(legate::TaskContext context) {} }; template struct GPUVariantTask : public legate::LegateTask> { static const int32_t TASK_ID = ID; - static void gpu_variant(legate::TaskContext& context) {} + static void gpu_variant(legate::TaskContext context) {} }; } // namespace test_registration diff --git a/tests/cpp/unit/scalar.cc b/tests/cpp/unit/scalar.cc index d74c7e4f11..313fe8b94b 100644 --- a/tests/cpp/unit/scalar.cc +++ b/tests/cpp/unit/scalar.cc @@ -244,9 +244,9 @@ void checkPack(const legate::Scalar& scalar) EXPECT_NE(legion_buffer.get_ptr(), nullptr); legate::BaseDeserializer deserializer(legion_buffer.get_ptr(), legion_buffer.get_size()); - auto scalar_unpack = deserializer._unpack_scalar(); - EXPECT_EQ(scalar_unpack.type().code(), scalar.type().code()); - EXPECT_EQ(scalar_unpack.size(), scalar.size()); + auto scalar_unpack = deserializer.unpack_scalar(); + EXPECT_EQ(scalar_unpack->type()->code, scalar.type().code()); + EXPECT_EQ(scalar_unpack->size(), scalar.size()); } TEST(ScalarUnit, Pack) diff --git a/tests/integration/collective/src/collective_test.cc b/tests/integration/collective/src/collective_test.cc index 27d89c9c0e..0a7a129ca5 100644 --- a/tests/integration/collective/src/collective_test.cc +++ b/tests/integration/collective/src/collective_test.cc @@ -3,7 +3,7 @@ namespace collective { -/* static*/ void CollectiveTestTask::cpu_variant(legate::TaskContext& context) +/* static*/ void CollectiveTestTask::cpu_variant(legate::TaskContext context) { // this task should be empty since we are only testing mapper logic } diff --git a/tests/integration/collective/src/collective_test.h b/tests/integration/collective/src/collective_test.h index 37ab9a5fdc..9f75b61a4e 100644 --- a/tests/integration/collective/src/collective_test.h +++ b/tests/integration/collective/src/collective_test.h @@ -19,7 +19,7 @@ namespace collective { class CollectiveTestTask : public Task { public: - static void cpu_variant(legate::TaskContext& context); + static void cpu_variant(legate::TaskContext context); }; } // namespace collective diff --git a/tests/integration/region_manager/src/tester.cc b/tests/integration/region_manager/src/tester.cc index a9d193883a..96be31df53 100644 --- a/tests/integration/region_manager/src/tester.cc +++ b/tests/integration/region_manager/src/tester.cc @@ -14,6 +14,6 @@ namespace region_manager { -/*static*/ void TesterTask::cpu_variant(legate::TaskContext& context) {} +/*static*/ void TesterTask::cpu_variant(legate::TaskContext context) {} } // namespace region_manager diff --git a/tests/integration/region_manager/src/tester.h b/tests/integration/region_manager/src/tester.h index cb67f16571..91cd0b3cea 100644 --- a/tests/integration/region_manager/src/tester.h +++ b/tests/integration/region_manager/src/tester.h @@ -19,7 +19,7 @@ namespace region_manager { struct TesterTask : public legate::LegateTask { static constexpr int TASK_ID = TESTER; - static void cpu_variant(legate::TaskContext& context); + static void cpu_variant(legate::TaskContext context); }; } // namespace region_manager diff --git a/tests/integration/registry/src/hello.cc b/tests/integration/registry/src/hello.cc index e82507ef8f..b3fb24eed4 100644 --- a/tests/integration/registry/src/hello.cc +++ b/tests/integration/registry/src/hello.cc @@ -17,7 +17,7 @@ namespace rg { class HelloTask : public Task { public: - static void cpu_variant(legate::TaskContext& context) { log_registry.info() << "Hello"; } + static void cpu_variant(legate::TaskContext context) { log_registry.info() << "Hello"; } }; } // namespace rg diff --git a/tests/integration/registry/src/world.cc b/tests/integration/registry/src/world.cc index 9ab6e8fb45..ff7112dfd8 100644 --- a/tests/integration/registry/src/world.cc +++ b/tests/integration/registry/src/world.cc @@ -14,7 +14,7 @@ namespace rg { -/*static*/ void WorldTask::cpu_variant(legate::TaskContext& context) +/*static*/ void WorldTask::cpu_variant(legate::TaskContext context) { log_registry.info() << "World!"; } diff --git a/tests/integration/registry/src/world.h b/tests/integration/registry/src/world.h index f616a40793..658fa1e6e4 100644 --- a/tests/integration/registry/src/world.h +++ b/tests/integration/registry/src/world.h @@ -18,7 +18,7 @@ namespace rg { struct WorldTask : public Task { - static void cpu_variant(legate::TaskContext& context); + static void cpu_variant(legate::TaskContext context); }; } // namespace rg diff --git a/tests/integration/scoping/src/library.cc b/tests/integration/scoping/src/library.cc index 602a0fb0e2..f067100e56 100644 --- a/tests/integration/scoping/src/library.cc +++ b/tests/integration/scoping/src/library.cc @@ -27,7 +27,7 @@ struct Task : public legate::LegateTask { namespace { -void validate(legate::TaskContext& context) +void validate(legate::TaskContext context) { if (context.is_single_task()) return; @@ -43,18 +43,18 @@ void validate(legate::TaskContext& context) class MultiVariantTask : public Task { public: - static void cpu_variant(legate::TaskContext& context) { validate(context); } + static void cpu_variant(legate::TaskContext context) { validate(context); } #ifdef LEGATE_USE_OPENMP - static void omp_variant(legate::TaskContext& context) { validate(context); } + static void omp_variant(legate::TaskContext context) { validate(context); } #endif #ifdef LEGATE_USE_CUDA - static void gpu_variant(legate::TaskContext& context) { validate(context); } + static void gpu_variant(legate::TaskContext context) { validate(context); } #endif }; class CpuVariantOnlyTask : public Task { public: - static void cpu_variant(legate::TaskContext& context) { validate(context); } + static void cpu_variant(legate::TaskContext context) { validate(context); } }; void registration_callback() diff --git a/tests/integration/tree_reduce/src/produce_normal.cc b/tests/integration/tree_reduce/src/produce_normal.cc index c6c5c8a6bf..87f81c60ff 100644 --- a/tests/integration/tree_reduce/src/produce_normal.cc +++ b/tests/integration/tree_reduce/src/produce_normal.cc @@ -14,6 +14,6 @@ namespace tree_reduce { -/*static*/ void ProduceNormalTask::cpu_variant(legate::TaskContext& context) {} +/*static*/ void ProduceNormalTask::cpu_variant(legate::TaskContext context) {} } // namespace tree_reduce diff --git a/tests/integration/tree_reduce/src/produce_normal.h b/tests/integration/tree_reduce/src/produce_normal.h index 6f4822dbbb..6e0930cfe6 100644 --- a/tests/integration/tree_reduce/src/produce_normal.h +++ b/tests/integration/tree_reduce/src/produce_normal.h @@ -18,7 +18,7 @@ namespace tree_reduce { struct ProduceNormalTask : public Task { - static void cpu_variant(legate::TaskContext& context); + static void cpu_variant(legate::TaskContext context); }; } // namespace tree_reduce diff --git a/tests/integration/tree_reduce/src/produce_unbound.cc b/tests/integration/tree_reduce/src/produce_unbound.cc index ffc6343e2b..fa365065b8 100644 --- a/tests/integration/tree_reduce/src/produce_unbound.cc +++ b/tests/integration/tree_reduce/src/produce_unbound.cc @@ -14,11 +14,12 @@ namespace tree_reduce { -/*static*/ void ProduceUnboundTask::cpu_variant(legate::TaskContext& context) +/*static*/ void ProduceUnboundTask::cpu_variant(legate::TaskContext context) { - auto& output = context.outputs().at(0); - auto size = context.get_task_index()[0] + 1; - auto buffer = output.create_output_buffer(legate::Point<1>(size), true /*bind*/); + auto output = context.output(0); + auto size = context.get_task_index()[0] + 1; + auto buffer = + output.data().create_output_buffer(legate::Point<1>(size), true /*bind*/); for (int64_t idx = 0; idx < size; ++idx) buffer[idx] = size; } diff --git a/tests/integration/tree_reduce/src/produce_unbound.h b/tests/integration/tree_reduce/src/produce_unbound.h index 6217259128..9fa1da8adf 100644 --- a/tests/integration/tree_reduce/src/produce_unbound.h +++ b/tests/integration/tree_reduce/src/produce_unbound.h @@ -18,7 +18,7 @@ namespace tree_reduce { struct ProduceUnboundTask : public Task { - static void cpu_variant(legate::TaskContext& context); + static void cpu_variant(legate::TaskContext context); }; } // namespace tree_reduce diff --git a/tests/integration/tree_reduce/src/reduce_normal.cc b/tests/integration/tree_reduce/src/reduce_normal.cc index 1a3aa50d78..d57e730688 100644 --- a/tests/integration/tree_reduce/src/reduce_normal.cc +++ b/tests/integration/tree_reduce/src/reduce_normal.cc @@ -14,15 +14,15 @@ namespace tree_reduce { -/*static*/ void ReduceNormalTask::cpu_variant(legate::TaskContext& context) +/*static*/ void ReduceNormalTask::cpu_variant(legate::TaskContext context) { - auto& inputs = context.inputs(); - auto& output = context.outputs().at(0); + auto inputs = context.inputs(); + auto output = context.output(0); for (auto& input : inputs) { auto shape = input.shape<1>(); assert(shape.empty() || shape.volume() == TILE_SIZE); } - output.create_output_buffer(legate::Point<1>(0), true); + output.data().create_output_buffer(legate::Point<1>(0), true); } } // namespace tree_reduce diff --git a/tests/integration/tree_reduce/src/reduce_normal.h b/tests/integration/tree_reduce/src/reduce_normal.h index 2b02dd1cdd..94c1219d05 100644 --- a/tests/integration/tree_reduce/src/reduce_normal.h +++ b/tests/integration/tree_reduce/src/reduce_normal.h @@ -18,7 +18,7 @@ namespace tree_reduce { struct ReduceNormalTask : public Task { - static void cpu_variant(legate::TaskContext& context); + static void cpu_variant(legate::TaskContext context); }; } // namespace tree_reduce diff --git a/tests/integration/tree_reduce/src/reduce_unbound.cc b/tests/integration/tree_reduce/src/reduce_unbound.cc index c2b16b31fd..d86af6c6d5 100644 --- a/tests/integration/tree_reduce/src/reduce_unbound.cc +++ b/tests/integration/tree_reduce/src/reduce_unbound.cc @@ -14,17 +14,17 @@ namespace tree_reduce { -/*static*/ void ReduceUnboundTask::cpu_variant(legate::TaskContext& context) +/*static*/ void ReduceUnboundTask::cpu_variant(legate::TaskContext context) { - auto& inputs = context.inputs(); - auto& output = context.outputs().at(0); + auto inputs = context.inputs(); + auto output = context.output(0); uint32_t expected = 1; for (auto& input : inputs) { auto shape = input.shape<1>(); assert(shape.volume() == expected); ++expected; } - output.create_output_buffer(legate::Point<1>(0), true); + output.data().create_output_buffer(legate::Point<1>(0), true); } } // namespace tree_reduce diff --git a/tests/integration/tree_reduce/src/reduce_unbound.h b/tests/integration/tree_reduce/src/reduce_unbound.h index 82454b1b6c..9527812ad9 100644 --- a/tests/integration/tree_reduce/src/reduce_unbound.h +++ b/tests/integration/tree_reduce/src/reduce_unbound.h @@ -18,7 +18,7 @@ namespace tree_reduce { struct ReduceUnboundTask : public Task { - static void cpu_variant(legate::TaskContext& context); + static void cpu_variant(legate::TaskContext context); }; } // namespace tree_reduce From e6beb2190e2b30d813bc133a42ac5fbc165e0608 Mon Sep 17 00:00:00 2001 From: Parag Kulkarni Date: Sun, 20 Aug 2023 15:02:40 +0530 Subject: [PATCH 0186/1425] Modify artifacts upload path --- .github/workflows/gh-build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index e78e244f1b..1fc79503e3 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -90,5 +90,4 @@ jobs: with: name: "legate.core.internal-${{ inputs.build-target }}-${{ inputs.sha }}" path: | - /tmp/conda-build - /tmp/out + artifacts From 1cac60034cf19b87c8e5c1f8811cb33a5cb17d39 Mon Sep 17 00:00:00 2001 From: Parag Kulkarni Date: Sun, 20 Aug 2023 15:14:57 +0530 Subject: [PATCH 0187/1425] Revert "Modify artifacts upload path" This reverts commit e6beb2190e2b30d813bc133a42ac5fbc165e0608. Revert my commit --- .github/workflows/gh-build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index 1fc79503e3..e78e244f1b 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -90,4 +90,5 @@ jobs: with: name: "legate.core.internal-${{ inputs.build-target }}-${{ inputs.sha }}" path: | - artifacts + /tmp/conda-build + /tmp/out From f2b9c979faca7f2f5ec38661d24d375806578df8 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Sun, 20 Aug 2023 11:46:50 -0700 Subject: [PATCH 0188/1425] Fix all doxygen warnings (#25) --- src/core/data/scalar.h | 8 ++++---- src/core/mapping/machine.h | 10 +++++++--- src/core/mapping/mapping.h | 22 +++++++++++----------- src/core/operation/task.h | 12 ++++++++---- 4 files changed, 30 insertions(+), 22 deletions(-) diff --git a/src/core/data/scalar.h b/src/core/data/scalar.h index 5b178773b6..8de713a795 100644 --- a/src/core/data/scalar.h +++ b/src/core/data/scalar.h @@ -50,11 +50,11 @@ class Scalar { public: /** * @brief Creates a shared `Scalar` with an existing allocation. The caller is responsible - * for passing in a sufficiently big allocation. When `copy` is true, the scalar copies the - * data stored in the allocation and becomes owned. + * for passing in a sufficiently big allocation. * - * @param type Type of the scalar(s) + * @param type Type of the scalar * @param data Allocation containing the data. + * @param copy If true, the scalar copies the data stored in the allocation and becomes owned. */ Scalar(Type type, const void* data, bool copy = false); /** @@ -104,7 +104,7 @@ class Scalar { /** * @brief Creates a rect scalar * - * @param point A rect from which the scalar should be constructed + * @param rect A rect from which the scalar should be constructed */ template Scalar(const Rect& rect); diff --git a/src/core/mapping/machine.h b/src/core/mapping/machine.h index b50e6ca364..a4e30a4206 100644 --- a/src/core/mapping/machine.h +++ b/src/core/mapping/machine.h @@ -134,6 +134,8 @@ class Machine { TaskTarget preferred_target() const; /** * @brief Returns the processor range for the preferred processor type in this descriptor + * + * @return A processor range */ ProcessorRange processor_range() const; /** @@ -141,7 +143,9 @@ class Machine { * * If the processor type does not exist in the descriptor, an empty range is returned * - * @target target Processor type to query + * @param target Processor type to query + * + * @return A processor range */ ProcessorRange processor_range(TaskTarget target) const; /** @@ -207,7 +211,7 @@ class Machine { * * @param from Starting index * @param to End index - * @param targets Processor type to slice + * @param target Processor type to slice * @param keep_others Optional flag to keep unsliced ranges in the returned machine descriptor * * @return Machine descriptor with the chosen procssor range sliced @@ -229,7 +233,7 @@ class Machine { * * This yields the same result as `.only(target)`. * - * @param targets Processor type to select + * @param target Processor type to select * * @return Machine descriptor with the chosen processor range */ diff --git a/src/core/mapping/mapping.h b/src/core/mapping/mapping.h index 1de024ade0..cc6af50959 100644 --- a/src/core/mapping/mapping.h +++ b/src/core/mapping/mapping.h @@ -244,7 +244,7 @@ struct InstanceMappingPolicy { /** * @brief Changes the store target * - * @param `target` A new store target + * @param target A new store target * * @return This instance mapping policy */ @@ -254,7 +254,7 @@ struct InstanceMappingPolicy { /** * @brief Changes the allocation policy * - * @param `allocation` A new allocation policy + * @param allocation A new allocation policy * * @return This instance mapping policy */ @@ -264,7 +264,7 @@ struct InstanceMappingPolicy { /** * @brief Changes the instance layout * - * @param `target` A new instance layout + * @param layout A new instance layout * * @return This instance mapping policy */ @@ -274,7 +274,7 @@ struct InstanceMappingPolicy { /** * @brief Changes the dimension ordering * - * @param `target` A new dimension ordering + * @param ordering A new dimension ordering * * @return This instance mapping policy */ @@ -284,7 +284,7 @@ struct InstanceMappingPolicy { /** * @brief Changes the value of `exact` * - * @param `target` A new value for the `exact` field + * @param exact A new value for the `exact` field * * @return This instance mapping policy */ @@ -296,31 +296,31 @@ struct InstanceMappingPolicy { /** * @brief Changes the store target * - * @param `target` A new store target + * @param target A new store target */ void set_target(StoreTarget target); /** * @brief Changes the allocation policy * - * @param `allocation` A new allocation policy + * @param allocation A new allocation policy */ void set_allocation_policy(AllocPolicy allocation); /** * @brief Changes the instance layout * - * @param `target` A new instance layout + * @param layout A new instance layout */ void set_instance_layout(InstLayout layout); /** * @brief Changes the dimension ordering * - * @param `target` A new dimension ordering + * @param ordering A new dimension ordering */ void set_ordering(DimOrdering ordering); /** * @brief Changes the value of `exact` * - * @param `target` A new value for the `exact` field + * @param exact A new value for the `exact` field */ void set_exact(bool exact); @@ -381,7 +381,7 @@ struct StoreMapping { /** * @brief Creates a mapping policy for the given set of stores using the instance mapping policy * - * @param store Target store for the mapping policy + * @param stores Target stores for the mapping policy * @param policy Instance mapping policy to apply * * @return A store mapping diff --git a/src/core/operation/task.h b/src/core/operation/task.h index eec2ac4f17..a3f6c368c7 100644 --- a/src/core/operation/task.h +++ b/src/core/operation/task.h @@ -45,7 +45,8 @@ class AutoTask { * associated with the array * * @param array An array to add to the task as input - * @param partition_symbol A partition symbol for the array + * + * @return The partition symbol assigned to the array */ Variable add_input(LogicalArray array); /** @@ -55,7 +56,8 @@ class AutoTask { * associated with the array * * @param array An array to add to the task as output - * @param partition_symbol A partition symbol for the array + * + * @return The partition symbol assigned to the array */ Variable add_output(LogicalArray array); /** @@ -66,7 +68,8 @@ class AutoTask { * * @param array An array to add to the task for reductions * @param redop ID of the reduction operator to use. The array's type must support the operator. - * @param partition_symbol A partition symbol for the array + * + * @return The partition symbol assigned to the array */ Variable add_reduction(LogicalArray array, ReductionOpKind redop); /** @@ -77,7 +80,8 @@ class AutoTask { * * @param array An array to add to the task for reductions * @param redop ID of the reduction operator to use. The array's type must support the operator. - * @param partition_symbol A partition symbol for the array + * + * @return The partition symbol assigned to the array */ Variable add_reduction(LogicalArray array, int32_t redop); From da423c677dadecfed28d6ead73b85c54c5f132f5 Mon Sep 17 00:00:00 2001 From: Parag Kulkarni Date: Mon, 21 Aug 2023 21:11:38 +0530 Subject: [PATCH 0189/1425] CI changes to generate legate.core.internal packages. (#26) build-- Modify image name to internal. This allows pushing the relevant packages (IMAGE_NAME=legate.core.internal doesn't seem to work, hence appended it with _pkg) Modify artifacts path test-- Modify image name to be same as build /home/coder/legate already exists. No need to add it. --- .github/workflows/gh-build.yml | 5 ++--- .github/workflows/gh-test.yml | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index e78e244f1b..4e1bb5015f 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -15,7 +15,7 @@ on: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - IMAGE_NAME: legate.core-${{ inputs.build-target }} + IMAGE_NAME: legate.core.internal_pkg-${{ inputs.build-target }} USE_CUDA: ${{ (inputs.build-target == 'cpu' && 'OFF') || 'ON' }} jobs: @@ -90,5 +90,4 @@ jobs: with: name: "legate.core.internal-${{ inputs.build-target }}-${{ inputs.sha }}" path: | - /tmp/conda-build - /tmp/out + artifacts diff --git a/.github/workflows/gh-test.yml b/.github/workflows/gh-test.yml index 23a2d730fe..a1ecbcdcb6 100644 --- a/.github/workflows/gh-test.yml +++ b/.github/workflows/gh-test.yml @@ -23,7 +23,7 @@ jobs: runs-on: ${{ inputs.runs-on }} container: options: -u root - image: ghcr.io/nv-legate/legate.core-${{ inputs.build-target }}:${{ inputs.sha }} + image: ghcr.io/nv-legate/legate.core.internal_pkg-${{ inputs.build-target }}:${{ inputs.sha }} env: PYTHONDONTWRITEBYTECODE: 1 NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} @@ -57,7 +57,6 @@ jobs: cp -ar legate/continuous_integration/home/coder/.gitconfig /home/coder/ cp -ar legate/continuous_integration/home/coder/.local /home/coder/ - mv legate /home/coder/legate chmod a+x /home/coder/.local/bin/* chown -R coder:coder /home/coder/ From 9cc5428d73b4cfcd6700ca8b94b1c98b6f4df093 Mon Sep 17 00:00:00 2001 From: Parag Kulkarni Date: Tue, 22 Aug 2023 16:05:45 +0530 Subject: [PATCH 0190/1425] Merge CI to sync branches from public repository. (#18) * - The workflow merges/syncs branches from public repository to internal. - It works with all branch-* branches. - It is scheduled to run weekly every Monday and Thursday at 10:00 AM. * - Verify using GITHUB_TOKEN instead of PAT. * Remove code to trigger the pipeline on schedule. Retain manual triggering. --- .github/workflows/merge-branches.sh | 49 +++++++++++++++++++++++++++++ .github/workflows/merge-ci.yml | 29 +++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100755 .github/workflows/merge-branches.sh create mode 100644 .github/workflows/merge-ci.yml diff --git a/.github/workflows/merge-branches.sh b/.github/workflows/merge-branches.sh new file mode 100755 index 0000000000..ea26ee507d --- /dev/null +++ b/.github/workflows/merge-branches.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +main() { + REMOTES="$@"; + if [ -z "$REMOTES" ]; then + REMOTES=$(git remote); + fi + REMOTES=$(echo "$REMOTES" | xargs -n1 echo) + CLB=$(git rev-parse --abbrev-ref HEAD); + echo "$REMOTES" | while read REMOTE; do + git remote update $REMOTE + git branch -r \ + | git branch -r | awk 'BEGIN { FS = "/" };/'"$REMOTE"'/{print $2}' \ + | while read BRANCH; do + if [[ $BRANCH == !(main|branch-*|gh-pages) ]]; then + echo "Skipping branch $BRANCH because it does not match the (main|branch-*) pattern."; + continue; + fi + # first, delete the local branch if there is one + git branch -D $BRANCH 2>/dev/null || true + # checkout the branch tracking from origin or fail if there isn't one yet + git checkout --track origin/$BRANCH 2>/dev/null || true + # reset the branch, or fail if the branch is not checked out + git reset --hard origin/$BRANCH 2>/dev/null || true + ARB="refs/remotes/$REMOTE/$BRANCH"; + ALB="refs/heads/$BRANCH"; + NBEHIND=$(( $(git rev-list --count $ALB..$ARB 2>/dev/null || echo "-1") )); + NAHEAD=$(( $(git rev-list --count $ARB..$ALB 2>/dev/null || true) )); + if [ "$NBEHIND" -gt 0 ]; then + if [ "$NAHEAD" -gt 0 ]; then + echo " branch $BRANCH is $NAHEAD commit(s) ahead of $REMOTE/$BRANCH. Public branches cannot contain internal commits."; + exit 1; + else + echo " branch $BRANCH was $NBEHIND commit(s) behind of $REMOTE/$BRANCH. resetting local branch to remote"; + git reset --hard $REMOTE/$BRANCH >/dev/null; + git push origin $BRANCH + fi + elif [ "$NBEHIND" -eq -1 ]; then + echo " branch $BRANCH does not exist yet. Creating a new branch to track remote"; + git branch -f $BRANCH -t $ARB >/dev/null; + git push origin $BRANCH + else + echo "Nothing to be done for branch $BRANCH" + fi + done + done +} + +main $@ diff --git a/.github/workflows/merge-ci.yml b/.github/workflows/merge-ci.yml new file mode 100644 index 0000000000..c39602d047 --- /dev/null +++ b/.github/workflows/merge-ci.yml @@ -0,0 +1,29 @@ +name: merge-public + +on: + workflow_dispatch: + +jobs: + merge: + runs-on: ubuntu-latest + steps: + - name: Legate Core Internal repository + uses: actions/checkout@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Merge the branches + run: | + DEFAULT_BRANCH=$(git remote show origin | awk '/HEAD branch/ {print $NF}') + git fetch origin + git checkout $DEFAULT_BRANCH + git reset --hard origin/$DEFAULT_BRANCH + git config --local user.name 'SyncAction' + git config --local user.email 'sync@nowhere' + git remote add GhLegatePublic https://github.com/nv-legate/legate.core.git || true + git branch -a + git remote show origin + git remote set-url --push origin https://github.com/nv-legate/legate.core.internal.git + git remote show origin + . .github/workflows/merge-branches.sh GhLegatePublic + git push origin --tags From fd5963869214e9d1ea2d1c9cb5fc50ec3233ff36 Mon Sep 17 00:00:00 2001 From: Parag Kulkarni Date: Fri, 25 Aug 2023 11:31:00 +0530 Subject: [PATCH 0191/1425] Provide requisite workflow permission (#27) * Provide requisite workflow permission - Replace GITHUB_TOKEN with WORKFLOW_TOKEN. - Trigger script with push also. (GitHub Apps added to repository can't access the secrets in the repository without adequate permission. The token issued for GitHub Actions doesn't have this permission by default.) * - Remove the push mechanism. It was only added for testing. --- .github/workflows/merge-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/merge-ci.yml b/.github/workflows/merge-ci.yml index c39602d047..69f6622d79 100644 --- a/.github/workflows/merge-ci.yml +++ b/.github/workflows/merge-ci.yml @@ -10,7 +10,7 @@ jobs: - name: Legate Core Internal repository uses: actions/checkout@v3 with: - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.WORKFLOW_TOKEN }} - name: Merge the branches run: | From 38b8d3504f7b8e896e0bb5db3d968bb01a38b44f Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Mon, 28 Aug 2023 18:44:04 -0700 Subject: [PATCH 0192/1425] Change assert(false) to exceptions in dispatch methods (#29) --- src/core/utilities/dispatch.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core/utilities/dispatch.h b/src/core/utilities/dispatch.h index 178472a6ae..8deb0656c8 100644 --- a/src/core/utilities/dispatch.h +++ b/src/core/utilities/dispatch.h @@ -70,7 +70,7 @@ struct inner_type_dispatch_fn { } default: break; } - assert(false); + throw std::runtime_error("Unsupported type code"); return f.template operator()(std::forward(args)...); } }; @@ -125,7 +125,7 @@ struct inner_dim_dispatch_fn { } #endif } - assert(false); + throw std::runtime_error("Unsupported number of dimensions"); return f.template operator()(std::forward(args)...); } }; @@ -194,7 +194,7 @@ constexpr decltype(auto) double_dispatch(int dim, Type::Code code, Functor f, Fn } #endif } - assert(false); + throw std::runtime_error("Unsupported number of dimensions"); return inner_type_dispatch_fn<1>{}(code, f, std::forward(args)...); } @@ -262,7 +262,7 @@ constexpr decltype(auto) double_dispatch(int dim1, int dim2, Functor f, Fnargs&& } #endif } - assert(false); + throw std::runtime_error("Unsupported number of dimensions"); return inner_dim_dispatch_fn<1>{}(dim2, f, std::forward(args)...); } @@ -329,7 +329,7 @@ constexpr decltype(auto) dim_dispatch(int dim, Functor f, Fnargs&&... args) } #endif } - assert(false); + throw std::runtime_error("Unsupported number of dimensions"); return f.template operator()<1>(std::forward(args)...); } @@ -394,7 +394,7 @@ constexpr decltype(auto) type_dispatch(Type::Code code, Functor f, Fnargs&&... a } default: break; } - assert(false); + throw std::runtime_error("Unsupported type code"); return f.template operator()(std::forward(args)...); } From 519e26043c68694711b7b54d9d7a02707532264c Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Tue, 29 Aug 2023 16:20:38 -0400 Subject: [PATCH 0193/1425] Use `copy-pr-bot` (#35) This PR replaces the `copy_prs` functionality from the `ops-bot` with the new dedicated `copy-pr-bot` GitHub application. Thorough documentation for the new `copy-pr-bot` application can be viewed below. - https://docs.gha-runners.nvidia.com/apps/copy-pr-bot/ **Important**: `copy-pr-bot` enforces signed commits. If an organization member opens a PR that contains unsigned commits, it will be deemed untrusted and therefore require an `/ok to test` comment. See the GitHub docs [here](https://docs.github.com/en/authentication/managing-commit-signature-verification/about-commit-signature-verification) for information on how to set up commit signing. Any time a PR is deemed untrusted, it will receive a comment that looks like this: https://github.com/rapidsai/ci-imgs/pull/63#issuecomment-1688973208. Every subsequent commit on an untrusted PR will require an additional `/ok to test` comment. Any existing PRs that have unsigned commits after this change is merged will require an `/ok to test` comment for each subsequent commit _or_ the PR can be rebased to include signed commits as mentioned in the docs below: https://docs.gha-runners.nvidia.com/cpr/contributors. This information is all included on the documentation page linked above. _I've skipped CI on this PR since it's not a change that is tested._ [skip ci] --- .github/copy-pr-bot.yaml | 4 ++++ .github/ops-bot.yaml | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 .github/copy-pr-bot.yaml delete mode 100644 .github/ops-bot.yaml diff --git a/.github/copy-pr-bot.yaml b/.github/copy-pr-bot.yaml new file mode 100644 index 0000000000..895ba83ee5 --- /dev/null +++ b/.github/copy-pr-bot.yaml @@ -0,0 +1,4 @@ +# Configuration file for `copy-pr-bot` GitHub App +# https://docs.gha-runners.nvidia.com/apps/copy-pr-bot/ + +enabled: true diff --git a/.github/ops-bot.yaml b/.github/ops-bot.yaml deleted file mode 100644 index 84bbe71f46..0000000000 --- a/.github/ops-bot.yaml +++ /dev/null @@ -1,4 +0,0 @@ -# This file controls which features from the `ops-bot` repository below are enabled. -# - https://github.com/rapidsai/ops-bot - -copy_prs: true From 9f1280a051bcb30b2a8ac07d8c742669cc3aed3f Mon Sep 17 00:00:00 2001 From: Jacob Faibussowitsch Date: Wed, 30 Aug 2023 19:20:07 -0400 Subject: [PATCH 0194/1425] Fix Many Build Warnings (#30) * Add more public API test in machine unit test * Revert "Add more public API test in machine unit test" This reverts commit 1e2df63b79989810042ec4d610bbd2e9ff8c13c8. * Add LEGATE_CPP_VERSION to specify current C++ version * src/legate_defines.h: define macro * Convert throw() to noexcept * src/core/task/exception.h: fix * src/core/data/detail/transform.h: fix * Fix inconsistent struct vs class declarations to work around MS ABI incompatibilities * src/core/data/array.h: class -> struct to match decl * src/core/data/detail/logical_array.h: class -> struct to match decl * src/core/data/detail/logical_region_field.h: class -> struct to match decl ... * src/core/runtime/detail/runtime.h: class -> struct to match decl * Fix pessimizing std::move on return value * src/core/data/detail/logical_store.cc: remove * src/core/data/detail/transform.cc: remove * src/core/data/scalar.cc: remove from temporary * src/core/mapping/array.cc: remove * src/core/mapping/mapping.cc: remove * src/core/mapping/operation.cc: remove * src/core/operation/detail/operation.cc: remove * src/core/task/task_context.cc: remove * src/core/utilities/deserializer.cc: remove * src/core/utilities/deserializer.inl: remove * Add missing override specificier for virtual overriding functions * src/core/partitioning/detail/constraint.h: add to validate() * Fix constructor initialization order * src/core/data/detail/logical_store.cc: initialize volume_ first * src/core/data/detail/store.cc: initialize num_elements_ first * src/core/mapping/detail/operation.cc: fix * src/core/operation/detail/operation.cc: fix * src/core/runtime/detail/projection.cc: fix * Remove dead code * src/core/data/detail/logical_store.cc: remove p_tiling * src/core/data/detail/transform.cc: fix * src/core/operation/detail/copy_launcher.cc: fix * src/core/partitioning/detail/constraint.cc: fix * src/core/partitioning/detail/partitioner.cc: fix * src/core/runtime/detail/shard.cc: remove call to get_runtime() * Fix destructor slicing error * src/core/data/transform.h: Transform destructor should be virtual, otherwise smart pointer destructors slice TransformStack in destructor * Split ConsensusMatchResult into its own header to resolve several issues. Primarily, this solves implicit instantiation errors for std::deque> in field_manager.h. Additionally, this also resolves a (percieved) circular dependence between runtime.h and field_manager.h * src/core/runtime/detail/consensus_match_result.h: new * src/core/runtime/detail/field_manager.h: include it * src/core/runtime/detail/runtime.h: include it * Remove pointless user-defined copy-constructors, which inhibit automatic generation of copy-assignment operators * src/core/utilities/span.h (Span): remove it * src/core/runtime/detail/library.h (ResourceIdScope): remove it * Silence unused private variable * src/core/operation/detail/fill_launcher.cc: cast tag to void * Add explicit trivial copy and move constructors for virtual bases as virtual destructors inhibit their generation * src/core/partitioning/detail/constraint.h (Partition): add * src/core/partitioning/detail/constraint.h (Expr): add * Pacify -Wformat-pedantic * src/core/runtime/detail/field_manager.cc: %p is used only for void * * src/core/runtime/detail/projection.cc: ditto * src/core/comm/local_comm.cc: ditto --------- Co-authored-by: monah --- src/core/comm/local_comm.cc | 14 ++---- src/core/data/array.h | 2 +- src/core/data/detail/logical_array.h | 4 +- src/core/data/detail/logical_region_field.h | 2 +- src/core/data/detail/logical_store.cc | 9 ++-- src/core/data/detail/logical_store.h | 4 +- src/core/data/detail/store.cc | 10 ++-- src/core/data/detail/store.h | 2 +- src/core/data/detail/transform.cc | 7 ++- src/core/data/detail/transform.h | 6 +-- src/core/data/logical_array.h | 2 +- src/core/data/logical_store.h | 2 +- src/core/data/scalar.cc | 2 +- src/core/mapping/array.cc | 2 +- src/core/mapping/detail/operation.cc | 2 +- src/core/mapping/detail/store.h | 2 +- src/core/mapping/machine.h | 2 +- src/core/mapping/mapping.cc | 2 +- src/core/mapping/mapping.h | 4 +- src/core/mapping/operation.cc | 2 +- src/core/operation/detail/copy.h | 2 +- src/core/operation/detail/copy_launcher.cc | 1 - src/core/operation/detail/copy_launcher.h | 4 +- src/core/operation/detail/fill_launcher.cc | 1 + src/core/operation/detail/fill_launcher.h | 2 +- src/core/operation/detail/gather.h | 2 +- src/core/operation/detail/launcher_arg.h | 4 +- src/core/operation/detail/operation.cc | 6 +-- src/core/operation/detail/operation.h | 2 +- src/core/operation/detail/scatter.h | 2 +- src/core/operation/detail/scatter_gather.h | 2 +- src/core/operation/detail/task.h | 2 +- src/core/operation/detail/task_launcher.h | 2 +- src/core/partitioning/constraint.h | 4 +- src/core/partitioning/detail/constraint.cc | 7 ++- src/core/partitioning/detail/constraint.h | 18 ++++--- .../partitioning/detail/constraint_solver.h | 2 +- src/core/partitioning/detail/partitioner.cc | 2 +- src/core/partitioning/detail/partitioner.h | 4 +- src/core/partitioning/partition.h | 8 +++- .../runtime/detail/consensus_match_result.h | 48 +++++++++++++++++++ src/core/runtime/detail/field_manager.cc | 20 ++++---- src/core/runtime/detail/field_manager.h | 4 +- src/core/runtime/detail/library.h | 3 -- src/core/runtime/detail/machine_manager.h | 2 +- src/core/runtime/detail/projection.cc | 5 +- src/core/runtime/detail/runtime.cc | 3 +- src/core/runtime/detail/runtime.h | 30 +----------- src/core/runtime/detail/shard.cc | 1 - src/core/task/detail/task_context.cc | 2 +- src/core/task/exception.h | 2 +- src/core/task/registrar.h | 2 +- src/core/task/task_context.cc | 2 +- src/core/type/detail/type_info.cc | 2 +- src/core/utilities/deserializer.cc | 5 +- src/core/utilities/deserializer.inl | 2 +- src/core/utilities/span.h | 3 +- src/legate_defines.h | 30 ++++++++++++ 58 files changed, 189 insertions(+), 136 deletions(-) create mode 100644 src/core/runtime/detail/consensus_match_result.h diff --git a/src/core/comm/local_comm.cc b/src/core/comm/local_comm.cc index b86ce73b59..1c6b273cf4 100644 --- a/src/core/comm/local_comm.cc +++ b/src/core/comm/local_comm.cc @@ -129,8 +129,6 @@ int LocalNetwork::alltoallv(const void* sendbuf, CollDataType type, CollComm global_comm) { - int res; - int total_size = global_comm->global_comm_size; int global_rank = global_comm->global_rank; @@ -167,11 +165,11 @@ int LocalNetwork::alltoallv(const void* sendbuf, recvfrom_global_rank, recvfrom_seg_id, sdispls[recvfrom_seg_id], - src, + static_cast(src), global_rank, recvfrom_global_rank, rdispls[recvfrom_global_rank], - dst); + static_cast(dst)); #endif memcpy(dst, src, recvcounts[recvfrom_global_rank] * type_extent); } @@ -189,8 +187,6 @@ int LocalNetwork::alltoallv(const void* sendbuf, int LocalNetwork::alltoall( const void* sendbuf, void* recvbuf, int count, CollDataType type, CollComm global_comm) { - int res; - int total_size = global_comm->global_comm_size; int global_rank = global_comm->global_rank; @@ -221,10 +217,10 @@ int LocalNetwork::alltoall( type_extent, recvfrom_global_rank, recvfrom_seg_id, - src, + static_cast(src), global_rank, recvfrom_global_rank, - dst); + static_cast(dst)); #endif memcpy(dst, src, count * type_extent); } @@ -271,7 +267,7 @@ int LocalNetwork::allgather( recvfrom_global_rank, src, global_rank, - dst); + static_cast(dst)); #endif memcpy(dst, src, count * type_extent); } diff --git a/src/core/data/array.h b/src/core/data/array.h index 64f4aa2e41..670b932d81 100644 --- a/src/core/data/array.h +++ b/src/core/data/array.h @@ -23,7 +23,7 @@ namespace legate { namespace detail { -class Array; +struct Array; } // namespace detail class ListArray; diff --git a/src/core/data/detail/logical_array.h b/src/core/data/detail/logical_array.h index 38b13c0c64..17fefe80e6 100644 --- a/src/core/data/detail/logical_array.h +++ b/src/core/data/detail/logical_array.h @@ -18,10 +18,10 @@ namespace legate::detail { -class Array; +struct Array; class AutoTask; class BaseArray; -class ConstraintSolver; +struct ConstraintSolver; class ListLogicalArray; class Variable; diff --git a/src/core/data/detail/logical_region_field.h b/src/core/data/detail/logical_region_field.h index 0a446f8792..21974b2849 100644 --- a/src/core/data/detail/logical_region_field.h +++ b/src/core/data/detail/logical_region_field.h @@ -22,7 +22,7 @@ #include "core/runtime/detail/field_manager.h" namespace legate { -class Partition; +struct Partition; class Tiling; } // namespace legate diff --git a/src/core/data/detail/logical_store.cc b/src/core/data/detail/logical_store.cc index efb8aea9c5..bcafd79fbe 100644 --- a/src/core/data/detail/logical_store.cc +++ b/src/core/data/detail/logical_store.cc @@ -51,8 +51,8 @@ Storage::Storage(const Shape& extents, std::shared_ptr type, bool optimize : storage_id_(Runtime::get_runtime()->get_unique_storage_id()), dim_(extents.size()), extents_(extents), - type_(std::move(type)), volume_(extents.volume()), + type_(std::move(type)), offsets_(dim_, 0) { if (optimize_scalar && volume_ == 1) kind_ = Kind::FUTURE; @@ -65,10 +65,10 @@ Storage::Storage(const Shape& extents, std::shared_ptr type, const Legion: : storage_id_(Runtime::get_runtime()->get_unique_storage_id()), dim_(extents.size()), extents_(extents), + volume_(extents.volume()), type_(std::move(type)), kind_(Kind::FUTURE), future_(future), - volume_(extents.volume()), offsets_(dim_, 0) { #ifdef DEBUG_LEGATE @@ -84,8 +84,8 @@ Storage::Storage(Shape&& extents, : storage_id_(Runtime::get_runtime()->get_unique_storage_id()), dim_(extents.size()), extents_(std::move(extents)), - type_(std::move(type)), volume_(extents_.volume()), + type_(std::move(type)), level_(parent->level() + 1), parent_(std::move(parent)), color_(std::move(color)), @@ -140,7 +140,6 @@ std::shared_ptr Storage::slice(Shape tile_shape, Shape offsets) auto tiling = create_tiling(std::move(tile_shape), std::move(color_shape), std::move(signed_offsets)); - auto* p_tiling = static_cast(tiling.get()); auto storage_partition = root->create_partition(std::move(tiling), can_tile_completely); return storage_partition->get_child_storage(color); } @@ -639,7 +638,7 @@ std::shared_ptr LogicalStore::find_or_create_key_partition( #ifdef DEBUG_LEGATE assert(store_part != nullptr); #endif - return std::move(store_part); + return store_part; } bool LogicalStore::has_key_partition(const mapping::detail::Machine& machine, diff --git a/src/core/data/detail/logical_store.h b/src/core/data/detail/logical_store.h index 4fa0d015a3..2b192aac30 100644 --- a/src/core/data/detail/logical_store.h +++ b/src/core/data/detail/logical_store.h @@ -26,9 +26,9 @@ namespace legate::detail { -class Analyzable; +struct Analyzable; class LogicalStorePartition; -class ProjectionInfo; +struct ProjectionInfo; class Strategy; class StoragePartition; class Store; diff --git a/src/core/data/detail/store.cc b/src/core/data/detail/store.cc index 4e8a256dd3..1456823603 100644 --- a/src/core/data/detail/store.cc +++ b/src/core/data/detail/store.cc @@ -76,15 +76,15 @@ Domain RegionField::domain() const { return dim_dispatch(dim_, get_domain_fn{}, void RegionField::unmap() { detail::Runtime::get_runtime()->unmap_physical_region(pr_); } UnboundRegionField::UnboundRegionField(const Legion::OutputRegion& out, Legion::FieldID fid) - : out_(out), - fid_(fid), - num_elements_( - Legion::UntypedDeferredValue(sizeof(size_t), find_memory_kind_for_executing_processor())) + : num_elements_( + Legion::UntypedDeferredValue(sizeof(size_t), find_memory_kind_for_executing_processor())), + out_(out), + fid_(fid) { } UnboundRegionField::UnboundRegionField(UnboundRegionField&& other) noexcept - : bound_(other.bound_), out_(other.out_), fid_(other.fid_), num_elements_(other.num_elements_) + : bound_(other.bound_), num_elements_(other.num_elements_), out_(other.out_), fid_(other.fid_) { other.bound_ = false; other.out_ = Legion::OutputRegion(); diff --git a/src/core/data/detail/store.h b/src/core/data/detail/store.h index 2bbf6f1848..b54893339d 100644 --- a/src/core/data/detail/store.h +++ b/src/core/data/detail/store.h @@ -22,7 +22,7 @@ class Store; namespace legate::detail { class BaseArray; -class TransformStack; +struct TransformStack; class RegionField { public: diff --git a/src/core/data/detail/transform.cc b/src/core/data/detail/transform.cc index 59c740219d..6832084f66 100644 --- a/src/core/data/detail/transform.cc +++ b/src/core/data/detail/transform.cc @@ -180,7 +180,7 @@ std::vector TransformStack::find_imaginary_dims() const std::vector dims; if (nullptr != parent_) { dims = parent_->find_imaginary_dims(); } if (nullptr != transform_) transform_->find_imaginary_dims(dims); - return std::move(dims); + return dims; } Shift::Shift(int32_t dim, int64_t offset) : dim_(dim), offset_(offset) {} @@ -271,7 +271,7 @@ Shape Shift::invert_point(const Shape& point) const { auto result = point; result[dim_] -= offset_; - return std::move(result); + return result; } void Shift::pack(BufferBuilder& buffer) const @@ -725,8 +725,7 @@ Domain Delinearize::transform(const Domain& input) const { auto delinearize = [](const auto dim, const auto ndim, const auto& strides, const Domain& input) { Domain output; - output.dim = input.dim - 1 + ndim; - int32_t in_dim = 0; + output.dim = input.dim - 1 + ndim; for (int32_t in_dim = 0, out_dim = 0; in_dim < input.dim; ++in_dim) { if (in_dim == dim) { auto lo = input.rect_data[in_dim]; diff --git a/src/core/data/detail/transform.h b/src/core/data/detail/transform.h index 14add35bc8..a9d4ebea2f 100644 --- a/src/core/data/detail/transform.h +++ b/src/core/data/detail/transform.h @@ -21,7 +21,7 @@ #include "core/utilities/typedefs.h" namespace legate { -class Partition; +struct Partition; } // namespace legate namespace legate::detail { @@ -30,13 +30,14 @@ class NonInvertibleTransformation : public std::exception { public: NonInvertibleTransformation() : error_message_("Non-invertible transformation") {} NonInvertibleTransformation(const std::string& error_message) : error_message_(error_message) {} - const char* what() const throw() { return error_message_.c_str(); } + const char* what() const noexcept { return error_message_.c_str(); } private: std::string error_message_; }; struct Transform { + virtual ~Transform() = default; virtual Domain transform(const Domain& input) const = 0; virtual Legion::DomainAffineTransform inverse_transform(int32_t in_dim) const = 0; virtual std::unique_ptr convert(const Partition* partition) const = 0; @@ -52,7 +53,6 @@ struct Transform { }; struct StoreTransform : public Transform { - virtual ~StoreTransform() {} virtual int32_t target_ndim(int32_t source_ndim) const = 0; virtual void find_imaginary_dims(std::vector&) const = 0; }; diff --git a/src/core/data/logical_array.h b/src/core/data/logical_array.h index a32aeec074..5e7a7a970a 100644 --- a/src/core/data/logical_array.h +++ b/src/core/data/logical_array.h @@ -27,7 +27,7 @@ namespace legate::detail { -class LogicalArray; +struct LogicalArray; } // namespace legate::detail diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index dfa85e6c6d..2dd6899207 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -27,7 +27,7 @@ */ namespace legate::detail { -class LogicalArray; +struct LogicalArray; class LogicalStore; class LogicalStorePartition; } // namespace legate::detail diff --git a/src/core/data/scalar.cc b/src/core/data/scalar.cc index 0e56826dd5..926c0a980c 100644 --- a/src/core/data/scalar.cc +++ b/src/core/data/scalar.cc @@ -41,7 +41,7 @@ const void* Scalar::ptr() const { return impl_->data(); } /*static*/ detail::Scalar* Scalar::create_impl(Type type, const void* data, bool copy) { - return new detail::Scalar(std::move(type.impl()), data, copy); + return new detail::Scalar(type.impl(), data, copy); } } // namespace legate diff --git a/src/core/mapping/array.cc b/src/core/mapping/array.cc index d6f134a937..a2d885af77 100644 --- a/src/core/mapping/array.cc +++ b/src/core/mapping/array.cc @@ -30,7 +30,7 @@ std::vector Array::stores() const std::vector result; result.push_back(data()); if (nullable()) result.push_back(null_mask()); - return std::move(result); + return result; } Domain Array::domain() const { return impl_->domain(); } diff --git a/src/core/mapping/detail/operation.cc b/src/core/mapping/detail/operation.cc index f295533953..7351a7adec 100644 --- a/src/core/mapping/detail/operation.cc +++ b/src/core/mapping/detail/operation.cc @@ -29,7 +29,7 @@ Task::Task(const Legion::Task* task, const legate::detail::Library* library, Legion::Mapping::MapperRuntime* runtime, const Legion::Mapping::MapperContext context) - : Mappable(task), task_(task), library_(library) + : Mappable(task), library_(library), task_(task) { TaskDeserializer dez(task, runtime, context); inputs_ = dez.unpack_arrays(); diff --git a/src/core/mapping/detail/store.h b/src/core/mapping/detail/store.h index e11e3db069..c55779df76 100644 --- a/src/core/mapping/detail/store.h +++ b/src/core/mapping/detail/store.h @@ -16,7 +16,7 @@ #include "core/utilities/typedefs.h" namespace legate::detail { -class TransformStack; +struct TransformStack; } // namespace legate::detail namespace legate::mapping::detail { diff --git a/src/core/mapping/machine.h b/src/core/mapping/machine.h index a4e30a4206..404c124077 100644 --- a/src/core/mapping/machine.h +++ b/src/core/mapping/machine.h @@ -113,7 +113,7 @@ struct ProcessorRange { std::ostream& operator<<(std::ostream& stream, const ProcessorRange& range); namespace detail { -class Machine; +struct Machine; } // namespace detail /** diff --git a/src/core/mapping/mapping.cc b/src/core/mapping/mapping.cc index fbf09bd604..1190c5e55f 100644 --- a/src/core/mapping/mapping.cc +++ b/src/core/mapping/mapping.cc @@ -261,7 +261,7 @@ std::vector StoreMapping::stores() const { std::vector result; for (auto& store : impl_->stores) { result.emplace_back(store); } - return std::move(result); + return result; } void StoreMapping::add_store(Store store) { impl_->stores.push_back(store.impl()); } diff --git a/src/core/mapping/mapping.h b/src/core/mapping/mapping.h index cc6af50959..7f4960c90a 100644 --- a/src/core/mapping/mapping.h +++ b/src/core/mapping/mapping.h @@ -29,8 +29,8 @@ namespace legate::mapping { namespace detail { class BaseMapper; -class DimOrdering; -class StoreMapping; +struct DimOrdering; +struct StoreMapping; } // namespace detail class Task; diff --git a/src/core/mapping/operation.cc b/src/core/mapping/operation.cc index 13743bab53..f661febd2a 100644 --- a/src/core/mapping/operation.cc +++ b/src/core/mapping/operation.cc @@ -25,7 +25,7 @@ std::vector convert_arrays(const Arrays& arrays) { std::vector result; for (auto& array : arrays) { result.emplace_back(array.get()); } - return std::move(result); + return result; } } // namespace diff --git a/src/core/operation/detail/copy.h b/src/core/operation/detail/copy.h index d2ac322a55..f3ffaadfe0 100644 --- a/src/core/operation/detail/copy.h +++ b/src/core/operation/detail/copy.h @@ -21,7 +21,7 @@ namespace legate::detail { -class ConstraintSolver; +struct ConstraintSolver; class Copy : public Operation { public: diff --git a/src/core/operation/detail/copy_launcher.cc b/src/core/operation/detail/copy_launcher.cc index 6e873ca9da..910697c5fb 100644 --- a/src/core/operation/detail/copy_launcher.cc +++ b/src/core/operation/detail/copy_launcher.cc @@ -96,7 +96,6 @@ void CopyLauncher::add_store(std::vector& args, { uint32_t req_idx = args.size(); auto region_field = store->get_region_field(); - auto region = region_field->region(); auto field_id = region_field->field_id(); if (LEGATE_CORE_KEY_STORE_TAG == proj_info->tag) key_proj_id_ = proj_info->proj_id; args.push_back(new CopyArg(req_idx, store, field_id, privilege, std::move(proj_info))); diff --git a/src/core/operation/detail/copy_launcher.h b/src/core/operation/detail/copy_launcher.h index 5ef595b98f..ad2d489b72 100644 --- a/src/core/operation/detail/copy_launcher.h +++ b/src/core/operation/detail/copy_launcher.h @@ -24,9 +24,9 @@ class Scalar; namespace legate::detail { -class CopyArg; +struct CopyArg; class LogicalStore; -class ProjectionInfo; +struct ProjectionInfo; class OutputRequirementAnalyzer; class RequirementAnalyzer; diff --git a/src/core/operation/detail/fill_launcher.cc b/src/core/operation/detail/fill_launcher.cc index b90f2f832c..2ec2f3184b 100644 --- a/src/core/operation/detail/fill_launcher.cc +++ b/src/core/operation/detail/fill_launcher.cc @@ -24,6 +24,7 @@ namespace legate::detail { FillLauncher::FillLauncher(const mapping::detail::Machine& machine, int64_t tag) : machine_(machine), tag_(tag) { + static_cast(tag_); } void FillLauncher::launch(const Legion::Domain& launch_domain, diff --git a/src/core/operation/detail/fill_launcher.h b/src/core/operation/detail/fill_launcher.h index 9d7b70e716..5e1194e58d 100644 --- a/src/core/operation/detail/fill_launcher.h +++ b/src/core/operation/detail/fill_launcher.h @@ -23,7 +23,7 @@ class BufferBuilder; namespace legate::detail { class LogicalStore; -class ProjectionInfo; +struct ProjectionInfo; class FillLauncher { public: diff --git a/src/core/operation/detail/gather.h b/src/core/operation/detail/gather.h index 8265252be1..e19652e53f 100644 --- a/src/core/operation/detail/gather.h +++ b/src/core/operation/detail/gather.h @@ -21,7 +21,7 @@ namespace legate::detail { -class ConstraintSolver; +struct ConstraintSolver; class Gather : public Operation { public: diff --git a/src/core/operation/detail/launcher_arg.h b/src/core/operation/detail/launcher_arg.h index 0e4f0d9c3b..54f923010e 100644 --- a/src/core/operation/detail/launcher_arg.h +++ b/src/core/operation/detail/launcher_arg.h @@ -22,8 +22,8 @@ namespace legate::detail { class BufferBuilder; -class OutputRegionArg; -class StoreAnalyzer; +struct OutputRegionArg; +struct StoreAnalyzer; class TaskLauncher; struct Serializable { diff --git a/src/core/operation/detail/operation.cc b/src/core/operation/detail/operation.cc index 982f59eb04..08936be041 100644 --- a/src/core/operation/detail/operation.cc +++ b/src/core/operation/detail/operation.cc @@ -20,8 +20,8 @@ namespace legate::detail { Operation::Operation(uint64_t unique_id, mapping::detail::Machine&& machine) : unique_id_(unique_id), - machine_(std::move(machine)), - provenance_(Runtime::get_runtime()->provenance_manager()->get_provenance()) + provenance_(Runtime::get_runtime()->provenance_manager()->get_provenance()), + machine_(std::move(machine)) { } @@ -65,7 +65,7 @@ std::unique_ptr Operation::create_projection_info(const Strategy auto store_partition = arg.store->create_partition(strategy[arg.variable]); auto proj_info = store_partition->create_projection_info(launch_domain); proj_info->tag = strategy.is_key_partition(arg.variable) ? LEGATE_CORE_KEY_STORE_TAG : 0; - return std::move(proj_info); + return proj_info; } } // namespace legate::detail diff --git a/src/core/operation/detail/operation.h b/src/core/operation/detail/operation.h index 41f6967dcf..929c68a117 100644 --- a/src/core/operation/detail/operation.h +++ b/src/core/operation/detail/operation.h @@ -19,7 +19,7 @@ #include "core/operation/detail/projection.h" namespace legate::detail { -class ConstraintSolver; +struct ConstraintSolver; class LogicalStore; class Strategy; class Variable; diff --git a/src/core/operation/detail/scatter.h b/src/core/operation/detail/scatter.h index 0209ed0e67..3aa8ab9a92 100644 --- a/src/core/operation/detail/scatter.h +++ b/src/core/operation/detail/scatter.h @@ -21,7 +21,7 @@ namespace legate::detail { -class ConstraintSolver; +struct ConstraintSolver; class Scatter : public Operation { public: diff --git a/src/core/operation/detail/scatter_gather.h b/src/core/operation/detail/scatter_gather.h index 0ba1e2ae9a..ad409ac89a 100644 --- a/src/core/operation/detail/scatter_gather.h +++ b/src/core/operation/detail/scatter_gather.h @@ -21,7 +21,7 @@ namespace legate::detail { -class ConstraintSolver; +struct ConstraintSolver; class ScatterGather : public Operation { public: diff --git a/src/core/operation/detail/task.h b/src/core/operation/detail/task.h index 2c59653cb9..1418a08e8b 100644 --- a/src/core/operation/detail/task.h +++ b/src/core/operation/detail/task.h @@ -26,7 +26,7 @@ class Scalar; namespace legate::detail { class CommunicatorFactory; -class ConstraintSolver; +struct ConstraintSolver; class Library; class LogicalStore; class LogicalStorePartition; diff --git a/src/core/operation/detail/task_launcher.h b/src/core/operation/detail/task_launcher.h index 6933948e16..b6575c52c0 100644 --- a/src/core/operation/detail/task_launcher.h +++ b/src/core/operation/detail/task_launcher.h @@ -26,7 +26,7 @@ namespace legate::detail { class Library; class LogicalStore; -class ProjectionInfo; +struct ProjectionInfo; class TaskLauncher { public: diff --git a/src/core/partitioning/constraint.h b/src/core/partitioning/constraint.h index 633b5065c9..381269b5a7 100644 --- a/src/core/partitioning/constraint.h +++ b/src/core/partitioning/constraint.h @@ -27,10 +27,10 @@ namespace legate { class AutoTask; namespace detail { -class Constraint; +struct Constraint; class Variable; } // namespace detail -extern template class default_delete; +extern template struct default_delete; /** * @ingroup partitioning diff --git a/src/core/partitioning/detail/constraint.cc b/src/core/partitioning/detail/constraint.cc index 63e18ed935..e46b2e55cd 100644 --- a/src/core/partitioning/detail/constraint.cc +++ b/src/core/partitioning/detail/constraint.cc @@ -140,9 +140,8 @@ std::string ImageConstraint::to_string() const std::unique_ptr ImageConstraint::resolve(const detail::Strategy& strategy) const { - const auto* src = var_function(); - const auto* range = var_range(); - auto src_part = strategy[src]; + const auto* src = var_function(); + auto src_part = strategy[src]; if (src_part->has_launch_domain()) { return create_image(src->operation()->find_store(src), src_part); } else { @@ -170,5 +169,5 @@ std::unique_ptr image(const Variable* var_function, const Varia // explicitly instantiate the deleter for std::unique_ptr namespace legate { -template class default_delete; +template struct default_delete; } diff --git a/src/core/partitioning/detail/constraint.h b/src/core/partitioning/detail/constraint.h index 405cc99414..1d8f6cd703 100644 --- a/src/core/partitioning/detail/constraint.h +++ b/src/core/partitioning/detail/constraint.h @@ -19,7 +19,7 @@ #include "core/utilities/tuple.h" namespace legate { -class Partition; +struct Partition; } // namespace legate namespace legate::detail { @@ -28,7 +28,7 @@ class Strategy; class Alignment; class Broadcast; -class Constraint; +struct Constraint; class ImageConstraint; class Literal; class Variable; @@ -38,7 +38,13 @@ struct Expr { LITERAL = 0, VARIABLE = 1, }; - virtual ~Expr() {} + Expr() = default; + virtual ~Expr() = default; + Expr(const Expr&) = default; + Expr(Expr&&) noexcept = default; + Expr& operator=(const Expr&) = default; + Expr& operator=(Expr&&) noexcept = default; + virtual void find_partition_symbols(std::vector& partition_symbols) const = 0; virtual bool closed() const = 0; virtual std::string to_string() const = 0; @@ -133,7 +139,7 @@ class Alignment : public Constraint { void find_partition_symbols(std::vector& partition_symbols) const override; public: - void validate() const; + void validate() const override; public: std::string to_string() const override; @@ -166,7 +172,7 @@ class Broadcast : public Constraint { void find_partition_symbols(std::vector& partition_symbols) const override; public: - void validate() const; + void validate() const override; public: std::string to_string() const override; @@ -196,7 +202,7 @@ class ImageConstraint : public Constraint { void find_partition_symbols(std::vector& partition_symbols) const override; public: - void validate() const; + void validate() const override; public: std::string to_string() const override; diff --git a/src/core/partitioning/detail/constraint_solver.h b/src/core/partitioning/detail/constraint_solver.h index 1978c50814..fd0f4e5eaf 100644 --- a/src/core/partitioning/detail/constraint_solver.h +++ b/src/core/partitioning/detail/constraint_solver.h @@ -54,7 +54,7 @@ struct ConstraintSolver { std::vector> constraints_{}; private: - class EquivClass; + struct EquivClass; std::map equiv_class_map_{}; std::vector equiv_classes_{}; diff --git a/src/core/partitioning/detail/partitioner.cc b/src/core/partitioning/detail/partitioner.cc index a903cd3350..3c854de9b2 100644 --- a/src/core/partitioning/detail/partitioner.cc +++ b/src/core/partitioning/detail/partitioner.cc @@ -246,7 +246,7 @@ std::unique_ptr Partitioner::partition_stores() std::stable_sort(remaining_symbols.begin(), remaining_symbols.end(), - [&solver, &comparison_key](const auto& part_symb_a, const auto& part_symb_b) { + [&comparison_key](const auto& part_symb_a, const auto& part_symb_b) { return comparison_key(part_symb_a) > comparison_key(part_symb_b); }); diff --git a/src/core/partitioning/detail/partitioner.h b/src/core/partitioning/detail/partitioner.h index f332d7a535..cc884f1148 100644 --- a/src/core/partitioning/detail/partitioner.h +++ b/src/core/partitioning/detail/partitioner.h @@ -21,12 +21,12 @@ #include "core/partitioning/restriction.h" namespace legate { -class Partition; +struct Partition; } // namespace legate namespace legate::detail { -class ConstraintSolver; +struct ConstraintSolver; class LogicalStore; class Partitioner; diff --git a/src/core/partitioning/partition.h b/src/core/partitioning/partition.h index e19ae94793..48cc323b87 100644 --- a/src/core/partitioning/partition.h +++ b/src/core/partitioning/partition.h @@ -37,8 +37,12 @@ struct Partition { }; public: - Partition() {} - virtual ~Partition() {} + Partition() = default; + virtual ~Partition() = default; + Partition(const Partition&) = default; + Partition(Partition&&) noexcept = default; + Partition& operator=(const Partition&) = default; + Partition& operator=(Partition&&) noexcept = default; public: virtual Kind kind() const = 0; diff --git a/src/core/runtime/detail/consensus_match_result.h b/src/core/runtime/detail/consensus_match_result.h new file mode 100644 index 0000000000..8e25299b62 --- /dev/null +++ b/src/core/runtime/detail/consensus_match_result.h @@ -0,0 +1,48 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#pragma once + +#include + +#include "core/runtime/library.h" // Legion + +namespace legate::detail { + +template +class ConsensusMatchResult { + private: + friend class Runtime; + ConsensusMatchResult(std::vector&& input, Legion::Context ctx, Legion::Runtime* runtime); + + public: + ~ConsensusMatchResult(); + ConsensusMatchResult(ConsensusMatchResult&&) = default; + ConsensusMatchResult& operator=(ConsensusMatchResult&&) = default; + + private: + ConsensusMatchResult(const ConsensusMatchResult&) = delete; + ConsensusMatchResult& operator=(const ConsensusMatchResult&) = delete; + + public: + void wait(); + const std::vector& input() const; + const std::vector& output() const; + + private: + std::vector input_; + std::vector output_; + Legion::Future future_; + bool complete_{false}; +}; + +} // namespace legate::detail diff --git a/src/core/runtime/detail/field_manager.cc b/src/core/runtime/detail/field_manager.cc index 30e7e7eb4b..2adf2d477a 100644 --- a/src/core/runtime/detail/field_manager.cc +++ b/src/core/runtime/detail/field_manager.cc @@ -33,7 +33,8 @@ std::shared_ptr FieldManager::allocate_field() if (!ordered_free_fields_.empty()) { const auto& field = ordered_free_fields_.front(); auto* rf = new LogicalRegionField(this, field.first, field.second); - log_legate.debug("Field %u recycled in field manager %p", field.second, this); + log_legate.debug( + "Field %u recycled in field manager %p", field.second, static_cast(this)); ordered_free_fields_.pop_front(); return std::shared_ptr(rf); } @@ -45,7 +46,7 @@ std::shared_ptr FieldManager::allocate_field() auto rgn_mgr = runtime_->find_or_create_region_manager(shape_); auto [lr, fid] = rgn_mgr->allocate_field(field_size_); auto* rf = new LogicalRegionField(this, lr, fid); - log_legate.debug("Field %u created in field manager %p", fid, this); + log_legate.debug("Field %u created in field manager %p", fid, static_cast(this)); return std::shared_ptr(rf); } @@ -55,7 +56,7 @@ std::shared_ptr FieldManager::import_field(const Legion::Log // Import the region only if the region manager is created fresh auto rgn_mgr = runtime_->find_or_create_region_manager(shape_); if (!rgn_mgr->has_space()) rgn_mgr->import_region(region); - log_legate.debug("Field %u imported in field manager %p", field_id, this); + log_legate.debug("Field %u imported in field manager %p", field_id, static_cast(this)); return std::make_shared(this, region, field_id); } @@ -64,10 +65,12 @@ void FieldManager::free_field(const Legion::LogicalRegion& region, bool unordered) { if (unordered && runtime_->consensus_match_required()) { - log_legate.debug("Field %u freed locally in field manager %p", field_id, this); + log_legate.debug( + "Field %u freed locally in field manager %p", field_id, static_cast(this)); unordered_free_fields_.push_back(FreeField(region, field_id)); } else { - log_legate.debug("Field %u freed in-order in field manager %p", field_id, this); + log_legate.debug( + "Field %u freed in-order in field manager %p", field_id, static_cast(this)); ordered_free_fields_.push_back(FreeField(region, field_id)); } } @@ -94,8 +97,9 @@ void FieldManager::issue_field_match() // Dispatch the match and put it on the queue of outstanding matches, but don't block on it yet. // We'll do that when we run out of ordered fields. matches_.push_back(runtime_->issue_consensus_match(std::move(input))); - log_legate.debug( - "Consensus match emitted with %zu local fields in field manager %p", regions.size(), this); + log_legate.debug("Consensus match emitted with %zu local fields in field manager %p", + regions.size(), + static_cast(this)); } void FieldManager::process_next_field_match() @@ -105,7 +109,7 @@ void FieldManager::process_next_field_match() auto& regions = regions_for_match_items_.front(); match.wait(); log_legate.debug("Consensus match result in field manager %p: %zu/%zu fields matched", - this, + static_cast(this), match.output().size(), match.input().size()); // Put all the matched fields into the ordered queue, in the same order as the match result, diff --git a/src/core/runtime/detail/field_manager.h b/src/core/runtime/detail/field_manager.h index 8e58124bfc..ae43299d38 100644 --- a/src/core/runtime/detail/field_manager.h +++ b/src/core/runtime/detail/field_manager.h @@ -16,12 +16,12 @@ #include "core/utilities/typedefs.h" +#include "core/runtime/detail/consensus_match_result.h" + namespace legate::detail { class LogicalRegionField; class Runtime; -template -class ConsensusMatchResult; class FieldManager { private: diff --git a/src/core/runtime/detail/library.h b/src/core/runtime/detail/library.h index 8dcc3b4ff7..908b5cf78c 100644 --- a/src/core/runtime/detail/library.h +++ b/src/core/runtime/detail/library.h @@ -31,9 +31,6 @@ class Library { ResourceIdScope() = default; ResourceIdScope(int64_t base, int64_t size) : base_(base), size_(size) {} - public: - ResourceIdScope(const ResourceIdScope&) = default; - public: int64_t translate(int64_t local_resource_id) const { return base_ + local_resource_id; } int64_t invert(int64_t resource_id) const diff --git a/src/core/runtime/detail/machine_manager.h b/src/core/runtime/detail/machine_manager.h index 20d0285609..d7be5bbd9b 100644 --- a/src/core/runtime/detail/machine_manager.h +++ b/src/core/runtime/detail/machine_manager.h @@ -15,7 +15,7 @@ #include "core/mapping/detail/machine.h" namespace legate::mapping::detail { -class Machine; +struct Machine; } // namespace legate::mapping::detail namespace legate::detail { diff --git a/src/core/runtime/detail/projection.cc b/src/core/runtime/detail/projection.cc index aa70153eb6..2fdb9ffd14 100644 --- a/src/core/runtime/detail/projection.cc +++ b/src/core/runtime/detail/projection.cc @@ -84,7 +84,7 @@ std::ostream& operator<<(std::ostream& out, const SymbolicExpr& expr) } RadixProjectionFunctor::RadixProjectionFunctor(int32_t radix, int32_t offset) - : radix_(radix), offset_(offset) + : offset_(offset), radix_(radix) { } @@ -317,7 +317,8 @@ void register_legate_core_projection_functors(Legion::Runtime* runtime, { auto proj_id = core_library->get_projection_id(LEGATE_CORE_DELINEARIZE_PROJ_ID); auto functor = new DelinearizationFunctor(runtime); - log_legate.debug("Register delinearizing functor: functor: %p, id: %d", functor, proj_id); + log_legate.debug( + "Register delinearizing functor: functor: %p, id: %d", static_cast(functor), proj_id); runtime->register_projection_functor(proj_id, functor, true /*silence warnings*/); { const std::lock_guard lock(functor_table_lock); diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index 68bc714d2b..18626e52e7 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -58,10 +58,10 @@ const char* TOPLEVEL_NAME = "Legate Core Toplevel Task"; Runtime::Runtime() : legion_runtime_(Legion::Runtime::get_runtime()), - next_type_uid_(CUSTOM_TYPE_UID_BASE), field_reuse_freq_( extract_env("LEGATE_FIELD_REUSE_FREQ", FIELD_REUSE_FREQ_DEFAULT, FIELD_REUSE_FREQ_TEST)), force_consensus_match_(extract_env("LEGATE_CONSENSUS", CONSENSUS_DEFAULT, CONSENSUS_TEST)), + next_type_uid_(CUSTOM_TYPE_UID_BASE), max_pending_exceptions_(extract_env( "LEGATE_MAX_PENDING_EXCEPTIONS", MAX_PENDING_EXCEPTIONS_DEFAULT, MAX_PENDING_EXCEPTIONS_TEST)) { @@ -1200,7 +1200,6 @@ void core_library_registration(Legion::Machine machine, config.max_shardings = LEGATE_CORE_MAX_FUNCTOR_ID; config.max_reduction_ops = LEGATE_CORE_MAX_REDUCTION_OP_ID; - auto runtime = Runtime::get_runtime(); auto core_lib = Runtime::get_runtime()->create_library( core_library_name, config, mapping::detail::create_core_mapper()); diff --git a/src/core/runtime/detail/runtime.h b/src/core/runtime/detail/runtime.h index e53628147e..abce5b34c6 100644 --- a/src/core/runtime/detail/runtime.h +++ b/src/core/runtime/detail/runtime.h @@ -21,6 +21,7 @@ #include "core/data/shape.h" #include "core/mapping/machine.h" #include "core/runtime/detail/communicator_manager.h" +#include "core/runtime/detail/consensus_match_result.h" #include "core/runtime/detail/field_manager.h" #include "core/runtime/detail/library.h" #include "core/runtime/detail/machine_manager.h" @@ -39,40 +40,13 @@ class AutoTask; class BaseLogicalArray; class Copy; class Library; -class LogicalArray; +struct LogicalArray; class LogicalRegionField; class LogicalStore; class ManualTask; class Operation; class StructLogicalArray; -template -class ConsensusMatchResult { - private: - friend class Runtime; - ConsensusMatchResult(std::vector&& input, Legion::Context ctx, Legion::Runtime* runtime); - - public: - ~ConsensusMatchResult(); - ConsensusMatchResult(ConsensusMatchResult&&) = default; - ConsensusMatchResult& operator=(ConsensusMatchResult&&) = default; - - private: - ConsensusMatchResult(const ConsensusMatchResult&) = delete; - ConsensusMatchResult& operator=(const ConsensusMatchResult&) = delete; - - public: - void wait(); - const std::vector& input() const; - const std::vector& output() const; - - private: - std::vector input_; - std::vector output_; - Legion::Future future_; - bool complete_{false}; -}; - class Runtime { public: Runtime(); diff --git a/src/core/runtime/detail/shard.cc b/src/core/runtime/detail/shard.cc index 1cd543c0d8..cfe10a6000 100644 --- a/src/core/runtime/detail/shard.cc +++ b/src/core/runtime/detail/shard.cc @@ -174,7 +174,6 @@ void legate_create_sharding_functor_using_projection(Legion::ShardID shard_id, uint32_t offset, uint32_t per_node_count) { - auto runtime = Legion::Runtime::get_runtime(); legate::detail::ShardingCallbackArgs args{ shard_id, proj_id, start_node, end_node, offset, per_node_count}; { diff --git a/src/core/task/detail/task_context.cc b/src/core/task/detail/task_context.cc index 5000832e63..e77b8b2964 100644 --- a/src/core/task/detail/task_context.cc +++ b/src/core/task/detail/task_context.cc @@ -30,7 +30,7 @@ TaskContext::TaskContext(const Legion::Task* task, machine_ = dez.unpack(); } - TaskDeserializer dez(task, regions); + TaskDeserializer dez(task, regions_); inputs_ = dez.unpack_arrays(); outputs_ = dez.unpack_arrays(); reductions_ = dez.unpack_arrays(); diff --git a/src/core/task/exception.h b/src/core/task/exception.h index 06fabfb9f8..ee5035c742 100644 --- a/src/core/task/exception.h +++ b/src/core/task/exception.h @@ -56,7 +56,7 @@ class TaskException : public std::exception { TaskException(const std::string& error_message) : index_(0), error_message_(error_message) {} public: - virtual const char* what() const throw() { return error_message_.c_str(); } + virtual const char* what() const noexcept { return error_message_.c_str(); } public: /** diff --git a/src/core/task/registrar.h b/src/core/task/registrar.h index 0f5b2c9ce4..834a7ecfa8 100644 --- a/src/core/task/registrar.h +++ b/src/core/task/registrar.h @@ -94,7 +94,7 @@ class TaskRegistrar { TaskRegistrar& operator=(TaskRegistrar&&) = delete; private: - class Impl; + struct Impl; Impl* impl_; }; diff --git a/src/core/task/task_context.cc b/src/core/task/task_context.cc index c4a769995f..ced695eabc 100644 --- a/src/core/task/task_context.cc +++ b/src/core/task/task_context.cc @@ -21,7 +21,7 @@ std::vector to_arrays(const std::vector>& { std::vector result; for (const auto& array_impl : array_impls) { result.emplace_back(array_impl); } - return std::move(result); + return result; } } // namespace diff --git a/src/core/type/detail/type_info.cc b/src/core/type/detail/type_info.cc index 7049adacab..8960e51b42 100644 --- a/src/core/type/detail/type_info.cc +++ b/src/core/type/detail/type_info.cc @@ -161,8 +161,8 @@ bool FixedArrayType::equal(const Type& other) const StructType::StructType(int32_t uid, std::vector>&& field_types, bool align) : ExtensionType(uid, Type::Code::STRUCT), aligned_(align), - alignment_(1), size_(0), + alignment_(1), field_types_(std::move(field_types)) { if (std::any_of( diff --git a/src/core/utilities/deserializer.cc b/src/core/utilities/deserializer.cc index 6874a77f79..450dabdf67 100644 --- a/src/core/utilities/deserializer.cc +++ b/src/core/utilities/deserializer.cc @@ -42,7 +42,7 @@ std::vector> TaskDeserializer::unpack_arrays() auto size = unpack(); arrays.reserve(size); for (uint32_t idx = 0; idx < size; ++idx) { arrays.emplace_back(unpack_array()); } - return std::move(arrays); + return arrays; } std::shared_ptr TaskDeserializer::unpack_array() @@ -140,7 +140,6 @@ void TaskDeserializer::_unpack(detail::RegionField& value) void TaskDeserializer::_unpack(detail::UnboundRegionField& value) { - auto dim = unpack(); auto idx = unpack(); auto fid = unpack(); @@ -185,7 +184,7 @@ std::vector> TaskDeserializer::unpack_arrays() auto size = unpack(); arrays.reserve(size); for (uint32_t idx = 0; idx < size; ++idx) { arrays.emplace_back(unpack_array()); } - return std::move(arrays); + return arrays; } std::shared_ptr TaskDeserializer::unpack_array() diff --git a/src/core/utilities/deserializer.inl b/src/core/utilities/deserializer.inl index 83a16a153e..d41a318b74 100644 --- a/src/core/utilities/deserializer.inl +++ b/src/core/utilities/deserializer.inl @@ -41,7 +41,7 @@ std::unique_ptr BaseDeserializer::unpack_scalar() auto type = unpack_type(); auto result = std::make_unique(type, args_.ptr(), false /*copy*/); args_ = args_.subspan(result->size()); - return std::move(result); + return result; } template diff --git a/src/core/utilities/span.h b/src/core/utilities/span.h index 88f47eecdf..c189bbcf0f 100644 --- a/src/core/utilities/span.h +++ b/src/core/utilities/span.h @@ -30,8 +30,7 @@ namespace legate { template struct Span { public: - Span() = default; - Span(const Span&) = default; + Span() = default; public: /** diff --git a/src/legate_defines.h b/src/legate_defines.h index 3fd346a09c..1d3fe3c306 100644 --- a/src/legate_defines.h +++ b/src/legate_defines.h @@ -12,6 +12,36 @@ #pragma once +#ifdef __cplusplus +#if __cplusplus <= 199711L +#ifdef _MSC_VER +// Unless a special flag is set, MSVC always reports C++ standard as C++98. But the floor is in +// fact C++14, so we assume that that is the case. +// See https://learn.microsoft.com/en-us/cpp/build/reference/zc-cplusplus?view=msvc-170#remarks +#define LEGATE_CPP_VERSION 14 +#else +// wrap C++98 to 0 since comparisons would otherwise fail +#define LEGATE_CPP_VERSION 0 +#endif +#elif __cplusplus <= 201103L +#define LEGATE_CPP_VERSION 11 +#elif __cplusplus <= 201402L +#define LEGATE_CPP_VERSION 14 +#elif __cplusplus <= 201703L +#define LEGATE_CPP_VERSION 17 +#elif __cplusplus <= 202002L +#define LEGATE_CPP_VERSION 20 +#else +#define LEGATE_CPP_VERSION 23 // current year, or date of c++2b ratification +#endif +#else +#define LEGATE_CPP_VERSION 0 // no C++ +#endif // __cplusplus + +#if defined(__cplusplus) && LEGATE_CPP_VERSION < 17 +#error "Legate requires C++17" +#endif + #define LEGATE_ABORT \ do { \ legate::log_legate.error( \ From 649be4cc5490d6b285ccbe0669f7d20f0b80c279 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Thu, 31 Aug 2023 16:29:07 -0700 Subject: [PATCH 0195/1425] Fix a bug introduced by #30 (#42) --- src/core/utilities/deserializer.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/utilities/deserializer.cc b/src/core/utilities/deserializer.cc index 450dabdf67..dd37c144b6 100644 --- a/src/core/utilities/deserializer.cc +++ b/src/core/utilities/deserializer.cc @@ -140,6 +140,7 @@ void TaskDeserializer::_unpack(detail::RegionField& value) void TaskDeserializer::_unpack(detail::UnboundRegionField& value) { + unpack(); // dim auto idx = unpack(); auto fid = unpack(); From 17861034d9577f5a57ae1f2860d31ad0fe2be452 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Thu, 31 Aug 2023 23:28:35 -0700 Subject: [PATCH 0196/1425] Try adding a base image name to fix CI (#45) --- .github/workflows/gh-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index 037173828b..1c847f4753 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -15,6 +15,7 @@ on: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BASE_IMAGE: rapidsai/devcontainers:23.06-cpp-cuda11.8-mambaforge-ubuntu22.04 IMAGE_NAME: legate.core.internal_pkg-${{ inputs.build-target }} USE_CUDA: ${{ (inputs.build-target == 'cpu' && 'OFF') || 'ON' }} From 1bed9ba7192102bda85703a247edbabadee09a45 Mon Sep 17 00:00:00 2001 From: Jacob Faibussowitsch Date: Fri, 1 Sep 2023 12:37:07 -0400 Subject: [PATCH 0197/1425] Add more names to gitignore (#50) --- .gitignore | 5 +++-- examples/hello/.gitignore | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 examples/hello/.gitignore diff --git a/.gitignore b/.gitignore index 0f366d85da..1af00dee8c 100644 --- a/.gitignore +++ b/.gitignore @@ -22,10 +22,10 @@ *.dylib install_info.py /dist -/build +build*/ /legion /install* -/_skbuild +_skbuild/ config.mk /docs/legate/core/build /docs/legate/core/source/api/generated @@ -41,3 +41,4 @@ examples/cpp/build tests/cpp/build .legate-test-last-failed out/ +venv/ diff --git a/examples/hello/.gitignore b/examples/hello/.gitignore new file mode 100644 index 0000000000..0d830a4e39 --- /dev/null +++ b/examples/hello/.gitignore @@ -0,0 +1,2 @@ +library.py +legate_library.* From c052801071488849a216f38a96122f6f9957a76f Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 1 Sep 2023 10:46:24 -0700 Subject: [PATCH 0198/1425] C++ Port for GH #821 (#49) --- src/core/data/detail/logical_store.cc | 5 ++- src/core/operation/detail/task.cc | 4 +- src/core/runtime/detail/runtime.cc | 8 +++- src/core/runtime/detail/runtime.h | 4 +- tests/cpp/integration/multi_scalar_out.cc | 51 ++++++++++++---------- tests/cpp/integration/tasks/task_simple.cc | 18 +++++--- 6 files changed, 53 insertions(+), 37 deletions(-) diff --git a/src/core/data/detail/logical_store.cc b/src/core/data/detail/logical_store.cc index bcafd79fbe..5be26fbbae 100644 --- a/src/core/data/detail/logical_store.cc +++ b/src/core/data/detail/logical_store.cc @@ -696,8 +696,9 @@ std::unique_ptr LogicalStore::to_launcher_arg(const Variable* variab int32_t redop) { if (has_scalar_storage()) { - auto has_storage = privilege != WRITE_ONLY; + if (nullptr == launch_domain && REDUCE == privilege) { privilege = READ_WRITE; } auto read_only = privilege == READ_ONLY; + auto has_storage = privilege == READ_ONLY || privilege == READ_WRITE; return std::make_unique(this, read_only, has_storage, redop); } else if (unbound()) { return std::make_unique(this, strategy.find_field_space(variable)); @@ -711,7 +712,7 @@ std::unique_ptr LogicalStore::to_launcher_arg(const Variable* variab if (privilege == REDUCE && store_partition->is_disjoint_for(launch_domain)) { privilege = READ_WRITE; } - if (privilege == WRITE_ONLY) { + if (privilege == WRITE_ONLY || privilege == READ_WRITE) { set_key_partition(variable->operation()->machine(), partition.get()); } return std::make_unique(this, privilege, std::move(proj_info)); diff --git a/src/core/operation/detail/task.cc b/src/core/operation/detail/task.cc index 52e05a1253..f8b1415500 100644 --- a/src/core/operation/detail/task.cc +++ b/src/core/operation/detail/task.cc @@ -180,7 +180,7 @@ void Task::demux_scalar_stores(const Legion::FutureMap& result, const Domain& la if (1 == total) { if (1 == num_scalar_reds) { auto& [store, redop] = scalar_reductions_.front(); - store->set_future(runtime->reduce_future_map(result, redop)); + store->set_future(runtime->reduce_future_map(result, redop, store->get_future())); } else if (can_throw_exception_) { auto* runtime = detail::Runtime::get_runtime(); runtime->record_pending_exception(runtime->reduce_exception_future_map(result)); @@ -194,7 +194,7 @@ void Task::demux_scalar_stores(const Legion::FutureMap& result, const Domain& la uint32_t idx = num_unbound_outs; for (auto& [store, redop] : scalar_reductions_) { auto values = runtime->extract_scalar(result, idx++, launch_domain); - store->set_future(runtime->reduce_future_map(values, redop)); + store->set_future(runtime->reduce_future_map(values, redop, store->get_future())); } if (can_throw_exception_) { auto exn_fm = runtime->extract_scalar(result, idx, launch_domain); diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index 18626e52e7..0ec7b61e69 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -845,13 +845,17 @@ Legion::FutureMap Runtime::extract_scalar(const Legion::FutureMap& result, } Legion::Future Runtime::reduce_future_map(const Legion::FutureMap& future_map, - int32_t reduction_op) const + int32_t reduction_op, + const Legion::Future& init_value) const { return legion_runtime_->reduce_future_map(legion_context_, future_map, reduction_op, false /*deterministic*/, - core_library_->get_mapper_id()); + core_library_->get_mapper_id(), + 0 /*tag*/, + nullptr /*provenance*/, + init_value); } Legion::Future Runtime::reduce_exception_future_map(const Legion::FutureMap& future_map) const diff --git a/src/core/runtime/detail/runtime.h b/src/core/runtime/detail/runtime.h index abce5b34c6..c31c857bb8 100644 --- a/src/core/runtime/detail/runtime.h +++ b/src/core/runtime/detail/runtime.h @@ -233,7 +233,9 @@ class Runtime { Legion::FutureMap extract_scalar(const Legion::FutureMap& result, uint32_t idx, const Legion::Domain& launch_domain) const; - Legion::Future reduce_future_map(const Legion::FutureMap& future_map, int32_t reduction_op) const; + Legion::Future reduce_future_map(const Legion::FutureMap& future_map, + int32_t reduction_op, + const Legion::Future& init_value = Legion::Future()) const; Legion::Future reduce_exception_future_map(const Legion::FutureMap& future_map) const; public: diff --git a/tests/cpp/integration/multi_scalar_out.cc b/tests/cpp/integration/multi_scalar_out.cc index 957dd52184..d99a39ae60 100644 --- a/tests/cpp/integration/multi_scalar_out.cc +++ b/tests/cpp/integration/multi_scalar_out.cc @@ -23,10 +23,8 @@ void test_writer_auto(legate::Library library, { auto runtime = legate::Runtime::get_runtime(); auto task = runtime->create_task(library, task::simple::WRITER); - auto part1 = task.declare_partition(); - auto part2 = task.declare_partition(); - task.add_output(scalar1, part1); - task.add_output(scalar2, part2); + task.add_output(scalar1); + task.add_output(scalar2); runtime->submit(std::move(task)); } @@ -37,36 +35,39 @@ void test_reducer_auto(legate::Library library, { auto runtime = legate::Runtime::get_runtime(); auto task = runtime->create_task(library, task::simple::REDUCER); - auto part1 = task.declare_partition(); - auto part2 = task.declare_partition(); - auto part3 = task.declare_partition(); - task.add_reduction(scalar1, legate::ReductionOpKind::ADD, part1); - task.add_reduction(scalar2, legate::ReductionOpKind::MUL, part2); - task.add_output(store, part3); + task.add_reduction(scalar1, legate::ReductionOpKind::ADD); + task.add_reduction(scalar2, legate::ReductionOpKind::MUL); + task.add_input(store); runtime->submit(std::move(task)); } void test_reducer_manual(legate::Library library, legate::LogicalStore scalar1, - legate::LogicalStore scalar2) + legate::LogicalStore scalar2, + legate::LogicalStore store) { auto runtime = legate::Runtime::get_runtime(); auto task = runtime->create_task(library, task::simple::REDUCER, legate::Shape({2})); task.add_reduction(scalar1, legate::ReductionOpKind::ADD); task.add_reduction(scalar2, legate::ReductionOpKind::MUL); + task.add_input(store.partition_by_tiling({3})); runtime->submit(std::move(task)); } -void print_stores(legate::LogicalStore scalar1, legate::LogicalStore scalar2) +void validate_stores(legate::LogicalStore scalar1, + legate::LogicalStore scalar2, + int32_t to_match1, + int64_t to_match2) { auto runtime = legate::Runtime::get_runtime(); auto p_scalar1 = scalar1.get_physical_store(); auto p_scalar2 = scalar2.get_physical_store(); - auto acc1 = p_scalar1.read_accessor(); - auto acc2 = p_scalar2.read_accessor(); - std::stringstream ss; - ss << static_cast(acc1[{0, 0}]) << " " << acc2[{0, 0, 0}]; - task::simple::logger.print() << ss.str(); + auto acc1 = p_scalar1.read_accessor(); + auto acc2 = p_scalar2.read_accessor(); + auto v1 = acc1[{0, 0}]; + auto v2 = acc2[{0, 0, 0}]; + EXPECT_EQ(v1, to_match1); + EXPECT_EQ(v2, to_match2); } TEST(Integration, MultiScalarOut) @@ -76,15 +77,17 @@ TEST(Integration, MultiScalarOut) auto runtime = legate::Runtime::get_runtime(); auto library = runtime->find_library(task::simple::library_name); - auto scalar1 = runtime->create_store({1, 1}, legate::int8(), true); - auto scalar2 = runtime->create_store({1, 1, 1}, legate::int32(), true); - auto store = runtime->create_store({10}, legate::int64()); + auto scalar1 = runtime->create_store({1, 1}, legate::int32(), true); + auto scalar2 = runtime->create_store({1, 1, 1}, legate::int64(), true); + auto store = runtime->create_store({5}, legate::int64()); + runtime->issue_fill(store, legate::Scalar(int64_t(0))); + test_writer_auto(library, scalar1, scalar2); - print_stores(scalar1, scalar2); + validate_stores(scalar1, scalar2, 10, 20); test_reducer_auto(library, scalar1, scalar2, store); - print_stores(scalar1, scalar2); - test_reducer_manual(library, scalar1, scalar2); - print_stores(scalar1, scalar2); + validate_stores(scalar1, scalar2, 60, 640); + test_reducer_manual(library, scalar1, scalar2, store); + validate_stores(scalar1, scalar2, 110, 20480); } } // namespace multiscalarout diff --git a/tests/cpp/integration/tasks/task_simple.cc b/tests/cpp/integration/tasks/task_simple.cc index e35bb66f49..17d8964c0d 100644 --- a/tests/cpp/integration/tasks/task_simple.cc +++ b/tests/cpp/integration/tasks/task_simple.cc @@ -44,8 +44,8 @@ void register_tasks() auto output1 = context.output(0).data(); auto output2 = context.output(1).data(); - auto acc1 = output1.write_accessor(); - auto acc2 = output2.write_accessor(); + auto acc1 = output1.write_accessor(); + auto acc2 = output2.write_accessor(); acc1[{0, 0}] = 10; acc2[{0, 0, 0}] = 20; @@ -53,14 +53,20 @@ void register_tasks() /*static*/ void ReducerTask::cpu_variant(legate::TaskContext context) { + auto in = context.input(0).data(); auto red1 = context.reduction(0).data(); auto red2 = context.reduction(1).data(); - auto acc1 = red1.reduce_accessor, true, 2>(); - auto acc2 = red2.reduce_accessor, true, 3>(); + auto shape = in.shape<1>(); + if (shape.empty()) return; + + auto red_acc1 = red1.reduce_accessor, true, 2>(); + auto red_acc2 = red2.reduce_accessor, true, 3>(); - acc1[{0, 0}].reduce(10); - acc2[{0, 0, 0}].reduce(2); + for (legate::PointInRectIterator<1> it(shape); it.valid(); ++it) { + red_acc1[{0, 0}].reduce(10); + red_acc2[{0, 0, 0}].reduce(2); + } } } // namespace simple From 573b3a8396b69917ab0612f7eb2b5efed39f7185 Mon Sep 17 00:00:00 2001 From: Parag Kulkarni Date: Tue, 5 Sep 2023 11:05:52 +0530 Subject: [PATCH 0199/1425] - Add cadence to run the script every 15 mins. (#48) --- .github/workflows/merge-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/merge-ci.yml b/.github/workflows/merge-ci.yml index 69f6622d79..89bc66c416 100644 --- a/.github/workflows/merge-ci.yml +++ b/.github/workflows/merge-ci.yml @@ -2,6 +2,8 @@ name: merge-public on: workflow_dispatch: + schedule: + - cron: '*/15 * * * *' # Run every 15 mins jobs: merge: From 9d9166c010204497bf7352dd12dcff87180e03a7 Mon Sep 17 00:00:00 2001 From: Bryan Van de Ven Date: Tue, 5 Sep 2023 15:13:45 -0700 Subject: [PATCH 0200/1425] Convert legate command line opts to machine config (#36) * Convert legate command line opts to machine config * factor out helper funcion * handle eager alloc * use constexpr * include lf:local with eager alloc * allow missing configs with zero values * ensure core config * ensure at least one utility --- src/core/runtime/detail/runtime.cc | 104 +++++++++++++++++++++++++++++ src/core/runtime/detail/runtime.h | 2 + src/core/runtime/runtime.h | 12 ++++ 3 files changed, 118 insertions(+) diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index 0ec7b61e69..c94f8351fd 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -35,6 +35,7 @@ #include "core/task/detail/task_context.h" #include "env_defaults.h" +#include "realm/cmdline.h" #include "realm/network.h" namespace legate { @@ -1010,6 +1011,8 @@ MachineManager* Runtime::machine_manager() const { return machine_manager_; } Legion::Runtime::add_registration_callback(registration_callback); + handle_legate_args(argc, argv); + result = Legion::Runtime::start(argc, argv, true); if (result != 0) { log_legate.error("Legion Runtime failed to start."); @@ -1236,4 +1239,105 @@ void registration_callback_for_python(Legion::Machine machine, Runtime::get_runtime()->initialize(Legion::Runtime::get_context()); } +template +void try_set_property(Runtime& runtime, + const std::string& config_name, + const std::string& property_name, + const Value& value, + const std::string& error_msg) +{ + if (value < 0) { + log_legate.error(error_msg.c_str()); + LEGATE_ABORT; + } + auto config = runtime.get_module_config(config_name); + if (!config && value > 0) { + const std::string msg = error_msg + " (" + config_name + " is not available)"; + log_legate.error(msg.c_str()); + LEGATE_ABORT; + } + auto success = config->set_property(property_name, value); + if (!success) { + log_legate.error(error_msg.c_str()); + LEGATE_ABORT; + } +} + +constexpr long MB = 1024 * 1024; + +void handle_legate_args(int32_t argc, char** argv) +{ + // Realm uses ints rather than unsigned ints + int cpus = DEFAULT_CPUS; + int gpus = DEFAULT_GPUS; + int omps = DEFAULT_OMPS; + int ompthreads = DEFAULT_OMPTHREADS; + int util = DEFAULT_UTILITY; + int sysmem = DEFAULT_SYSMEM; + int numamem = DEFAULT_NUMAMEM; + int fbmem = DEFAULT_FBMEM; + int zcmem = DEFAULT_ZCMEM; + int regmem = DEFAULT_REGMEM; + int eager_alloc_percent = DEFAULT_EAGER_ALLOC_PERCENT; + + Realm::CommandLineParser cp; + cp.add_option_int("--cpus", cpus) + .add_option_int("--gpus", gpus) + .add_option_int("--omps", omps) + .add_option_int("--ompthreads", ompthreads) + .add_option_int("--utility", util) + .add_option_int("--sysmem", sysmem) + .add_option_int("--numamem", numamem) + .add_option_int("--fbmem", fbmem) + .add_option_int("--zcmem", zcmem) + .add_option_int("--regmem", regmem) + .add_option_int("--eager-alloc-percentage", eager_alloc_percent) + .parse_command_line(argc, argv); + + auto rt = Realm::Runtime::get_runtime(); + + // ensure core module + if (!rt.get_module_config("core")) { + log_legate.error("core module config is missing"); + LEGATE_ABORT; + } + + // ensure sensible utility + if (util < 1) { + log_legate.error("--utility must be at least 1"); + LEGATE_ABORT; + } + + // Set core configuration properties + try_set_property(rt, "core", "cpu", cpus, "unable to set --cpus"); + try_set_property(rt, "core", "util", util, "unable to set --utility"); + try_set_property(rt, "core", "sysmem", sysmem * MB, "unable to set --sysmem"); + try_set_property(rt, "core", "regmem", regmem * MB, "unable to set --regmem"); + + // Set CUDA configuration properties + try_set_property(rt, "cuda", "gpu", gpus, "unable to set --gpus"); + try_set_property(rt, "cuda", "fbmem", fbmem * MB, "unable to set --fbmem"); + try_set_property(rt, "cuda", "zcmem", zcmem * MB, "unable to set --zcmem"); + + // Set OpenMP configuration properties + if (omps > 0 && ompthreads == 0) { + log_legate.error("--omps configured with zero threads"); + LEGATE_ABORT; + } + try_set_property(rt, "openmp", "ocpu", omps, "unable to set --omps"); + try_set_property(rt, "openmp", "othr", ompthreads, "unable to set --ompthreads"); + + // Set NUMA configuration properties + try_set_property(rt, "numa", "numamem", numamem * MB, "unable to set --numamem"); + + // eager alloc has to be passed via env var + const char* existing_default_args = getenv("LEGION_DEFAULT_ARGS"); + const std::string eager_alloc_arg = + " -lg:eager_alloc_percentage " + std::to_string(eager_alloc_percent); + const std::string new_default_args = + (existing_default_args == nullptr ? "" : std::string(existing_default_args)) + eager_alloc_arg + + " -lg:local 0"; + setenv("LEGION_DEFAULT_ARGS", new_default_args.c_str(), true); +} + } // namespace legate::detail diff --git a/src/core/runtime/detail/runtime.h b/src/core/runtime/detail/runtime.h index c31c857bb8..923a09b723 100644 --- a/src/core/runtime/detail/runtime.h +++ b/src/core/runtime/detail/runtime.h @@ -335,6 +335,8 @@ void registration_callback_for_python(Legion::Machine machine, Legion::Runtime* legion_runtime, const std::set& local_procs); +void handle_legate_args(int32_t argc, char** argv); + } // namespace legate::detail #include "core/runtime/detail/runtime.inl" diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index e79d014666..34db4a7ffa 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -98,6 +98,18 @@ namespace detail { class Runtime; } // namespace detail +constexpr auto DEFAULT_CPUS = 1; +constexpr auto DEFAULT_GPUS = 0; +constexpr auto DEFAULT_OMPS = 0; +constexpr auto DEFAULT_OMPTHREADS = 2; +constexpr auto DEFAULT_UTILITY = 1; +constexpr auto DEFAULT_SYSMEM = 4000; // MB +constexpr auto DEFAULT_NUMAMEM = 0; // MB +constexpr auto DEFAULT_FBMEM = 4000; // MB +constexpr auto DEFAULT_ZCMEM = 32; // MB +constexpr auto DEFAULT_REGMEM = 0; // MB +constexpr auto DEFAULT_EAGER_ALLOC_PERCENT = 50; + class Runtime { public: /** From 23b934db0191a413b5b3cdd5861c644c1ed2071b Mon Sep 17 00:00:00 2001 From: Jacob Faibussowitsch Date: Wed, 6 Sep 2023 09:46:31 -0400 Subject: [PATCH 0201/1425] Support std::size_t as a typecode, normalize to std::uint64_t (#40) --- src/core/data/scalar.h | 6 ++++- src/core/data/scalar.inl | 48 +++++++++++++++++++++++++++++++++++----- 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/src/core/data/scalar.h b/src/core/data/scalar.h index 8de713a795..4d289a94f5 100644 --- a/src/core/data/scalar.h +++ b/src/core/data/scalar.h @@ -170,7 +170,11 @@ class Scalar { private: static detail::Scalar* create_impl(Type type, const void* data, bool copy); - private: + struct private_tag {}; + + template + Scalar(T value, private_tag); + friend class AutoTask; friend class ManualTask; friend class Runtime; diff --git a/src/core/data/scalar.inl b/src/core/data/scalar.inl index 3f1b8b8543..b8588b4cbb 100644 --- a/src/core/data/scalar.inl +++ b/src/core/data/scalar.inl @@ -12,20 +12,58 @@ #pragma once +#include +#include +#include // std::addressof #include +#include // Useful for IDEs #include "core/data/scalar.h" namespace legate { +namespace detail { + +template +inline constexpr legate::Type::Code canonical_type_code_of() noexcept +{ + using legate::Type; // to disambiguate from legate::detail::Type; + + if constexpr (std::is_same_v) { + static_assert(sizeof(T) == sizeof(std::uint64_t)); + return Type::Code::UINT64; + } else { + constexpr auto ret = legate_type_code_of; + + static_assert(ret != Type::Code::FIXED_ARRAY); + static_assert(ret != Type::Code::STRUCT); + static_assert(ret != Type::Code::STRING); + static_assert(ret != Type::Code::INVALID); + return ret; + } +} + +template +inline decltype(auto) canonical_value_of(T&& v) noexcept +{ + return std::forward(v); +} + +inline std::uint64_t canonical_value_of(std::size_t v) noexcept { return std::uint64_t{v}; } + +} // namespace detail + +template +Scalar::Scalar(T value, private_tag) + : impl_{ + create_impl(primitive_type(detail::canonical_type_code_of()), std::addressof(value), true)} +{ +} + template -Scalar::Scalar(T value) : impl_(create_impl(primitive_type(legate_type_code_of), &value, true)) +Scalar::Scalar(T value) : Scalar{detail::canonical_value_of(std::move(value)), private_tag{}} { - static_assert(legate_type_code_of != Type::Code::FIXED_ARRAY); - static_assert(legate_type_code_of != Type::Code::STRUCT); - static_assert(legate_type_code_of != Type::Code::STRING); - static_assert(legate_type_code_of != Type::Code::INVALID); } template From 53e3620d0589d21c4151de7616a888b24cee4060 Mon Sep 17 00:00:00 2001 From: Joy Shen <110752938+joyshennv@users.noreply.github.com> Date: Fri, 8 Sep 2023 09:21:01 +0800 Subject: [PATCH 0202/1425] Add tests for Point/Rect scalar. (#2) * Add tests for Point/Rect scalar. * Address review comments. --- tests/cpp/unit/scalar.cc | 401 +++++++++++++++++++++++++++++---------- 1 file changed, 301 insertions(+), 100 deletions(-) diff --git a/tests/cpp/unit/scalar.cc b/tests/cpp/unit/scalar.cc index 313fe8b94b..c95d4f0afb 100644 --- a/tests/cpp/unit/scalar.cc +++ b/tests/cpp/unit/scalar.cc @@ -47,6 +47,29 @@ struct __attribute__((packed)) NoPaddingStructData { uint64_t uint64_data; }; +template +struct MultiDimRectStruct { + int64_t lo[DIM]; + int64_t hi[DIM]; +}; + +template +void check_type(T value); +template +void check_struct_type_scalar(T& struct_data, bool align); +template +void check_point_scalar(const int64_t bounds[]); +template +void check_rect_scalar(const int64_t lo[], const int64_t hi[]); +template +void check_rect_bounds(const MultiDimRectStruct& to_compare, + const MultiDimRectStruct& expect); +void check_pack(const legate::Scalar& scalar); +template +void check_pack_point_scalar(); +template +void check_pack_rect_scalar(); + class ScalarUnitTestDeserializer : public legate::BaseDeserializer { public: ScalarUnitTestDeserializer(const void* args, size_t arglen); @@ -60,6 +83,152 @@ ScalarUnitTestDeserializer::ScalarUnitTestDeserializer(const void* args, size_t { } +template +void check_type(T value, legate::Type type) +{ + { + legate::Scalar scalar(value); + EXPECT_EQ(scalar.type().code(), legate::legate_type_code_of); + EXPECT_EQ(scalar.size(), sizeof(T)); + EXPECT_EQ(scalar.value(), value); + EXPECT_EQ(scalar.values().size(), 1); + EXPECT_NE(scalar.ptr(), nullptr); + } + + { + legate::Scalar scalar(value, type); + EXPECT_EQ(scalar.type().code(), type.code()); + EXPECT_EQ(scalar.size(), type.size()); + EXPECT_EQ(scalar.value(), value); + EXPECT_EQ(scalar.values().size(), 1); + EXPECT_NE(scalar.ptr(), nullptr); + } +} + +template +void check_struct_type_scalar(T& struct_data, bool align) +{ + legate::Scalar scalar( + struct_data, legate::struct_type(align, legate::bool_(), legate::int32(), legate::uint64())); + EXPECT_EQ(scalar.type().code(), legate::Type::Code::STRUCT); + EXPECT_EQ(scalar.size(), sizeof(T)); + EXPECT_NE(scalar.ptr(), nullptr); + + // Check value + T actual_data = scalar.value(); + EXPECT_EQ(actual_data.bool_data, struct_data.bool_data); + EXPECT_EQ(actual_data.int32_data, struct_data.int32_data); + EXPECT_EQ(actual_data.uint64_data, struct_data.uint64_data); + + // Check values + legate::Span actual_values(scalar.values()); + legate::Span expected_values = legate::Span(&struct_data, 1); + EXPECT_EQ(actual_values.size(), expected_values.size()); + EXPECT_EQ(actual_values.begin()->bool_data, expected_values.begin()->bool_data); + EXPECT_EQ(actual_values.begin()->int32_data, expected_values.begin()->int32_data); + EXPECT_EQ(actual_values.begin()->uint64_data, expected_values.begin()->uint64_data); + EXPECT_NE(actual_values.ptr(), expected_values.ptr()); +} + +template +void check_point_scalar(const int64_t bounds[]) +{ + auto point = legate::Point(bounds); + legate::Scalar scalar(point); + EXPECT_EQ(scalar.type().code(), legate::Type::Code::FIXED_ARRAY); + auto fixed_type = legate::fixed_array_type(legate::int64(), DIM); + EXPECT_EQ(scalar.size(), fixed_type.size()); + EXPECT_NE(scalar.ptr(), nullptr); + + // Check values + legate::Span expectedValues = legate::Span(bounds, DIM); + legate::Span actualValues(scalar.values()); + for (int i = 0; i < DIM; i++) { EXPECT_EQ(actualValues[i], expectedValues[i]); } + EXPECT_EQ(actualValues.size(), DIM); + EXPECT_EQ(actualValues.size(), expectedValues.size()); + + // Invalid type + EXPECT_THROW(scalar.values(), std::invalid_argument); +} + +template +void check_rect_scalar(const int64_t lo[], const int64_t hi[]) +{ + auto point_lo = legate::Point(lo); + auto point_hi = legate::Point(hi); + auto rect = legate::Rect(point_lo, point_hi); + legate::Scalar scalar(rect); + EXPECT_EQ(scalar.type().code(), legate::Type::Code::STRUCT); + auto struct_type = legate::struct_type(true, + legate::fixed_array_type(legate::int64(), DIM), + legate::fixed_array_type(legate::int64(), DIM)); + EXPECT_EQ(scalar.size(), struct_type.size()); + EXPECT_NE(scalar.ptr(), nullptr); + + // Check values + MultiDimRectStruct actual_data = scalar.value>(); + MultiDimRectStruct expected_data; + for (int i = 0; i < DIM; i++) { + expected_data.lo[i] = lo[i]; + expected_data.hi[i] = hi[i]; + } + check_rect_bounds(actual_data, expected_data); + + legate::Span actual_values(scalar.values>()); + legate::Span expected_values = legate::Span>(&expected_data, 1); + EXPECT_EQ(actual_values.size(), 1); + EXPECT_EQ(actual_values.size(), expected_values.size()); + check_rect_bounds(*actual_values.begin(), *expected_values.begin()); + EXPECT_NE(actual_values.ptr(), expected_values.ptr()); + + // Invalid type + EXPECT_THROW(scalar.value(), std::invalid_argument); + EXPECT_THROW(scalar.values(), std::invalid_argument); +} + +template +void check_rect_bounds(const MultiDimRectStruct& to_compare, + const MultiDimRectStruct& expect) +{ + EXPECT_EQ(std::size(to_compare.lo), std::size(to_compare.hi)); + EXPECT_EQ(std::size(expect.lo), std::size(expect.hi)); + EXPECT_EQ(std::size(to_compare.lo), std::size(expect.lo)); + + for (int i = 0; i < std::size(to_compare.lo); i++) { + EXPECT_EQ(to_compare.lo[i], expect.lo[i]); + EXPECT_EQ(to_compare.hi[i], expect.hi[i]); + } +} + +void check_pack(const legate::Scalar& scalar) +{ + legate::detail::BufferBuilder buf; + scalar.impl()->pack(buf); + auto legion_buffer = buf.to_legion_buffer(); + EXPECT_NE(legion_buffer.get_ptr(), nullptr); + legate::BaseDeserializer deserializer(legion_buffer.get_ptr(), + legion_buffer.get_size()); + auto scalar_unpack = deserializer.unpack_scalar(); + EXPECT_EQ(scalar_unpack->type()->code, scalar.type().code()); + EXPECT_EQ(scalar_unpack->size(), scalar.size()); +} + +template +void check_pack_point_scalar() +{ + auto point = legate::Point::ONES(); + legate::Scalar scalar(point); + check_pack(scalar); +} + +template +void check_pack_rect_scalar() +{ + auto rect = legate::Rect(legate::Point::ZEROES(), legate::Point::ONES()); + legate::Scalar scalar(rect); + check_pack(scalar); +} + TEST(ScalarUnit, CreateWithObject) { // constructor with Scalar object @@ -80,51 +249,61 @@ TEST(ScalarUnit, CreateWithObject) TEST(ScalarUnit, CreateSharedScalar) { - auto data_vec = std::vector(DATA_SIZE, UINT64_VALUE); - const auto* data = data_vec.data(); - legate::Scalar scalar(legate::uint64(), data); - EXPECT_EQ(scalar.type().code(), legate::Type::Code::UINT64); - EXPECT_EQ(scalar.size(), legate::uint64().size()); - EXPECT_EQ(scalar.ptr(), data); - - EXPECT_EQ(scalar.value(), UINT64_VALUE); - legate::Span actualValues(scalar.values()); - legate::Span expectedValues = legate::Span(data, 1); - EXPECT_EQ(*actualValues.begin(), *expectedValues.begin()); - EXPECT_EQ(actualValues.size(), expectedValues.size()); + // unowned + { + auto data_vec = std::vector(DATA_SIZE, UINT64_VALUE); + const auto* data = data_vec.data(); + legate::Scalar scalar(legate::uint64(), data); + EXPECT_EQ(scalar.type().code(), legate::Type::Code::UINT64); + EXPECT_EQ(scalar.size(), legate::uint64().size()); + EXPECT_EQ(scalar.ptr(), data); + + EXPECT_EQ(scalar.value(), UINT64_VALUE); + legate::Span actualValues(scalar.values()); + legate::Span expectedValues = legate::Span(data, 1); + EXPECT_EQ(*actualValues.begin(), *expectedValues.begin()); + EXPECT_EQ(actualValues.size(), expectedValues.size()); - // Invalid type - EXPECT_THROW(scalar.value(), std::invalid_argument); - EXPECT_THROW(scalar.values(), std::invalid_argument); -} + // Invalid type + EXPECT_THROW(scalar.value(), std::invalid_argument); + EXPECT_THROW(scalar.values(), std::invalid_argument); + } -template -void checkType(T value) -{ - legate::Scalar scalar(value); - EXPECT_EQ(scalar.type().code(), legate::legate_type_code_of); - EXPECT_EQ(scalar.size(), sizeof(T)); - EXPECT_EQ(scalar.value(), value); - EXPECT_EQ(scalar.values().size(), 1); - EXPECT_NE(scalar.ptr(), nullptr); + // owned + { + auto data_vec = std::vector(DATA_SIZE, UINT32_VALUE); + const auto* data = data_vec.data(); + legate::Scalar scalar(legate::uint32(), data, true); + EXPECT_NE(scalar.ptr(), data); + } } TEST(ScalarUnit, CreateWithValue) { - checkType(BOOL_VALUE); - checkType(INT8_VALUE); - checkType(INT16_VALUE); - checkType(INT32_VALUE); - checkType(INT64_VALUE); - checkType(UINT8_VALUE); - checkType(UINT16_VALUE); - checkType(UINT32_VALUE); - checkType(UINT64_VALUE); - checkType(FLOAT_VALUE); - checkType(DOUBLE_VALUE); - checkType(FLOAT16_VALUE); - checkType(COMPLEX_FLOAT_VALUE); - checkType(COMPLEX_DOUBLE_VALUE); + check_type(BOOL_VALUE, legate::bool_()); + check_type(INT8_VALUE, legate::int8()); + check_type(INT16_VALUE, legate::int16()); + check_type(INT32_VALUE, legate::int32()); + check_type(INT64_VALUE, legate::int64()); + check_type(UINT8_VALUE, legate::uint8()); + check_type(UINT16_VALUE, legate::uint16()); + check_type(UINT32_VALUE, legate::uint32()); + check_type(UINT64_VALUE, legate::uint64()); + check_type(FLOAT_VALUE, legate::float32()); + check_type(DOUBLE_VALUE, legate::float64()); + check_type(FLOAT16_VALUE, legate::float16()); + check_type(COMPLEX_FLOAT_VALUE, legate::complex64()); + check_type(COMPLEX_DOUBLE_VALUE, legate::complex128()); + + // Invalid type + // Note: test fails + // Expected: legate::Scalar(BOOL_VALUE, legate::primitive_type(legate::Type::Code::INVALID)) + // throws an exception of type std::invalid_argument. Actual: it throws std::out_of_range with + // description "_Map_base::at". EXPECT_THROW(legate::Scalar(BOOL_VALUE, + // legate::primitive_type(legate::Type::Code::INVALID)), std::invalid_argument); + + // Size mismatch + EXPECT_THROW(legate::Scalar(FLOAT16_VALUE, legate::float32()), std::invalid_argument); } TEST(ScalarUnit, CreateWithVector) @@ -141,8 +320,7 @@ TEST(ScalarUnit, CreateWithVector) const auto* data = data_vec.data(); legate::Span expectedValues = legate::Span(data, scalar_data.size()); legate::Span actualValues(scalar.values()); - EXPECT_EQ(*actualValues.begin(), *expectedValues.begin()); - EXPECT_EQ(*actualValues.end(), *expectedValues.end()); + for (int i = 0; i < scalar_data.size(); i++) { EXPECT_EQ(actualValues[i], expectedValues[i]); } EXPECT_EQ(actualValues.size(), expectedValues.size()); // Invalid type @@ -176,53 +354,73 @@ TEST(ScalarUnit, CreateWithStructType) { // with struct padding { - PaddingStructData structData = {BOOL_VALUE, INT32_VALUE, UINT64_VALUE}; - legate::Scalar scalar( - structData, legate::struct_type(true, legate::bool_(), legate::int32(), legate::uint64())); - EXPECT_EQ(scalar.type().code(), legate::Type::Code::STRUCT); - EXPECT_EQ(scalar.size(), sizeof(PaddingStructData)); - EXPECT_NE(scalar.ptr(), nullptr); + PaddingStructData struct_data = {BOOL_VALUE, INT32_VALUE, UINT64_VALUE}; + check_struct_type_scalar(struct_data, true); + } - // Check value - PaddingStructData actualData = scalar.value(); - EXPECT_EQ(actualData.bool_data, structData.bool_data); - EXPECT_EQ(actualData.int32_data, structData.int32_data); - EXPECT_EQ(actualData.uint64_data, structData.uint64_data); + // without struct padding + { + NoPaddingStructData struct_data = {BOOL_VALUE, INT32_VALUE, UINT64_VALUE}; + check_struct_type_scalar(struct_data, false); + } +} - // Check values - legate::Span actualValues(scalar.values()); - legate::Span expectedValues = legate::Span(&structData, 1); - EXPECT_EQ(actualValues.size(), expectedValues.size()); - EXPECT_EQ(actualValues.begin()->bool_data, expectedValues.begin()->bool_data); - EXPECT_EQ(actualValues.begin()->int32_data, expectedValues.begin()->int32_data); - EXPECT_EQ(actualValues.begin()->uint64_data, expectedValues.begin()->uint64_data); - EXPECT_NE(actualValues.ptr(), expectedValues.ptr()); +TEST(ScalarUnit, CreateWithPoint) +{ + { + const int64_t bounds[] = {0}; + check_point_scalar<1>(bounds); } - // without struct padding { - NoPaddingStructData structData = {BOOL_VALUE, INT32_VALUE, UINT64_VALUE}; - legate::Scalar scalar( - structData, legate::struct_type(false, legate::bool_(), legate::int32(), legate::uint64())); - EXPECT_EQ(scalar.type().code(), legate::Type::Code::STRUCT); - EXPECT_EQ(scalar.size(), sizeof(NoPaddingStructData)); - EXPECT_NE(scalar.ptr(), nullptr); + const int64_t bounds[] = {1, 8}; + check_point_scalar<2>(bounds); + } - // Check value - NoPaddingStructData actualData = scalar.value(); - EXPECT_EQ(actualData.bool_data, structData.bool_data); - EXPECT_EQ(actualData.int32_data, structData.int32_data); - EXPECT_EQ(actualData.uint64_data, structData.uint64_data); + { + const int64_t bounds[] = {-10, 2, -1}; + check_point_scalar<3>(bounds); + } - // Check values - legate::Span actualValues(scalar.values()); - legate::Span expectedValues = legate::Span(&structData, 1); - EXPECT_EQ(actualValues.size(), expectedValues.size()); - EXPECT_EQ(actualValues.begin()->bool_data, expectedValues.begin()->bool_data); - EXPECT_EQ(actualValues.begin()->int32_data, expectedValues.begin()->int32_data); - EXPECT_EQ(actualValues.begin()->uint64_data, expectedValues.begin()->uint64_data); - EXPECT_NE(actualValues.ptr(), expectedValues.ptr()); + { + const int64_t bounds[] = {1, 5, 7, 200}; + check_point_scalar<4>(bounds); + } + + // Invalid dim + EXPECT_THROW(legate::Scalar(legate::Point<10>::ONES()), std::out_of_range); +} + +TEST(ScalarUnit, CreateWithRect) +{ + { + const int64_t lo[] = {1}; + const int64_t hi[] = {-9}; + check_rect_scalar<1>(lo, hi); + } + + { + const int64_t lo[] = {-1, 3}; + const int64_t hi[] = {9, -2}; + check_rect_scalar<2>(lo, hi); + } + + { + const int64_t lo[] = {0, 1, 2}; + const int64_t hi[] = {3, 4, 5}; + check_rect_scalar<3>(lo, hi); } + + { + const int64_t lo[] = {-5, 1, -7, 10}; + const int64_t hi[] = {4, 5, 6, 7}; + check_rect_scalar<4>(lo, hi); + } + + // Invalid dim + EXPECT_THROW( + legate::Scalar(legate::Rect<100>(legate::Point<100>::ZEROES(), legate::Point<100>::ZEROES())), + std::out_of_range); } TEST(ScalarUnit, OperatorEqual) @@ -236,37 +434,24 @@ TEST(ScalarUnit, OperatorEqual) EXPECT_EQ(scalar2.values().size(), scalar1.values().size()); } -void checkPack(const legate::Scalar& scalar) -{ - legate::detail::BufferBuilder buf; - scalar.impl()->pack(buf); - auto legion_buffer = buf.to_legion_buffer(); - EXPECT_NE(legion_buffer.get_ptr(), nullptr); - legate::BaseDeserializer deserializer(legion_buffer.get_ptr(), - legion_buffer.get_size()); - auto scalar_unpack = deserializer.unpack_scalar(); - EXPECT_EQ(scalar_unpack->type()->code, scalar.type().code()); - EXPECT_EQ(scalar_unpack->size(), scalar.size()); -} - TEST(ScalarUnit, Pack) { // test pack for a fixed array scalar { legate::Scalar scalar(std::vector{UINT32_VALUE, UINT32_VALUE}); - checkPack(scalar); + check_pack(scalar); } // test pack for a single value scalar { legate::Scalar scalar(BOOL_VALUE); - checkPack(scalar); + check_pack(scalar); } // test pack for string scalar { legate::Scalar scalar(STRING_VALUE); - checkPack(scalar); + check_pack(scalar); } // test pack for padding struct type scalar @@ -274,7 +459,7 @@ TEST(ScalarUnit, Pack) PaddingStructData structData = {BOOL_VALUE, INT32_VALUE, UINT64_VALUE}; legate::Scalar scalar( structData, legate::struct_type(true, legate::bool_(), legate::int32(), legate::uint64())); - checkPack(scalar); + check_pack(scalar); } // test pack for no padding struct type scalar @@ -282,7 +467,7 @@ TEST(ScalarUnit, Pack) NoPaddingStructData structData = {BOOL_VALUE, INT32_VALUE, UINT64_VALUE}; legate::Scalar scalar( structData, legate::struct_type(false, legate::bool_(), legate::int32(), legate::uint64())); - checkPack(scalar); + check_pack(scalar); } // test pack for a shared scalar @@ -290,14 +475,30 @@ TEST(ScalarUnit, Pack) auto data_vec = std::vector(DATA_SIZE, UINT64_VALUE); const auto* data = data_vec.data(); legate::Scalar scalar(legate::uint64(), data); - checkPack(scalar); + check_pack(scalar); } // test pack for scalar created by another scalar { legate::Scalar scalar1(INT32_VALUE); legate::Scalar scalar2(scalar1); - checkPack(scalar2); + check_pack(scalar2); + } + + // test pack for scalar created by point + { + check_pack_point_scalar<1>(); + check_pack_point_scalar<2>(); + check_pack_point_scalar<3>(); + check_pack_point_scalar<4>(); + } + + // test pack for scalar created by rect + { + check_pack_rect_scalar<1>(); + check_pack_rect_scalar<2>(); + check_pack_rect_scalar<3>(); + check_pack_rect_scalar<4>(); } } } // namespace scalar_test From d7f6f16a9ce59857fdd5a3139c6102545b27d0b8 Mon Sep 17 00:00:00 2001 From: Joy Shen <110752938+joyshennv@users.noreply.github.com> Date: Fri, 8 Sep 2023 09:25:02 +0800 Subject: [PATCH 0203/1425] Add unit tests for APIs in type_info.h/type_traits.h. (#10) * Add unit tests for APIs in type_info.h/type_traits.h. * Address review comments; Add test cases for list type. --- tests/cpp/unit/type.cc | 630 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 630 insertions(+) create mode 100644 tests/cpp/unit/type.cc diff --git a/tests/cpp/unit/type.cc b/tests/cpp/unit/type.cc new file mode 100644 index 0000000000..cddc95ed18 --- /dev/null +++ b/tests/cpp/unit/type.cc @@ -0,0 +1,630 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include +#include "legate.h" + +namespace type_test { +constexpr int32_t GLOBAL_OP_ID = 0x1F; + +const std::vector PRIMITIVE_TYPE = {legate::bool_(), + legate::int16(), + legate::int32(), + legate::int64(), + legate::uint8(), + legate::uint16(), + legate::uint32(), + legate::uint64(), + legate::float16(), + legate::float32(), + legate::float64(), + legate::complex64(), + legate::complex128()}; + +template +void test_primitive_type(const legate::Type& type, std::string type_string) +{ + EXPECT_EQ(type.code(), legate::legate_type_code_of); + EXPECT_EQ(type.size(), sizeof(T)); + EXPECT_EQ(type.alignment(), sizeof(T)); + EXPECT_FALSE(type.variable_size()); + EXPECT_TRUE(type.is_primitive()); + EXPECT_EQ(type.to_string(), type_string); + legate::Type other(type); + EXPECT_EQ(other, type); +} + +void test_string_type(const legate::Type& type) +{ + EXPECT_EQ(type.code(), legate::Type::Code::STRING); + EXPECT_THROW(type.size(), std::invalid_argument); + EXPECT_EQ(type.alignment(), 0); + EXPECT_TRUE(type.variable_size()); + EXPECT_FALSE(type.is_primitive()); + EXPECT_EQ(type.to_string(), "string"); + legate::Type other(type); + EXPECT_EQ(other, type); +} + +void test_fixed_array_type(const legate::Type& type, + const legate::Type& element_type, + uint32_t N, + std::string to_string) +{ + EXPECT_EQ(type.code(), legate::Type::Code::FIXED_ARRAY); + EXPECT_EQ(type.size(), element_type.size() * N); + EXPECT_EQ(type.alignment(), element_type.alignment()); + EXPECT_FALSE(type.variable_size()); + EXPECT_FALSE(type.is_primitive()); + EXPECT_EQ(type.to_string(), to_string); + legate::Type other(type); + EXPECT_EQ(other, type); + + auto fixed_array_type = type.as_fixed_array_type(); + EXPECT_EQ(fixed_array_type.num_elements(), N); + EXPECT_EQ(fixed_array_type.element_type(), element_type); +} + +void test_struct_type(const legate::Type& type, + bool aligned, + uint32_t size, + uint32_t alignment, + std::string to_string, + std::vector field_types, + std::vector offsets) +{ + EXPECT_EQ(type.code(), legate::Type::Code::STRUCT); + EXPECT_EQ(type.size(), size); + EXPECT_EQ(type.alignment(), alignment); + EXPECT_FALSE(type.variable_size()); + EXPECT_FALSE(type.is_primitive()); + legate::Type other(type); + EXPECT_EQ(other, type); + + auto struct_type = type.as_struct_type(); + EXPECT_EQ(type.to_string(), to_string); + EXPECT_EQ(struct_type.aligned(), aligned); + EXPECT_EQ(struct_type.num_fields(), field_types.size()); + for (uint32_t idx = 0; idx < field_types.size(); ++idx) { + EXPECT_EQ(struct_type.field_type(idx), field_types.at(idx)); + } +} + +void test_list_type(const legate::Type& element_type, std::string to_string) +{ + auto type = legate::list_type(element_type); + EXPECT_EQ(type.code(), legate::Type::Code::LIST); + EXPECT_THROW(type.size(), std::invalid_argument); + EXPECT_EQ(type.alignment(), 0); + EXPECT_TRUE(type.variable_size()); + EXPECT_FALSE(type.is_primitive()); + legate::Type other(type); + EXPECT_EQ(other, type); + + auto list_type = type.as_list_type(); + EXPECT_EQ(list_type.to_string(), to_string); + EXPECT_EQ(list_type.element_type(), element_type); +} + +void test_reduction_op(const legate::Type& type) +{ + type.record_reduction_operator(static_cast(legate::ReductionOpKind::ADD), GLOBAL_OP_ID); + EXPECT_EQ(type.find_reduction_operator(static_cast(legate::ReductionOpKind::ADD)), + GLOBAL_OP_ID); + EXPECT_EQ(type.find_reduction_operator(legate::ReductionOpKind::ADD), GLOBAL_OP_ID); + + // repeat records for same type + EXPECT_THROW(type.record_reduction_operator(static_cast(legate::ReductionOpKind::ADD), + GLOBAL_OP_ID), + std::invalid_argument); + + // reduction op doesn't exist + EXPECT_THROW(type.find_reduction_operator(static_cast(legate::ReductionOpKind::SUB)), + std::invalid_argument); + EXPECT_THROW(type.find_reduction_operator(legate::ReductionOpKind::SUB), std::invalid_argument); +} + +TEST(TypeUnit, PrimitiveType) +{ + test_primitive_type(legate::bool_(), "bool"); + test_primitive_type(legate::int8(), "int8"); + test_primitive_type(legate::int16(), "int16"); + test_primitive_type(legate::int32(), "int32"); + test_primitive_type(legate::int64(), "int64"); + test_primitive_type(legate::uint8(), "uint8"); + test_primitive_type(legate::uint16(), "uint16"); + test_primitive_type(legate::uint32(), "uint32"); + test_primitive_type(legate::uint64(), "uint64"); + test_primitive_type<__half>(legate::float16(), "float16"); + test_primitive_type(legate::float32(), "float32"); + test_primitive_type(legate::float64(), "float64"); + test_primitive_type>(legate::complex64(), "complex64"); + test_primitive_type>(legate::complex128(), "complex128"); + + EXPECT_EQ(legate::bool_(), legate::primitive_type(legate::Type::Code::BOOL)); + EXPECT_EQ(legate::int8(), legate::primitive_type(legate::Type::Code::INT8)); + EXPECT_EQ(legate::int16(), legate::primitive_type(legate::Type::Code::INT16)); + EXPECT_EQ(legate::int32(), legate::primitive_type(legate::Type::Code::INT32)); + EXPECT_EQ(legate::int64(), legate::primitive_type(legate::Type::Code::INT64)); + EXPECT_EQ(legate::uint8(), legate::primitive_type(legate::Type::Code::UINT8)); + EXPECT_EQ(legate::uint16(), legate::primitive_type(legate::Type::Code::UINT16)); + EXPECT_EQ(legate::uint32(), legate::primitive_type(legate::Type::Code::UINT32)); + EXPECT_EQ(legate::uint64(), legate::primitive_type(legate::Type::Code::UINT64)); + EXPECT_EQ(legate::float16(), legate::primitive_type(legate::Type::Code::FLOAT16)); + EXPECT_EQ(legate::float32(), legate::primitive_type(legate::Type::Code::FLOAT32)); + EXPECT_EQ(legate::float64(), legate::primitive_type(legate::Type::Code::FLOAT64)); + EXPECT_EQ(legate::complex64(), legate::primitive_type(legate::Type::Code::COMPLEX64)); + EXPECT_EQ(legate::complex128(), legate::primitive_type(legate::Type::Code::COMPLEX128)); + + // Invalid code + // Note: issue #20, throws std::out_of_range with description "_Map_base::at" + // legate::primitive_type(legate::Type::Code::FIXED_ARRAY); + // legate::primitive_type(legate::Type::Code::STRUCT); + // legate::primitive_type(legate::Type::Code::STRING); + // legate::primitive_type(legate::Type::Code::LIST); + // legate::primitive_type(legate::Type::Code::INVALID); +} + +TEST(TypeUnit, StringType) { test_string_type(legate::string_type()); } + +TEST(TypeUnit, FixedArrayType) +{ + // element type is a primitive type + { + uint32_t N = 10; + auto element_type = legate::uint64(); + auto fixed_array_type = legate::fixed_array_type(element_type, N); + test_fixed_array_type(fixed_array_type, element_type, N, "uint64[10]"); + } + + // element type is not a primitive type + { + uint32_t N = 10; + auto element_type = legate::fixed_array_type(legate::uint16(), N); + auto fixed_array_type = legate::fixed_array_type(element_type, N); + test_fixed_array_type(fixed_array_type, element_type, N, "uint16[10][10]"); + } + + // N > 0xFFU + { + uint32_t N = 256; + auto element_type = legate::float64(); + auto fixed_array_type = legate::fixed_array_type(element_type, N); + test_fixed_array_type(fixed_array_type, element_type, N, "float64[256]"); + } + + // N = 0 + { + // Note: No exceptions throwed when N of fixed array type is 0 + // Expected: legate::fixed_array_type(legate::int64(), 0) throws an exception of type + // std::out_of_range. Actual: it throws nothing. + // EXPECT_THROW(legate::fixed_array_type(legate::int64(), 0), std::out_of_range); + } + + // element type has variable size + EXPECT_THROW(legate::fixed_array_type(legate::string_type(), 10), std::invalid_argument); + + // invalid casts + EXPECT_THROW(legate::uint32().as_fixed_array_type(), std::invalid_argument); +} + +TEST(TypeUnit, StructType) +{ + // aligned + { + auto type = legate::struct_type(true, legate::int16(), legate::bool_(), legate::float64()); + std::vector field_types = {legate::int16(), legate::bool_(), legate::float64()}; + std::vector offsets = {0, 2, 8}; + // size: 16 = 8 (int16_t bool) + 8 (double) + test_struct_type( + type, true, 16, sizeof(double), "{int16:0,bool:2,float64:8}", field_types, offsets); + } + + // aligned + { + std::vector field_types = {legate::bool_(), legate::float64(), legate::int16()}; + auto type = legate::struct_type(field_types, true); + std::vector offsets = {0, 8, 16}; + // size: 24 = 8 (bool) + 8 (double) + 8 (int16_t) + test_struct_type( + type, true, 24, sizeof(double), "{bool:0,float64:8,int16:16}", field_types, offsets); + } + + // not aligned + { + auto type = legate::struct_type(false, legate::int16(), legate::bool_(), legate::float64()); + std::vector field_types = {legate::int16(), legate::bool_(), legate::float64()}; + std::vector offsets = {0, 2, 3}; + auto size = sizeof(int16_t) + sizeof(bool) + sizeof(double); + test_struct_type(type, false, size, 1, "{int16:0,bool:2,float64:3}", field_types, offsets); + } + + // not aligned + { + std::vector field_types = {legate::bool_(), legate::float64(), legate::int16()}; + auto type = legate::struct_type(field_types); + std::vector offsets = {0, 1, 9}; + auto size = sizeof(int16_t) + sizeof(bool) + sizeof(double); + test_struct_type(type, false, size, 1, "{bool:0,float64:1,int16:9}", field_types, offsets); + } + + // field type has variable size + EXPECT_THROW(legate::struct_type(true, legate::string_type(), legate::int16()), + std::runtime_error); + EXPECT_THROW(legate::struct_type(false, legate::string_type()), std::runtime_error); + EXPECT_THROW(legate::struct_type(true), std::invalid_argument); + EXPECT_THROW(legate::struct_type(false), std::invalid_argument); + + // invalid casts + EXPECT_THROW(legate::uint32().as_struct_type(), std::invalid_argument); +} + +TEST(TypeUnit, PointType) +{ + for (uint32_t idx = 1; idx <= LEGATE_MAX_DIM; ++idx) { + auto type = legate::point_type(idx); + test_fixed_array_type(type, legate::int64(), idx, "int64[" + std::to_string(idx) + "]"); + EXPECT_TRUE(legate::is_point_type(type, idx)); + } + + // point type checks + EXPECT_FALSE(legate::is_point_type(legate::point_type(1), 2)); + EXPECT_FALSE(legate::is_point_type(legate::point_type(1), 0)); + EXPECT_FALSE(legate::is_point_type(legate::point_type(1), -1)); + + EXPECT_FALSE(legate::is_point_type(legate::rect_type(1), 1)); + EXPECT_FALSE(legate::is_point_type(legate::string_type(), 4)); + // Note: There are several cases in the runtime where 64-bit integers need to be interpreted as 1D + // points, so we need a more lenient type checking in those cases. + EXPECT_TRUE(legate::is_point_type(legate::int64(), 1)); + + // invalid dim + EXPECT_THROW(legate::point_type(-1), std::out_of_range); + EXPECT_THROW(legate::point_type(0), std::out_of_range); + EXPECT_THROW(legate::point_type(LEGATE_MAX_DIM + 1), std::out_of_range); +} + +TEST(TypeUnit, RectType) +{ + for (uint32_t idx = 1; idx <= LEGATE_MAX_DIM; ++idx) { + auto type = legate::rect_type(idx); + std::vector field_types = {legate::point_type(idx), legate::point_type(idx)}; + std::vector offsets = {0, (uint32_t)(sizeof(uint64_t)) * idx}; + auto full_size = (field_types.size() * sizeof(uint64_t)) * idx; + auto to_string = "{int64[" + std::to_string(idx) + "]:0,int64[" + std::to_string(idx) + + "]:" + std::to_string(idx * sizeof(int64_t)) + "}"; + test_struct_type(type, true, full_size, sizeof(uint64_t), to_string, field_types, offsets); + EXPECT_TRUE(legate::is_rect_type(type, idx)); + } + + // rect type checks + EXPECT_FALSE(legate::is_rect_type(legate::rect_type(1), 2)); + EXPECT_FALSE(legate::is_rect_type(legate::rect_type(1), 0)); + EXPECT_FALSE(legate::is_rect_type(legate::rect_type(1), -1)); + + EXPECT_FALSE(legate::is_rect_type(legate::point_type(1), 1)); + EXPECT_FALSE(legate::is_rect_type(legate::fixed_array_type(legate::int64(), 1), 1)); + EXPECT_FALSE(legate::is_rect_type(legate::int64(), 1)); + + // invalid dim + EXPECT_THROW(legate::rect_type(-1), std::out_of_range); + EXPECT_THROW(legate::rect_type(0), std::out_of_range); + EXPECT_THROW(legate::rect_type(LEGATE_MAX_DIM + 1), std::out_of_range); +} + +TEST(TypeUnit, ListType) +{ + test_list_type(legate::bool_(), "list(bool)"); + test_list_type(legate::int8(), "list(int8)"); + test_list_type(legate::int16(), "list(int16)"); + test_list_type(legate::int32(), "list(int32)"); + test_list_type(legate::int64(), "list(int64)"); + test_list_type(legate::uint8(), "list(uint8)"); + test_list_type(legate::uint16(), "list(uint16)"); + test_list_type(legate::uint32(), "list(uint32)"); + test_list_type(legate::uint64(), "list(uint64)"); + test_list_type(legate::float16(), "list(float16)"); + test_list_type(legate::float32(), "list(float32)"); + test_list_type(legate::float64(), "list(float64)"); + test_list_type(legate::complex64(), "list(complex64)"); + test_list_type(legate::complex128(), "list(complex128)"); + test_list_type(legate::struct_type(true, legate::bool_(), legate::int32()), + "list({bool:0,int32:4})"); + test_list_type(legate::struct_type(false, legate::bool_(), legate::int32()), + "list({bool:0,int32:1})"); + test_list_type(legate::point_type(1), "list(int64[1])"); + test_list_type(legate::rect_type(2), "list({int64[2]:0,int64[2]:16})"); + + // variable size types + EXPECT_THROW(legate::list_type(legate::string_type()), std::runtime_error); + EXPECT_THROW(legate::list_type(legate::list_type(legate::uint32())), std::runtime_error); + + // invald casts + EXPECT_THROW(legate::string_type().as_struct_type(), std::invalid_argument); +} + +TEST(TypeUnit, Uid) +{ + // fixed array type + for (uint32_t idx = 0; idx < PRIMITIVE_TYPE.size(); ++idx) { + auto element_type = PRIMITIVE_TYPE.at(idx); + auto N = idx + 1; + auto fixed_array_type = legate::fixed_array_type(element_type, N); + EXPECT_EQ(fixed_array_type.uid() & 0x00FF, static_cast(element_type.code())); + EXPECT_EQ(fixed_array_type.uid() >> 8, N); + EXPECT_EQ(fixed_array_type.as_fixed_array_type().num_elements(), N); + + EXPECT_NE(fixed_array_type.uid(), legate::string_type().uid()); + EXPECT_NE(fixed_array_type.uid(), static_cast(element_type.code())); + + auto same_array_type = legate::fixed_array_type(element_type, N); + EXPECT_EQ(fixed_array_type.uid(), same_array_type.uid()); + + auto diff_array_type = legate::fixed_array_type(element_type, N + 1); + EXPECT_NE(fixed_array_type.uid(), diff_array_type.uid()); + + // N > 0xFFU + auto big_array_type = legate::fixed_array_type(element_type, 300); + EXPECT_TRUE(big_array_type.uid() >= 0x10000); + + // array of array types + auto array_of_array_type1 = + legate::fixed_array_type(legate::fixed_array_type(element_type, N), N); + auto array_of_array_type2 = + legate::fixed_array_type(legate::fixed_array_type(element_type, N), N); + EXPECT_NE(array_of_array_type1.uid(), array_of_array_type2.uid()); + + // array of point types + auto dim = N % LEGATE_MAX_DIM + 1; + auto array_of_point_type1 = legate::fixed_array_type(legate::point_type(dim), N); + auto array_of_point_type2 = legate::fixed_array_type(legate::point_type(dim), N); + EXPECT_NE(array_of_point_type1.uid(), array_of_point_type2.uid()); + + // array of rect types + auto array_of_rect_type1 = legate::fixed_array_type(legate::rect_type(dim), N); + auto array_of_rect_type2 = legate::fixed_array_type(legate::rect_type(dim), N); + EXPECT_NE(array_of_rect_type1.uid(), array_of_rect_type2.uid()); + + // array of struct types + auto array_of_struct_type1 = + legate::fixed_array_type(legate::struct_type(true, element_type), N); + auto array_of_struct_type2 = + legate::fixed_array_type(legate::struct_type(true, element_type), N); + EXPECT_NE(array_of_struct_type1.uid(), array_of_struct_type2.uid()); + } + + // string type + EXPECT_EQ(legate::string_type().uid(), static_cast(legate::Type::Code::STRING)); + + // struct type + for (uint32_t idx = 0; idx < PRIMITIVE_TYPE.size(); ++idx) { + auto element_type = PRIMITIVE_TYPE.at(idx); + auto struct_type1 = legate::struct_type(true, element_type); + auto struct_type2 = legate::struct_type(true, element_type); + EXPECT_NE(struct_type1.uid(), struct_type2.uid()); + EXPECT_TRUE(struct_type1.uid() >= 0x10000); + EXPECT_TRUE(struct_type2.uid() >= 0x10000); + + EXPECT_NE(struct_type1.uid(), legate::string_type().uid()); + EXPECT_NE(struct_type2.uid(), legate::string_type().uid()); + EXPECT_NE(struct_type1.uid(), static_cast(element_type.code())); + EXPECT_NE(struct_type2.uid(), static_cast(element_type.code())); + } + + // list type + for (uint32_t idx = 0; idx < PRIMITIVE_TYPE.size(); ++idx) { + auto element_type = PRIMITIVE_TYPE.at(idx); + auto list_type1 = legate::list_type(element_type); + auto list_type2 = legate::list_type(element_type); + EXPECT_NE(list_type1.uid(), list_type2.uid()); + EXPECT_TRUE(list_type1.uid() >= 0x10000); + EXPECT_TRUE(list_type2.uid() >= 0x10000); + + EXPECT_NE(list_type1.uid(), legate::string_type().uid()); + EXPECT_NE(list_type2.uid(), legate::string_type().uid()); + EXPECT_NE(list_type1.uid(), static_cast(element_type.code())); + EXPECT_NE(list_type2.uid(), static_cast(element_type.code())); + } +} + +TEST(TypeUnit, ReductionOperator) +{ + test_reduction_op(legate::string_type()); + test_reduction_op(legate::fixed_array_type(legate::int64(), 10)); + test_reduction_op(legate::struct_type(true, legate::int64(), legate::bool_(), legate::float64())); +} + +TEST(TypeUnit, TypeCodeOf) +{ + EXPECT_EQ(legate::legate_type_code_of, legate::Type::Code::INVALID); + EXPECT_EQ(legate::legate_type_code_of, legate::Type::Code::BOOL); + EXPECT_EQ(legate::legate_type_code_of, legate::Type::Code::INT8); + EXPECT_EQ(legate::legate_type_code_of, legate::Type::Code::INT16); + EXPECT_EQ(legate::legate_type_code_of, legate::Type::Code::INT32); + EXPECT_EQ(legate::legate_type_code_of, legate::Type::Code::INT64); + EXPECT_EQ(legate::legate_type_code_of, legate::Type::Code::UINT8); + EXPECT_EQ(legate::legate_type_code_of, legate::Type::Code::UINT16); + EXPECT_EQ(legate::legate_type_code_of, legate::Type::Code::UINT32); + EXPECT_EQ(legate::legate_type_code_of, legate::Type::Code::UINT64); + EXPECT_EQ(legate::legate_type_code_of<__half>, legate::Type::Code::FLOAT16); + EXPECT_EQ(legate::legate_type_code_of, legate::Type::Code::FLOAT32); + EXPECT_EQ(legate::legate_type_code_of, legate::Type::Code::FLOAT64); + EXPECT_EQ(legate::legate_type_code_of>, legate::Type::Code::COMPLEX64); + EXPECT_EQ(legate::legate_type_code_of>, legate::Type::Code::COMPLEX128); + EXPECT_EQ(legate::legate_type_code_of, legate::Type::Code::STRING); +} + +TEST(TypeUnit, TypeOf) +{ + EXPECT_TRUE((std::is_same_v, bool>)); + EXPECT_TRUE((std::is_same_v, int8_t>)); + EXPECT_TRUE((std::is_same_v, int16_t>)); + EXPECT_TRUE((std::is_same_v, int32_t>)); + EXPECT_TRUE((std::is_same_v, int64_t>)); + EXPECT_TRUE((std::is_same_v, uint8_t>)); + EXPECT_TRUE((std::is_same_v, uint16_t>)); + EXPECT_TRUE((std::is_same_v, uint32_t>)); + EXPECT_TRUE((std::is_same_v, uint64_t>)); + EXPECT_TRUE((std::is_same_v, __half>)); + EXPECT_TRUE((std::is_same_v, float>)); + EXPECT_TRUE((std::is_same_v, double>)); + EXPECT_TRUE( + (std::is_same_v, complex>)); + EXPECT_TRUE( + (std::is_same_v, complex>)); + EXPECT_TRUE((std::is_same_v, std::string>)); + + EXPECT_TRUE((std::is_same_v, void>)); + EXPECT_TRUE((std::is_same_v, void>)); + EXPECT_TRUE((std::is_same_v, void>)); + EXPECT_TRUE((std::is_same_v, void>)); +} + +TEST(TypeUnit, TypeUtils) +{ + // is_integral + EXPECT_TRUE(legate::is_integral::value); + EXPECT_TRUE(legate::is_integral::value); + EXPECT_TRUE(legate::is_integral::value); + EXPECT_TRUE(legate::is_integral::value); + EXPECT_TRUE(legate::is_integral::value); + EXPECT_TRUE(legate::is_integral::value); + EXPECT_TRUE(legate::is_integral::value); + EXPECT_TRUE(legate::is_integral::value); + EXPECT_TRUE(legate::is_integral::value); + + EXPECT_FALSE(legate::is_integral::value); + EXPECT_FALSE(legate::is_integral::value); + EXPECT_FALSE(legate::is_integral::value); + EXPECT_FALSE(legate::is_integral::value); + EXPECT_FALSE(legate::is_integral::value); + + EXPECT_FALSE(legate::is_integral::value); + EXPECT_FALSE(legate::is_integral::value); + EXPECT_FALSE(legate::is_integral::value); + EXPECT_FALSE(legate::is_integral::value); + EXPECT_FALSE(legate::is_integral::value); + + // is_signed + EXPECT_TRUE(legate::is_signed::value); + EXPECT_TRUE(legate::is_signed::value); + EXPECT_TRUE(legate::is_signed::value); + EXPECT_TRUE(legate::is_signed::value); + EXPECT_TRUE(legate::is_signed::value); + EXPECT_TRUE(legate::is_signed::value); + // Note: issue #5 float16 turns out to be unsigned + // EXPECT_TRUE(legate::is_signed::value); + + EXPECT_FALSE(legate::is_signed::value); + EXPECT_FALSE(legate::is_signed::value); + EXPECT_FALSE(legate::is_signed::value); + EXPECT_FALSE(legate::is_signed::value); + EXPECT_FALSE(legate::is_signed::value); + EXPECT_FALSE(legate::is_signed::value); + EXPECT_FALSE(legate::is_signed::value); + + EXPECT_FALSE(legate::is_signed::value); + EXPECT_FALSE(legate::is_signed::value); + EXPECT_FALSE(legate::is_signed::value); + EXPECT_FALSE(legate::is_signed::value); + EXPECT_FALSE(legate::is_signed::value); + + // is_unsigned + EXPECT_TRUE(legate::is_unsigned::value); + EXPECT_TRUE(legate::is_unsigned::value); + EXPECT_TRUE(legate::is_unsigned::value); + EXPECT_TRUE(legate::is_unsigned::value); + EXPECT_TRUE(legate::is_unsigned::value); + + EXPECT_FALSE(legate::is_unsigned::value); + EXPECT_FALSE(legate::is_unsigned::value); + EXPECT_FALSE(legate::is_unsigned::value); + EXPECT_FALSE(legate::is_unsigned::value); + EXPECT_FALSE(legate::is_unsigned::value); + EXPECT_FALSE(legate::is_unsigned::value); + EXPECT_FALSE(legate::is_unsigned::value); + EXPECT_FALSE(legate::is_unsigned::value); + EXPECT_FALSE(legate::is_unsigned::value); + + EXPECT_FALSE(legate::is_unsigned::value); + EXPECT_FALSE(legate::is_unsigned::value); + EXPECT_FALSE(legate::is_unsigned::value); + EXPECT_FALSE(legate::is_unsigned::value); + EXPECT_FALSE(legate::is_unsigned::value); + + // is_floating_point + EXPECT_TRUE(legate::is_floating_point::value); + EXPECT_TRUE(legate::is_floating_point::value); + EXPECT_TRUE(legate::is_floating_point::value); + + EXPECT_FALSE(legate::is_floating_point::value); + EXPECT_FALSE(legate::is_floating_point::value); + EXPECT_FALSE(legate::is_floating_point::value); + EXPECT_FALSE(legate::is_floating_point::value); + EXPECT_FALSE(legate::is_floating_point::value); + EXPECT_FALSE(legate::is_floating_point::value); + EXPECT_FALSE(legate::is_floating_point::value); + EXPECT_FALSE(legate::is_floating_point::value); + EXPECT_FALSE(legate::is_floating_point::value); + EXPECT_FALSE(legate::is_floating_point::value); + EXPECT_FALSE(legate::is_floating_point::value); + + EXPECT_FALSE(legate::is_floating_point::value); + EXPECT_FALSE(legate::is_floating_point::value); + EXPECT_FALSE(legate::is_floating_point::value); + EXPECT_FALSE(legate::is_floating_point::value); + EXPECT_FALSE(legate::is_floating_point::value); + + // is_complex + EXPECT_TRUE(legate::is_complex::value); + EXPECT_TRUE(legate::is_complex::value); + + EXPECT_FALSE(legate::is_complex::value); + EXPECT_FALSE(legate::is_complex::value); + EXPECT_FALSE(legate::is_complex::value); + EXPECT_FALSE(legate::is_complex::value); + EXPECT_FALSE(legate::is_complex::value); + EXPECT_FALSE(legate::is_complex::value); + EXPECT_FALSE(legate::is_complex::value); + EXPECT_FALSE(legate::is_complex::value); + EXPECT_FALSE(legate::is_complex::value); + EXPECT_FALSE(legate::is_complex::value); + EXPECT_FALSE(legate::is_complex::value); + EXPECT_FALSE(legate::is_complex::value); + + EXPECT_FALSE(legate::is_complex::value); + EXPECT_FALSE(legate::is_complex::value); + EXPECT_FALSE(legate::is_complex::value); + EXPECT_FALSE(legate::is_complex::value); + EXPECT_FALSE(legate::is_complex::value); + + // is_complex_type + EXPECT_TRUE(legate::is_complex_type>::value); + EXPECT_TRUE(legate::is_complex_type>::value); + + EXPECT_FALSE(legate::is_complex_type::value); + EXPECT_FALSE(legate::is_complex_type::value); + EXPECT_FALSE(legate::is_complex_type::value); + EXPECT_FALSE(legate::is_complex_type::value); + EXPECT_FALSE(legate::is_complex_type::value); + EXPECT_FALSE(legate::is_complex_type::value); + EXPECT_FALSE(legate::is_complex_type::value); + EXPECT_FALSE(legate::is_complex_type::value); + EXPECT_FALSE(legate::is_complex_type::value); + EXPECT_FALSE(legate::is_complex_type<__half>::value); + EXPECT_FALSE(legate::is_complex_type::value); + EXPECT_FALSE(legate::is_complex_type::value); + + EXPECT_FALSE(legate::is_complex_type::value); + EXPECT_FALSE(legate::is_complex_type::value); +} +} // namespace type_test From 3d611fe78218a4cd6e96af2b69f247fdaf4edc12 Mon Sep 17 00:00:00 2001 From: Eric Niebler Date: Fri, 8 Sep 2023 07:56:13 -0700 Subject: [PATCH 0204/1425] change the first argument of fold_cuda from LHS& to RHS& (#55) --- src/core/runtime/library.inl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/runtime/library.inl b/src/core/runtime/library.inl index f248d37b03..41c3aff67e 100644 --- a/src/core/runtime/library.inl +++ b/src/core/runtime/library.inl @@ -40,7 +40,7 @@ class CUDAReductionOpWrapper : public T { } template - __device__ static void fold_cuda(typename T::LHS& lhs, typename T::RHS rhs) + __device__ static void fold_cuda(typename T::RHS& lhs, typename T::RHS rhs) { T::template fold(lhs, rhs); } From e393f05a7f91b5640764200f76922d5ee10129b9 Mon Sep 17 00:00:00 2001 From: Parag Kulkarni Date: Fri, 8 Sep 2023 21:14:31 +0530 Subject: [PATCH 0205/1425] Run this workflow only for nv-legate user (#59) --- .github/workflows/merge-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/merge-ci.yml b/.github/workflows/merge-ci.yml index 89bc66c416..ea06eb5fd7 100644 --- a/.github/workflows/merge-ci.yml +++ b/.github/workflows/merge-ci.yml @@ -7,6 +7,7 @@ on: jobs: merge: + if: github.repository_owner == 'nv-legate' runs-on: ubuntu-latest steps: - name: Legate Core Internal repository From 2d0a02ad4c1f295fb447a2f50882caffb30057b4 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 8 Sep 2023 09:06:23 -0700 Subject: [PATCH 0206/1425] Make sure we don't try to configure Realm modules that don't exist (#58) * Make sure we don't configure nonexistent modules * Use Legate flags in test scripts * Make scales template arguments * Minor tweak to the comment --- src/core/runtime/detail/runtime.cc | 107 +++++++++++++++++++---------- src/core/runtime/runtime.h | 12 ---- tests/cpp/run.py | 2 +- tests/cpp/run.sh | 4 +- 4 files changed, 73 insertions(+), 52 deletions(-) diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index c94f8351fd..b018776b55 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -12,6 +12,8 @@ #include "core/runtime/detail/runtime.h" +#include + #include "core/comm/comm.h" #include "core/data/detail/array_tasks.h" #include "core/data/detail/logical_array.h" @@ -1239,20 +1241,36 @@ void registration_callback_for_python(Legion::Machine machine, Runtime::get_runtime()->initialize(Legion::Runtime::get_context()); } -template +// Simple wrapper for variables with default values +template +class VarWithDefault { + public: + VAL value() const { return (has_value() ? value_ : DEFAULT) * SCALE; } + bool has_value() const { return value_ != UNSET; } + VAL& ref() { return value_; } + + private: + static constexpr VAL UNSET{std::numeric_limits::max()}; + VAL value_{UNSET}; +}; + +template void try_set_property(Runtime& runtime, - const std::string& config_name, + const std::string& module_name, const std::string& property_name, - const Value& value, + const VarWithDefault& var, const std::string& error_msg) { + auto value = var.value(); if (value < 0) { log_legate.error(error_msg.c_str()); LEGATE_ABORT; } - auto config = runtime.get_module_config(config_name); - if (!config && value > 0) { - const std::string msg = error_msg + " (" + config_name + " is not available)"; + auto config = runtime.get_module_config(module_name); + if (nullptr == config) { + // If the variable doesn't have a value, we don't care if the module is nonexistent + if (!var.has_value()) { return; } + const std::string msg = error_msg + " (the " + module_name + " module is not available)"; log_legate.error(msg.c_str()); LEGATE_ABORT; } @@ -1263,35 +1281,50 @@ void try_set_property(Runtime& runtime, } } -constexpr long MB = 1024 * 1024; +namespace { + +constexpr int64_t DEFAULT_CPUS = 1; +constexpr int64_t DEFAULT_GPUS = 0; +constexpr int64_t DEFAULT_OMPS = 0; +constexpr int64_t DEFAULT_OMPTHREADS = 2; +constexpr int64_t DEFAULT_UTILITY = 1; +constexpr int64_t DEFAULT_SYSMEM = 4000; // MB +constexpr int64_t DEFAULT_NUMAMEM = 0; // MB +constexpr int64_t DEFAULT_FBMEM = 4000; // MB +constexpr int64_t DEFAULT_ZCMEM = 32; // MB +constexpr int64_t DEFAULT_REGMEM = 0; // MB +constexpr int64_t DEFAULT_EAGER_ALLOC_PERCENT = 50; +constexpr int64_t MB = 1024 * 1024; + +} // namespace void handle_legate_args(int32_t argc, char** argv) { // Realm uses ints rather than unsigned ints - int cpus = DEFAULT_CPUS; - int gpus = DEFAULT_GPUS; - int omps = DEFAULT_OMPS; - int ompthreads = DEFAULT_OMPTHREADS; - int util = DEFAULT_UTILITY; - int sysmem = DEFAULT_SYSMEM; - int numamem = DEFAULT_NUMAMEM; - int fbmem = DEFAULT_FBMEM; - int zcmem = DEFAULT_ZCMEM; - int regmem = DEFAULT_REGMEM; - int eager_alloc_percent = DEFAULT_EAGER_ALLOC_PERCENT; + VarWithDefault cpus; + VarWithDefault gpus; + VarWithDefault omps; + VarWithDefault ompthreads; + VarWithDefault util; + VarWithDefault sysmem; + VarWithDefault numamem; + VarWithDefault fbmem; + VarWithDefault zcmem; + VarWithDefault regmem; + VarWithDefault eager_alloc_percent; Realm::CommandLineParser cp; - cp.add_option_int("--cpus", cpus) - .add_option_int("--gpus", gpus) - .add_option_int("--omps", omps) - .add_option_int("--ompthreads", ompthreads) - .add_option_int("--utility", util) - .add_option_int("--sysmem", sysmem) - .add_option_int("--numamem", numamem) - .add_option_int("--fbmem", fbmem) - .add_option_int("--zcmem", zcmem) - .add_option_int("--regmem", regmem) - .add_option_int("--eager-alloc-percentage", eager_alloc_percent) + cp.add_option_int("--cpus", cpus.ref()) + .add_option_int("--gpus", gpus.ref()) + .add_option_int("--omps", omps.ref()) + .add_option_int("--ompthreads", ompthreads.ref()) + .add_option_int("--utility", util.ref()) + .add_option_int("--sysmem", sysmem.ref()) + .add_option_int("--numamem", numamem.ref()) + .add_option_int("--fbmem", fbmem.ref()) + .add_option_int("--zcmem", zcmem.ref()) + .add_option_int("--regmem", regmem.ref()) + .add_option_int("--eager-alloc-percentage", eager_alloc_percent.ref()) .parse_command_line(argc, argv); auto rt = Realm::Runtime::get_runtime(); @@ -1303,7 +1336,7 @@ void handle_legate_args(int32_t argc, char** argv) } // ensure sensible utility - if (util < 1) { + if (util.value() < 1) { log_legate.error("--utility must be at least 1"); LEGATE_ABORT; } @@ -1311,16 +1344,16 @@ void handle_legate_args(int32_t argc, char** argv) // Set core configuration properties try_set_property(rt, "core", "cpu", cpus, "unable to set --cpus"); try_set_property(rt, "core", "util", util, "unable to set --utility"); - try_set_property(rt, "core", "sysmem", sysmem * MB, "unable to set --sysmem"); - try_set_property(rt, "core", "regmem", regmem * MB, "unable to set --regmem"); + try_set_property(rt, "core", "sysmem", sysmem, "unable to set --sysmem"); + try_set_property(rt, "core", "regmem", regmem, "unable to set --regmem"); // Set CUDA configuration properties try_set_property(rt, "cuda", "gpu", gpus, "unable to set --gpus"); - try_set_property(rt, "cuda", "fbmem", fbmem * MB, "unable to set --fbmem"); - try_set_property(rt, "cuda", "zcmem", zcmem * MB, "unable to set --zcmem"); + try_set_property(rt, "cuda", "fbmem", fbmem, "unable to set --fbmem"); + try_set_property(rt, "cuda", "zcmem", zcmem, "unable to set --zcmem"); // Set OpenMP configuration properties - if (omps > 0 && ompthreads == 0) { + if (omps.value() > 0 && ompthreads.value() == 0) { log_legate.error("--omps configured with zero threads"); LEGATE_ABORT; } @@ -1328,12 +1361,12 @@ void handle_legate_args(int32_t argc, char** argv) try_set_property(rt, "openmp", "othr", ompthreads, "unable to set --ompthreads"); // Set NUMA configuration properties - try_set_property(rt, "numa", "numamem", numamem * MB, "unable to set --numamem"); + try_set_property(rt, "numa", "numamem", numamem, "unable to set --numamem"); // eager alloc has to be passed via env var const char* existing_default_args = getenv("LEGION_DEFAULT_ARGS"); const std::string eager_alloc_arg = - " -lg:eager_alloc_percentage " + std::to_string(eager_alloc_percent); + " -lg:eager_alloc_percentage " + std::to_string(eager_alloc_percent.value()); const std::string new_default_args = (existing_default_args == nullptr ? "" : std::string(existing_default_args)) + eager_alloc_arg + " -lg:local 0"; diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 34db4a7ffa..e79d014666 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -98,18 +98,6 @@ namespace detail { class Runtime; } // namespace detail -constexpr auto DEFAULT_CPUS = 1; -constexpr auto DEFAULT_GPUS = 0; -constexpr auto DEFAULT_OMPS = 0; -constexpr auto DEFAULT_OMPTHREADS = 2; -constexpr auto DEFAULT_UTILITY = 1; -constexpr auto DEFAULT_SYSMEM = 4000; // MB -constexpr auto DEFAULT_NUMAMEM = 0; // MB -constexpr auto DEFAULT_FBMEM = 4000; // MB -constexpr auto DEFAULT_ZCMEM = 32; // MB -constexpr auto DEFAULT_REGMEM = 0; // MB -constexpr auto DEFAULT_EAGER_ALLOC_PERCENT = 50; - class Runtime { public: /** diff --git a/tests/cpp/run.py b/tests/cpp/run.py index 5cb8489585..a927035c6b 100755 --- a/tests/cpp/run.py +++ b/tests/cpp/run.py @@ -80,7 +80,7 @@ def is_launcher_var(name: str) -> bool: test_command += [config.binary_path] test_command += [f"--gtest_filter={test_name}"] - test_command += ["-ll:cpu", str(config.cpus)] + test_command += ["--cpus", str(config.cpus)] test_command += extra_args if test_name in test_args_dict: diff --git a/tests/cpp/run.sh b/tests/cpp/run.sh index d5f8f86f6e..deb3c6f71d 100755 --- a/tests/cpp/run.sh +++ b/tests/cpp/run.sh @@ -2,12 +2,12 @@ if [ $# -eq 0 ] then - REALM_BACKTRACE=1 LEGATE_TEST=1 python run.py + REALM_BACKTRACE=1 LEGATE_TEST=1 python run.py --cpus 4 elif [ $# -ge 1 ] && [ "$1" = "ctest" ] then echo "Using ctest" cd build - REALM_BACKTRACE=1 LEGATE_TEST=1 LEGION_DEFAULT_ARGS="-ll:cpu 4" ctest --output-on-failure "$@" + REALM_BACKTRACE=1 LEGATE_TEST=1 ctest --output-on-failure "$@" else echo "Invalid arguments" fi From d2d9ae2db9a413d9aa44a324ee5e7b7a296187de Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 8 Sep 2023 09:06:47 -0700 Subject: [PATCH 0207/1425] Minor fixes (#60) * float16 is a signed type * Missing error checks in the type constructors * Re-enable unit tests for types * Type checks for fills * Raise exceptions for manual tasks with empty launch shapes * Error check for the tiling --- src/core/data/detail/logical_store.cc | 3 ++ src/core/operation/detail/copy.cc | 4 +-- src/core/operation/detail/fill.cc | 7 ++++- src/core/operation/detail/gather.cc | 2 +- src/core/operation/detail/scatter.cc | 2 +- src/core/operation/detail/scatter_gather.cc | 2 +- src/core/runtime/detail/runtime.cc | 1 + src/core/type/detail/type_info.cc | 12 +++++++- src/core/type/type_traits.h | 4 +++ tests/cpp/build.sh | 2 +- tests/cpp/integration/fill.cc | 19 ++++++++++-- tests/cpp/unit/type.cc | 32 +++++++++------------ 12 files changed, 60 insertions(+), 30 deletions(-) diff --git a/src/core/data/detail/logical_store.cc b/src/core/data/detail/logical_store.cc index 5be26fbbae..66b7bd32cb 100644 --- a/src/core/data/detail/logical_store.cc +++ b/src/core/data/detail/logical_store.cc @@ -456,6 +456,9 @@ std::shared_ptr LogicalStore::partition_by_tiling(Shape t std::to_string(extents().size()) + "-tuple, got a " + std::to_string(tile_shape.size()) + "-tuple"); } + if (tile_shape.volume() == 0) { + throw std::invalid_argument("Tile shape must have a volume greater than 0"); + } auto color_shape = apply([](auto c, auto t) { return (c + t - 1) / t; }, extents(), tile_shape); auto partition = create_tiling(std::move(tile_shape), std::move(color_shape)); return create_partition(std::move(partition), true); diff --git a/src/core/operation/detail/copy.cc b/src/core/operation/detail/copy.cc index 2feb94cf29..e755703d3c 100644 --- a/src/core/operation/detail/copy.cc +++ b/src/core/operation/detail/copy.cc @@ -38,8 +38,8 @@ Copy::Copy(std::shared_ptr target, void Copy::validate() { - if (source_.store->type() != target_.store->type()) { - throw std::invalid_argument("Source and targets must have the same type"); + if (*source_.store->type() != *target_.store->type()) { + throw std::invalid_argument("Source and target must have the same type"); } auto validate_store = [](auto* store) { if (store->unbound() || store->has_scalar_storage() || store->transformed()) { diff --git a/src/core/operation/detail/fill.cc b/src/core/operation/detail/fill.cc index a8e9ae3bc1..093afb00bc 100644 --- a/src/core/operation/detail/fill.cc +++ b/src/core/operation/detail/fill.cc @@ -37,7 +37,12 @@ Fill::Fill(std::shared_ptr&& lhs, throw std::runtime_error("Fill value should be a Future-back store"); } -void Fill::validate() {} +void Fill::validate() +{ + if (*lhs_->type() != *value_->type()) { + throw std::invalid_argument("Fill value and target must have the same type"); + } +} void Fill::launch(Strategy* strategy) { diff --git a/src/core/operation/detail/gather.cc b/src/core/operation/detail/gather.cc index 3dc03f86d0..03b17b0d5c 100644 --- a/src/core/operation/detail/gather.cc +++ b/src/core/operation/detail/gather.cc @@ -40,7 +40,7 @@ Gather::Gather(std::shared_ptr target, void Gather::validate() { - if (source_.store->type() != target_.store->type()) { + if (*source_.store->type() != *target_.store->type()) { throw std::invalid_argument("Source and targets must have the same type"); } auto validate_store = [](auto* store) { diff --git a/src/core/operation/detail/scatter.cc b/src/core/operation/detail/scatter.cc index 251c1a98c1..0a88be1543 100644 --- a/src/core/operation/detail/scatter.cc +++ b/src/core/operation/detail/scatter.cc @@ -40,7 +40,7 @@ Scatter::Scatter(std::shared_ptr target, void Scatter::validate() { - if (source_.store->type() != target_.store->type()) { + if (*source_.store->type() != *target_.store->type()) { throw std::invalid_argument("Source and targets must have the same type"); } auto validate_store = [](auto* store) { diff --git a/src/core/operation/detail/scatter_gather.cc b/src/core/operation/detail/scatter_gather.cc index a1e6223f68..e348adaf14 100644 --- a/src/core/operation/detail/scatter_gather.cc +++ b/src/core/operation/detail/scatter_gather.cc @@ -53,7 +53,7 @@ void ScatterGather::set_target_indirect_out_of_range(bool flag) void ScatterGather::validate() { - if (source_.store->type() != target_.store->type()) { + if (*source_.store->type() != *target_.store->type()) { throw std::invalid_argument("Source and targets must have the same type"); } auto validate_store = [](auto* store) { diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index b018776b55..40ea40ddb7 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -199,6 +199,7 @@ std::unique_ptr Runtime::create_task(const Library* library, int64_t task_id, const Shape& launch_shape) { + if (launch_shape.volume() == 0) { throw std::invalid_argument("Launch shape must not be empty"); } auto machine = slice_machine_for_task(library, task_id); auto task = new ManualTask(library, task_id, launch_shape, next_unique_id_++, std::move(machine)); return std::unique_ptr(task); diff --git a/src/core/type/detail/type_info.cc b/src/core/type/detail/type_info.cc index 8960e51b42..bb24fb155f 100644 --- a/src/core/type/detail/type_info.cc +++ b/src/core/type/detail/type_info.cc @@ -260,7 +260,16 @@ bool StringType::equal(const Type& other) const { return code == other.code; } std::shared_ptr primitive_type(Type::Code code) { - return std::make_shared(code); + static std::unordered_map> cache{}; + if (SIZEOF.find(code) == SIZEOF.end()) { + throw std::invalid_argument(std::to_string(static_cast(code)) + + " is not a valid type code for a primitive type"); + } + auto finder = cache.find(code); + if (finder != cache.end()) { return finder->second; } + auto result = std::make_shared(code); + cache[code] = result; + return result; } ListType::ListType(int32_t uid, std::shared_ptr element_type) @@ -309,6 +318,7 @@ std::shared_ptr string_type() std::shared_ptr fixed_array_type(std::shared_ptr element_type, uint32_t N) { + if (N == 0) { throw std::out_of_range("Size of array must be greater than 0"); } // We use UIDs of the following format for "common" fixed array types // 1B 1B // +--------+-------------------+ diff --git a/src/core/type/type_traits.h b/src/core/type/type_traits.h index 3b2f820351..21162a76a3 100644 --- a/src/core/type/type_traits.h +++ b/src/core/type/type_traits.h @@ -169,6 +169,10 @@ template struct is_signed { static constexpr bool value = std::is_signed>::value; }; +template <> +struct is_signed { + static constexpr bool value = true; +}; /** * @ingroup util diff --git a/tests/cpp/build.sh b/tests/cpp/build.sh index cb953ded04..3a2ecf0bd2 100755 --- a/tests/cpp/build.sh +++ b/tests/cpp/build.sh @@ -4,4 +4,4 @@ legate_root=`python -c 'import legate.install_info as i; from pathlib import Pat echo "Using Legate at $legate_root" rm -rf build cmake -B build -S . -D legate_core_ROOT="$legate_root" -D CMAKE_BUILD_TYPE=Debug -cmake --build build -j 8 +cmake --build build --parallel 80 diff --git a/tests/cpp/integration/fill.cc b/tests/cpp/integration/fill.cc index 4c47e3f2e7..a333d33d1d 100644 --- a/tests/cpp/integration/fill.cc +++ b/tests/cpp/integration/fill.cc @@ -163,7 +163,18 @@ void test_fill_slice(int32_t dim) check_output_slice(lhs, v1, v2, offset); } -TEST(Integration, FillIndex) +void test_invalid() +{ + auto runtime = legate::Runtime::get_runtime(); + auto store1 = runtime->create_store({10, 10}, legate::int64()); + auto store2 = runtime->create_store(10.0); + auto scalar_v = legate::Scalar(10.0); + + EXPECT_THROW(runtime->issue_fill(store1, store2), std::invalid_argument); + EXPECT_THROW(runtime->issue_fill(store1, scalar_v), std::invalid_argument); +} + +TEST(Fill, Index) { legate::Core::perform_registration(); test_fill_index(1); @@ -171,7 +182,7 @@ TEST(Integration, FillIndex) test_fill_index(3); } -TEST(Integration, FillSingle) +TEST(Fill, Single) { legate::Core::perform_registration(); test_fill_single(1); @@ -179,7 +190,7 @@ TEST(Integration, FillSingle) test_fill_single(3); } -TEST(Integration, FillSlice) +TEST(Fill, Slice) { legate::Core::perform_registration(); test_fill_slice(1); @@ -187,4 +198,6 @@ TEST(Integration, FillSlice) test_fill_slice(3); } +TEST(Fill, Invalid) { test_invalid(); } + } // namespace fill diff --git a/tests/cpp/unit/type.cc b/tests/cpp/unit/type.cc index cddc95ed18..f3760b8443 100644 --- a/tests/cpp/unit/type.cc +++ b/tests/cpp/unit/type.cc @@ -165,13 +165,11 @@ TEST(TypeUnit, PrimitiveType) EXPECT_EQ(legate::complex64(), legate::primitive_type(legate::Type::Code::COMPLEX64)); EXPECT_EQ(legate::complex128(), legate::primitive_type(legate::Type::Code::COMPLEX128)); - // Invalid code - // Note: issue #20, throws std::out_of_range with description "_Map_base::at" - // legate::primitive_type(legate::Type::Code::FIXED_ARRAY); - // legate::primitive_type(legate::Type::Code::STRUCT); - // legate::primitive_type(legate::Type::Code::STRING); - // legate::primitive_type(legate::Type::Code::LIST); - // legate::primitive_type(legate::Type::Code::INVALID); + EXPECT_THROW(legate::primitive_type(legate::Type::Code::FIXED_ARRAY), std::invalid_argument); + EXPECT_THROW(legate::primitive_type(legate::Type::Code::STRUCT), std::invalid_argument); + EXPECT_THROW(legate::primitive_type(legate::Type::Code::STRING), std::invalid_argument); + EXPECT_THROW(legate::primitive_type(legate::Type::Code::LIST), std::invalid_argument); + EXPECT_THROW(legate::primitive_type(legate::Type::Code::INVALID), std::invalid_argument); } TEST(TypeUnit, StringType) { test_string_type(legate::string_type()); } @@ -180,34 +178,31 @@ TEST(TypeUnit, FixedArrayType) { // element type is a primitive type { - uint32_t N = 10; - auto element_type = legate::uint64(); + uint32_t N = 10; + auto element_type = legate::uint64(); auto fixed_array_type = legate::fixed_array_type(element_type, N); test_fixed_array_type(fixed_array_type, element_type, N, "uint64[10]"); } // element type is not a primitive type { - uint32_t N = 10; - auto element_type = legate::fixed_array_type(legate::uint16(), N); + uint32_t N = 10; + auto element_type = legate::fixed_array_type(legate::uint16(), N); auto fixed_array_type = legate::fixed_array_type(element_type, N); test_fixed_array_type(fixed_array_type, element_type, N, "uint16[10][10]"); } // N > 0xFFU { - uint32_t N = 256; - auto element_type = legate::float64(); + uint32_t N = 256; + auto element_type = legate::float64(); auto fixed_array_type = legate::fixed_array_type(element_type, N); test_fixed_array_type(fixed_array_type, element_type, N, "float64[256]"); } // N = 0 { - // Note: No exceptions throwed when N of fixed array type is 0 - // Expected: legate::fixed_array_type(legate::int64(), 0) throws an exception of type - // std::out_of_range. Actual: it throws nothing. - // EXPECT_THROW(legate::fixed_array_type(legate::int64(), 0), std::out_of_range); + EXPECT_THROW(legate::fixed_array_type(legate::int64(), 0), std::out_of_range); } // element type has variable size @@ -521,8 +516,7 @@ TEST(TypeUnit, TypeUtils) EXPECT_TRUE(legate::is_signed::value); EXPECT_TRUE(legate::is_signed::value); EXPECT_TRUE(legate::is_signed::value); - // Note: issue #5 float16 turns out to be unsigned - // EXPECT_TRUE(legate::is_signed::value); + EXPECT_TRUE(legate::is_signed::value); EXPECT_FALSE(legate::is_signed::value); EXPECT_FALSE(legate::is_signed::value); From e3f90c4fa475eec6b55fa34e2c883edb4c9db4c3 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 8 Sep 2023 11:21:05 -0700 Subject: [PATCH 0208/1425] Scaling constraints (#61) * Scale constraints * Fix the scale constraint test --- src/core/data/logical_store.h | 2 +- src/core/partitioning/constraint.cc | 5 + src/core/partitioning/constraint.h | 24 +++ src/core/partitioning/detail/constraint.cc | 50 ++++++ src/core/partitioning/detail/constraint.h | 48 +++++- .../partitioning/detail/constraint_solver.cc | 16 ++ src/core/partitioning/partition.cc | 24 +++ src/core/partitioning/partition.h | 15 ++ tests/cpp/integration/scale_constraints.cc | 143 ++++++++++++++++++ 9 files changed, 324 insertions(+), 3 deletions(-) create mode 100644 tests/cpp/integration/scale_constraints.cc diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index 2dd6899207..21444791fd 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -164,7 +164,7 @@ class LogicalStore { /** * @brief Projects out a dimension of the store. * - * Each dimension @f$@f$, where @f$i@f$ > `dim`, is mapped to dimension @f$i-1@f$ in a returned + * Each dimension @f$i@f$, where @f$i@f$ > `dim`, is mapped to dimension @f$i-1@f$ in a returned * store. A returned store provides a view to the input store where the values are on hyperplane * @f$x_\mathtt{dim} = \mathtt{index}@f$. * diff --git a/src/core/partitioning/constraint.cc b/src/core/partitioning/constraint.cc index 7607d9dcef..a31bdeb9dd 100644 --- a/src/core/partitioning/constraint.cc +++ b/src/core/partitioning/constraint.cc @@ -40,4 +40,9 @@ Constraint image(Variable var_function, Variable var_range) return Constraint(detail::image(var_function.impl(), var_range.impl()).release()); } +Constraint scale(const Shape& factors, Variable var_smaller, Variable var_bigger) +{ + return Constraint(detail::scale(factors, var_smaller.impl(), var_bigger.impl()).release()); +} + } // namespace legate diff --git a/src/core/partitioning/constraint.h b/src/core/partitioning/constraint.h index 381269b5a7..ce4575ceba 100644 --- a/src/core/partitioning/constraint.h +++ b/src/core/partitioning/constraint.h @@ -12,6 +12,7 @@ #pragma once +#include "core/data/shape.h" #include "core/utilities/memory.h" #include "core/utilities/tuple.h" @@ -104,4 +105,27 @@ Constraint broadcast(Variable variable, const tuple& axes); */ Constraint image(Variable var_function, Variable var_range); +/** + * @ingroup partitioning + * @brief Creates a scaling constraint between partitions + * + * If two stores `A` and `B` are constrained by a scaling constraint + * + * `legate::scale(S, pA, pB)` + * + * where `pA` and `pB ` are partition symbols for `A` and `B`, respectively, `A` and `B` will be + * partitioned such that each pair of sub-stores `Ak` and `Bk` satisfy the following property: + * + * @f$\mathtt{S} \cdot \mathit{dom}(\mathtt{Ak}) \cap \mathit{dom}(\mathtt{B}) \subseteq @f$ + * @f$\mathit{dom}(\mathtt{Bk})@f$ + * + * @param factors Scaling factors + * @param var_smaller Partition symbol for the smaller store (i.e., the one whose extents are + * scaled) + * @param var_bigger Partition symbol of the bigger store + * + * @return Scaling constraint + */ +Constraint scale(const Shape& factors, Variable var_smaller, Variable var_bigger); + } // namespace legate diff --git a/src/core/partitioning/detail/constraint.cc b/src/core/partitioning/detail/constraint.cc index e46b2e55cd..08cb501387 100644 --- a/src/core/partitioning/detail/constraint.cc +++ b/src/core/partitioning/detail/constraint.cc @@ -149,6 +149,49 @@ std::unique_ptr ImageConstraint::resolve(const detail::Strategy& stra } } +ScaleConstraint::ScaleConstraint(const Shape& factors, + const Variable* var_smaller, + const Variable* var_bigger) + : factors_(factors), var_smaller_(var_smaller), var_bigger_(var_bigger) +{ +} + +void ScaleConstraint::find_partition_symbols(std::vector& partition_symbols) const +{ + partition_symbols.push_back(var_smaller_); + partition_symbols.push_back(var_bigger_); +} + +void ScaleConstraint::validate() const +{ + auto smaller = var_smaller_->operation()->find_store(var_smaller_); + auto bigger = var_bigger_->operation()->find_store(var_bigger_); + + if (smaller->dim() != bigger->dim()) { + throw std::invalid_argument( + "Scaling constraint requires the stores to have the same number of dimensions"); + } + + if (smaller->dim() != factors_.size()) { + throw std::invalid_argument( + "Scaling constraint requires the number of factors to match the number of dimensions"); + } +} + +std::string ScaleConstraint::to_string() const +{ + std::stringstream ss; + ss << "ScaleConstraint(" << factors_ << ", " << var_smaller_->to_string() << ", " + << var_bigger_->to_string() << ")"; + return std::move(ss).str(); +} + +std::unique_ptr ScaleConstraint::resolve(const detail::Strategy& strategy) const +{ + auto src_part = strategy[var_smaller()]; + return src_part->scale(factors_); +} + std::unique_ptr align(const Variable* lhs, const Variable* rhs) { return std::make_unique(lhs, rhs); @@ -165,6 +208,13 @@ std::unique_ptr image(const Variable* var_function, const Varia return std::make_unique(var_function, var_range); } +std::unique_ptr scale(const Shape& factors, + const Variable* var_smaller, + const Variable* var_bigger) +{ + return std::make_unique(factors, var_smaller, var_bigger); +} + } // namespace legate::detail // explicitly instantiate the deleter for std::unique_ptr diff --git a/src/core/partitioning/detail/constraint.h b/src/core/partitioning/detail/constraint.h index 1d8f6cd703..71cb7d08b6 100644 --- a/src/core/partitioning/detail/constraint.h +++ b/src/core/partitioning/detail/constraint.h @@ -12,10 +12,9 @@ #pragma once -#include #include -#include +#include "core/data/shape.h" #include "core/utilities/tuple.h" namespace legate { @@ -31,6 +30,7 @@ class Broadcast; struct Constraint; class ImageConstraint; class Literal; +class ScaleConstraint; class Variable; struct Expr { @@ -117,6 +117,7 @@ struct Constraint { ALIGNMENT = 0, BROADCAST = 1, IMAGE = 2, + SCALE = 3, }; virtual ~Constraint() {} virtual void find_partition_symbols(std::vector& partition_symbols) const = 0; @@ -126,6 +127,7 @@ struct Constraint { virtual const Alignment* as_alignment() const = 0; virtual const Broadcast* as_broadcast() const = 0; virtual const ImageConstraint* as_image_constraint() const = 0; + virtual const ScaleConstraint* as_scale_constraint() const = 0; }; class Alignment : public Constraint { @@ -148,6 +150,7 @@ class Alignment : public Constraint { const Alignment* as_alignment() const override { return this; } const Broadcast* as_broadcast() const override { return nullptr; } const ImageConstraint* as_image_constraint() const override { return nullptr; } + const ScaleConstraint* as_scale_constraint() const override { return nullptr; } public: const Variable* lhs() const { return lhs_; } @@ -181,6 +184,7 @@ class Broadcast : public Constraint { const Alignment* as_alignment() const override { return nullptr; } const Broadcast* as_broadcast() const override { return this; } const ImageConstraint* as_image_constraint() const override { return nullptr; } + const ScaleConstraint* as_scale_constraint() const override { return nullptr; } public: const Variable* variable() const { return variable_; } @@ -211,6 +215,7 @@ class ImageConstraint : public Constraint { const Alignment* as_alignment() const override { return nullptr; } const Broadcast* as_broadcast() const override { return nullptr; } const ImageConstraint* as_image_constraint() const override { return this; } + const ScaleConstraint* as_scale_constraint() const override { return nullptr; } public: const Variable* var_function() const { return var_function_; } @@ -224,10 +229,49 @@ class ImageConstraint : public Constraint { const Variable* var_range_; }; +class ScaleConstraint : public Constraint { + public: + ScaleConstraint(const Shape& factors, const Variable* var_smaller, const Variable* var_bigger); + + public: + Kind kind() const override { return Kind::SCALE; } + + public: + void find_partition_symbols(std::vector& partition_symbols) const override; + + public: + void validate() const override; + + public: + std::string to_string() const override; + + public: + const Alignment* as_alignment() const override { return nullptr; } + const Broadcast* as_broadcast() const override { return nullptr; } + const ImageConstraint* as_image_constraint() const override { return nullptr; } + const ScaleConstraint* as_scale_constraint() const override { return this; } + + public: + const Variable* var_smaller() const { return var_smaller_; } + const Variable* var_bigger() const { return var_bigger_; } + + public: + std::unique_ptr resolve(const Strategy& strategy) const; + + private: + Shape factors_; + const Variable* var_smaller_; + const Variable* var_bigger_; +}; + std::unique_ptr align(const Variable* lhs, const Variable* rhs); std::unique_ptr broadcast(const Variable* variable, const tuple& axes); std::unique_ptr image(const Variable* var_function, const Variable* var_range); +std::unique_ptr scale(const Shape& factors, + const Variable* var_smaller, + const Variable* var_bigger); + } // namespace legate::detail diff --git a/src/core/partitioning/detail/constraint_solver.cc b/src/core/partitioning/detail/constraint_solver.cc index 1286934f3c..06564ea325 100644 --- a/src/core/partitioning/detail/constraint_solver.cc +++ b/src/core/partitioning/detail/constraint_solver.cc @@ -159,6 +159,9 @@ void ConstraintSolver::solve_constraints() auto handle_image_constraint = [&](const ImageConstraint* image_constraint) { is_dependent_[*image_constraint->var_range()] = true; }; + auto handle_scale_constraint = [&](const ScaleConstraint* scale_constraint) { + is_dependent_[*scale_constraint->var_bigger()] = true; + }; // Reflect each constraint to the solver state for (auto& constraint : constraints_) { @@ -176,6 +179,10 @@ void ConstraintSolver::solve_constraints() handle_image_constraint(constraint->as_image_constraint()); break; } + case Constraint::Kind::SCALE: { + handle_scale_constraint(constraint->as_scale_constraint()); + break; + } } } @@ -197,12 +204,21 @@ void ConstraintSolver::solve_dependent_constraints(Strategy& strategy) strategy.insert(image_constraint->var_range(), std::move(image)); }; + auto solve_scale_constraint = [&strategy](const ScaleConstraint* scale_constraint) { + auto scaled = scale_constraint->resolve(strategy); + strategy.insert(scale_constraint->var_bigger(), std::move(scaled)); + }; + for (auto& constraint : constraints_) { switch (constraint->kind()) { case Constraint::Kind::IMAGE: { solve_image_constraint(constraint->as_image_constraint()); break; } + case Constraint::Kind::SCALE: { + solve_scale_constraint(constraint->as_scale_constraint()); + break; + } default: { continue; } diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index 70b3a07ce9..ee886f910c 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -34,6 +34,11 @@ bool NoPartition::is_disjoint_for(const Domain* launch_domain) const bool NoPartition::satisfies_restrictions(const Restrictions& restrictions) const { return true; } +std::unique_ptr NoPartition::scale(const Shape& factors) const +{ + return create_no_partition(); +} + Legion::LogicalPartition NoPartition::construct(Legion::LogicalRegion region, bool complete) const { return Legion::LogicalPartition::NO_PART; @@ -121,6 +126,13 @@ bool Tiling::satisfies_restrictions(const Restrictions& restrictions) const return apply(satisfies_restriction, restrictions, color_shape_).all(); } +std::unique_ptr Tiling::scale(const Shape& factors) const +{ + auto new_offsets = + apply([](int64_t off, size_t factor) -> int64_t { return off * factor; }, offsets_, factors); + return create_tiling(tile_shape_ * factors, Shape(color_shape_), std::move(new_offsets)); +} + Legion::LogicalPartition Tiling::construct(Legion::LogicalRegion region, bool complete) const { auto index_space = region.get_index_space(); @@ -220,6 +232,12 @@ bool Weighted::satisfies_restrictions(const Restrictions& restrictions) const return apply(satisfies_restriction, restrictions, color_shape_).all(); } +std::unique_ptr Weighted::scale(const Shape& factors) const +{ + throw std::runtime_error("Not implemented"); + return nullptr; +} + Legion::LogicalPartition Weighted::construct(Legion::LogicalRegion region, bool) const { auto runtime = detail::Runtime::get_runtime(); @@ -294,6 +312,12 @@ bool Image::satisfies_restrictions(const Restrictions& restrictions) const return apply(satisfies_restriction, restrictions, color_shape()).all(); } +std::unique_ptr Image::scale(const Shape& factors) const +{ + throw std::runtime_error("Not implemented"); + return nullptr; +} + Legion::LogicalPartition Image::construct(Legion::LogicalRegion region, bool complete) const { if (!has_launch_domain()) { return Legion::LogicalPartition::NO_PART; } diff --git a/src/core/partitioning/partition.h b/src/core/partitioning/partition.h index 48cc323b87..47534bc239 100644 --- a/src/core/partitioning/partition.h +++ b/src/core/partitioning/partition.h @@ -53,6 +53,9 @@ struct Partition { virtual bool satisfies_restrictions(const Restrictions& restrictions) const = 0; virtual bool is_convertible() const = 0; + public: + virtual std::unique_ptr scale(const Shape& factors) const = 0; + public: virtual Legion::LogicalPartition construct(Legion::LogicalRegion region, bool complete = false) const = 0; @@ -84,6 +87,9 @@ class NoPartition : public Partition { bool satisfies_restrictions(const Restrictions& restrictions) const override; bool is_convertible() const override { return true; } + public: + std::unique_ptr scale(const Shape& factors) const override; + public: Legion::LogicalPartition construct(Legion::LogicalRegion region, bool complete) const override; @@ -125,6 +131,9 @@ class Tiling : public Partition { bool satisfies_restrictions(const Restrictions& restrictions) const override; bool is_convertible() const override { return true; } + public: + std::unique_ptr scale(const Shape& factors) const override; + public: Legion::LogicalPartition construct(Legion::LogicalRegion region, bool complete) const override; @@ -173,6 +182,9 @@ class Weighted : public Partition { bool satisfies_restrictions(const Restrictions& restrictions) const override; bool is_convertible() const override { return false; } + public: + std::unique_ptr scale(const Shape& factors) const override; + public: Legion::LogicalPartition construct(Legion::LogicalRegion region, bool complete) const override; @@ -215,6 +227,9 @@ class Image : public Partition { bool satisfies_restrictions(const Restrictions& restrictions) const override; bool is_convertible() const override { return false; } + public: + std::unique_ptr scale(const Shape& factors) const override; + public: Legion::LogicalPartition construct(Legion::LogicalRegion region, bool complete) const override; diff --git a/tests/cpp/integration/scale_constraints.cc b/tests/cpp/integration/scale_constraints.cc new file mode 100644 index 0000000000..a4dd449ad0 --- /dev/null +++ b/tests/cpp/integration/scale_constraints.cc @@ -0,0 +1,143 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include + +#include "core/data/detail/logical_store.h" +#include "legate.h" + +namespace scale_constraints { + +static const char* library_name = "test_scale_constraints"; + +static const int32_t TEST_MAX_DIM = 3; + +static legate::Logger logger(library_name); + +enum TaskIDs { + SCALE_TESTER = 0, +}; + +template +struct ScaleTester : public legate::LegateTask> { + static const int32_t TASK_ID = SCALE_TESTER + DIM; + static void cpu_variant(legate::TaskContext context) + { + auto smaller = context.output(0); + auto bigger = context.output(1); + + auto extents = context.scalar(0).values(); + auto factors = context.scalar(1).values(); + + auto smaller_shape = smaller.shape(); + auto bigger_shape = bigger.shape(); + + if (bigger_shape.empty()) return; + + for (int32_t idx = 0; idx < DIM; ++idx) { + EXPECT_EQ(smaller_shape.lo[idx] * factors[idx], bigger_shape.lo[idx]); + auto high = context.is_single_task() + ? extents[idx] + : std::min((smaller_shape.hi[idx] + 1) * factors[idx], extents[idx]); + EXPECT_EQ(bigger_shape.hi[idx], high - 1); + } + } +}; + +void prepare() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + ScaleTester<1>::register_variants(context); + ScaleTester<2>::register_variants(context); + ScaleTester<3>::register_variants(context); +} + +struct ScaleTestSpec { + legate::Shape factors; + std::vector smaller_extents; + std::vector bigger_extents; +}; + +void test_scale(const ScaleTestSpec& spec) +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + auto smaller = runtime->create_store(spec.smaller_extents, legate::float16()); + auto bigger = runtime->create_store(spec.bigger_extents, legate::int64()); + + auto task = runtime->create_task(context, SCALE_TESTER + smaller.dim()); + auto part_smaller = task.add_output(smaller); + auto part_bigger = task.add_output(bigger); + task.add_constraint(legate::scale(spec.factors, part_smaller, part_bigger)); + task.add_scalar_arg(spec.bigger_extents); + task.add_scalar_arg(spec.factors.data()); + + runtime->submit(std::move(task)); +} + +void test_invalid() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + { + auto smaller = runtime->create_store({1, 2}, legate::float16()); + auto bigger = runtime->create_store({2, 3, 4}, legate::int64()); + + auto task = runtime->create_task(context, SCALE_TESTER + 2); + auto part_smaller = task.add_output(smaller); + auto part_bigger = task.add_output(bigger); + task.add_constraint(legate::scale({2, 3}, part_smaller, part_bigger)); + + EXPECT_THROW(runtime->submit(std::move(task)), std::invalid_argument); + } + + { + auto smaller = runtime->create_store({1, 2, 3}, legate::float16()); + auto bigger = runtime->create_store({2, 3, 4}, legate::int64()); + + auto task = runtime->create_task(context, SCALE_TESTER + 3); + auto part_smaller = task.add_output(smaller); + auto part_bigger = task.add_output(bigger); + task.add_constraint(legate::scale({2, 3}, part_smaller, part_bigger)); + + EXPECT_THROW(runtime->submit(std::move(task)), std::invalid_argument); + } +} + +TEST(ScaleConstraint, 1D) +{ + legate::Core::perform_registration(); + test_scale({{3}, {10}, {29}}); +} + +TEST(ScaleConstraint, 2D) +{ + legate::Core::perform_registration(); + test_scale({{4, 5}, {2, 7}, {10, 30}}); +} + +TEST(ScaleConstraint, 3D) +{ + legate::Core::perform_registration(); + test_scale({{2, 3, 4}, {5, 5, 5}, {10, 15, 20}}); +} + +TEST(ScaleConstraint, Invalid) +{ + legate::Core::perform_registration(); + test_invalid(); +} + +} // namespace scale_constraints From a93320433c986771f69a9f7f1cb54f0929f7ead4 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Mon, 11 Sep 2023 12:44:37 -0700 Subject: [PATCH 0209/1425] Null type and binary types (#62) * Null type * Opaque binary types * Overload of record_reduction_operator that takes a ReductionOpKind * Address review comments * Minor fix --- legate/core/_lib/types.pyx | 12 ++- legate/core/launcher.py | 6 ++ legate/core/operation.py | 6 +- legate/core/types.py | 5 ++ legate/core/types.pyi | 4 + src/core/data/detail/scalar.h | 2 +- src/core/data/scalar.cc | 4 + src/core/data/scalar.h | 6 ++ src/core/data/scalar.inl | 7 +- src/core/data/store.inl | 2 +- src/core/legate_c.h | 5 +- src/core/runtime/detail/runtime.cc | 6 +- src/core/runtime/detail/runtime.h | 2 - src/core/type/detail/type_info.cc | 88 +++++++++++++++++---- src/core/type/detail/type_info.h | 19 +++++ src/core/type/type_info.cc | 11 ++- src/core/type/type_info.h | 31 +++++++- src/core/type/type_traits.h | 2 +- src/core/utilities/deserializer.h | 2 +- src/core/utilities/deserializer.inl | 66 ++++++++++++---- src/core/utilities/detail/buffer_builder.cc | 1 + tests/cpp/unit/scalar.cc | 47 ++++++++--- tests/cpp/unit/type.cc | 38 +++++++-- 23 files changed, 303 insertions(+), 69 deletions(-) diff --git a/legate/core/_lib/types.pyx b/legate/core/_lib/types.pyx index a9f91bbb08..2eb2fd4a06 100644 --- a/legate/core/_lib/types.pyx +++ b/legate/core/_lib/types.pyx @@ -20,6 +20,7 @@ import numpy as np cdef extern from "core/legate_c.h" nogil: ctypedef enum legate_core_type_code_t: + _NULL "NULL_LT" _BOOL "BOOL_LT" _INT8 "INT8_LT" _INT16 "INT16_LT" @@ -34,10 +35,10 @@ cdef extern from "core/legate_c.h" nogil: _FLOAT64 "FLOAT64_LT" _COMPLEX64 "COMPLEX64_LT" _COMPLEX128 "COMPLEX128_LT" + _BINARY "BINARY_LT" _FIXED_ARRAY "FIXED_ARRAY_LT" _STRUCT "STRUCT_LT" _STRING "STRING_LT" - _INVALID "INVALID_LT" ctypedef enum legate_core_reduction_op_kind_t: _ADD "ADD_LT" @@ -50,6 +51,7 @@ cdef extern from "core/legate_c.h" nogil: _AND "AND_LT" _XOR "XOR_LT" +NIL = legate_core_type_code_t._NULL BOOL = legate_core_type_code_t._BOOL INT8 = legate_core_type_code_t._INT8 INT16 = legate_core_type_code_t._INT16 @@ -64,10 +66,10 @@ FLOAT32 = legate_core_type_code_t._FLOAT32 FLOAT64 = legate_core_type_code_t._FLOAT64 COMPLEX64 = legate_core_type_code_t._COMPLEX64 COMPLEX128 = legate_core_type_code_t._COMPLEX128 +BINARY = legate_core_type_code_t._BINARY FIXED_ARRAY = legate_core_type_code_t._FIXED_ARRAY STRUCT = legate_core_type_code_t._STRUCT STRING = legate_core_type_code_t._STRING -INVALID = legate_core_type_code_t._INVALID ADD = legate_core_reduction_op_kind_t._ADD SUB = legate_core_reduction_op_kind_t._SUB @@ -125,6 +127,8 @@ cdef extern from "core/type/detail/type_info.h" namespace "legate::detail" nogil cdef shared_ptr[Type] string_type() + cdef shared_ptr[Type] binary_type(unsigned int size) + cdef shared_ptr[Type] fixed_array_type( shared_ptr[Type] element_type, unsigned int N ) except+ @@ -157,6 +161,10 @@ cdef class Dtype: def string_type() -> Dtype: return from_ptr(string_type()) + @staticmethod + def binary_type(unsigned size) -> Dtype: + return from_ptr(binary_type(size)) + @staticmethod def fixed_array_type( Dtype element_type, unsigned N diff --git a/legate/core/launcher.py b/legate/core/launcher.py index d00fbce3ce..6f98f64e1c 100644 --- a/legate/core/launcher.py +++ b/legate/core/launcher.py @@ -78,7 +78,13 @@ class Permission(IntEnum): Serializer = Callable[[BufferBuilder, Any], None] + +def ignore(_: BufferBuilder, __: Any) -> None: + pass + + _SERIALIZERS: dict[int, Serializer] = { + ty.null.uid: ignore, ty.bool_.uid: BufferBuilder.pack_bool, ty.int8.uid: BufferBuilder.pack_8bit_int, ty.int16.uid: BufferBuilder.pack_16bit_int, diff --git a/legate/core/operation.py b/legate/core/operation.py index bdd8197e64..aa7de63493 100644 --- a/legate/core/operation.py +++ b/legate/core/operation.py @@ -423,7 +423,11 @@ def add_scalar_arg( raise TypeError(f"Unsupported type: {dtype}") if not isinstance(value, (tuple, Shape)): raise ValueError(f"{value} is not a valid scalar") - sanitized = ty.array_type(dtype[0], len(value)) + sanitized = ( + ty.array_type(dtype[0], len(value)) + if len(value) > 0 + else ty.null + ) else: raise TypeError(f"Unsupported type: {dtype}") diff --git a/legate/core/types.py b/legate/core/types.py index 36a21ced50..5dfc619133 100644 --- a/legate/core/types.py +++ b/legate/core/types.py @@ -34,6 +34,7 @@ class ReductionOp(IntEnum): StructDtype = ext.StructDtype +null = Dtype.primitive_type(ext.NIL) bool_ = Dtype.primitive_type(ext.BOOL) int8 = Dtype.primitive_type(ext.INT8) int16 = Dtype.primitive_type(ext.INT16) @@ -51,6 +52,10 @@ class ReductionOp(IntEnum): string = Dtype.string_type() +def binary_type(size: int) -> Dtype: + return Dtype.binary_type(size) + + def array_type(element_type: Dtype, N: int) -> FixedArrayDtype: return Dtype.fixed_array_type(element_type, N) diff --git a/legate/core/types.pyi b/legate/core/types.pyi index c9afe507ae..e7fa840b7a 100644 --- a/legate/core/types.pyi +++ b/legate/core/types.pyi @@ -26,6 +26,8 @@ class ReductionOp: XOR: int class Dtype: + @staticmethod + def binary_type(size: int) -> Dtype: ... @staticmethod def fixed_array_type(element_type: Dtype, N: int) -> Dtype: ... @staticmethod @@ -59,6 +61,7 @@ class StructDtype(Dtype): def num_fields(self) -> int: ... def field_type(self, field_idx: int) -> Dtype: ... +null: Dtype bool_: Dtype int8: Dtype int16: Dtype @@ -75,6 +78,7 @@ complex64: Dtype complex128: Dtype string: Dtype +def binary_type(size: int) -> Dtype: ... def array_type(element_type: Dtype, N: int) -> FixedArrayDtype: ... def struct_type( field_types: list[Dtype], align: bool = False diff --git a/src/core/data/detail/scalar.h b/src/core/data/detail/scalar.h index 25e6b1be68..658a818899 100644 --- a/src/core/data/detail/scalar.h +++ b/src/core/data/detail/scalar.h @@ -35,7 +35,7 @@ class Scalar { static_assert(legate_type_code_of != Type::Code::FIXED_ARRAY); static_assert(legate_type_code_of != Type::Code::STRUCT); static_assert(legate_type_code_of != Type::Code::STRING); - static_assert(legate_type_code_of != Type::Code::INVALID); + static_assert(legate_type_code_of != Type::Code::NIL); data_ = copy_data(&value, sizeof(T)); } diff --git a/src/core/data/scalar.cc b/src/core/data/scalar.cc index 926c0a980c..969df8740c 100644 --- a/src/core/data/scalar.cc +++ b/src/core/data/scalar.cc @@ -21,6 +21,8 @@ Scalar::Scalar(const Scalar& other) : impl_(new detail::Scalar(*other.impl_)) {} Scalar::Scalar(Scalar&& other) : impl_(other.impl_) { other.impl_ = nullptr; } +Scalar::Scalar() : impl_(create_impl(null_type(), nullptr, false)) {} + Scalar::~Scalar() { delete impl_; } Scalar::Scalar(Type type, const void* data, bool copy) : impl_(create_impl(type, data, copy)) {} @@ -44,4 +46,6 @@ const void* Scalar::ptr() const { return impl_->data(); } return new detail::Scalar(type.impl(), data, copy); } +Scalar null() { return Scalar(); } + } // namespace legate diff --git a/src/core/data/scalar.h b/src/core/data/scalar.h index 4d289a94f5..0132282638 100644 --- a/src/core/data/scalar.h +++ b/src/core/data/scalar.h @@ -48,6 +48,10 @@ class Scalar { ~Scalar(); public: + /** + * @brief Creates a null scalar + */ + Scalar(); /** * @brief Creates a shared `Scalar` with an existing allocation. The caller is responsible * for passing in a sufficiently big allocation. @@ -181,6 +185,8 @@ class Scalar { detail::Scalar* impl_{nullptr}; }; +Scalar null(); + } // namespace legate #include "core/data/scalar.inl" diff --git a/src/core/data/scalar.inl b/src/core/data/scalar.inl index b8588b4cbb..a471d52ec1 100644 --- a/src/core/data/scalar.inl +++ b/src/core/data/scalar.inl @@ -39,7 +39,7 @@ inline constexpr legate::Type::Code canonical_type_code_of() noexcept static_assert(ret != Type::Code::FIXED_ARRAY); static_assert(ret != Type::Code::STRUCT); static_assert(ret != Type::Code::STRING); - static_assert(ret != Type::Code::INVALID); + static_assert(ret != Type::Code::NIL); return ret; } } @@ -69,8 +69,7 @@ Scalar::Scalar(T value) : Scalar{detail::canonical_value_of(std::move(value)), p template Scalar::Scalar(T value, Type type) : impl_(create_impl(type, &value, true)) { - if (type.code() == Type::Code::INVALID) - throw std::invalid_argument("Invalid type cannot be used"); + if (type.code() == Type::Code::NIL) throw std::invalid_argument("Null type cannot be used"); if (type.size() != sizeof(T)) throw std::invalid_argument("Size of the value doesn't match with the type"); } @@ -148,6 +147,8 @@ Span Scalar::values() const auto len = *static_cast(data); const auto* begin = static_cast(data) + sizeof(uint32_t); return Span(reinterpret_cast(begin), len); + } else if (ty.code() == Type::Code::NIL) { + return Span(nullptr, 0); } else { if (sizeof(VAL) != ty.size()) throw std::invalid_argument("Size of the scalar is " + std::to_string(ty.size()) + diff --git a/src/core/data/store.inl b/src/core/data/store.inl index 9097f9053b..53014a09b4 100644 --- a/src/core/data/store.inl +++ b/src/core/data/store.inl @@ -225,7 +225,7 @@ void Store::check_accessor_type() const auto in_type = legate_type_code_of; if (in_type == this->code()) return; // Test exact match for primitive types - if (in_type != Type::Code::INVALID) { + if (in_type != Type::Code::NIL) { throw std::invalid_argument( "Type mismatch: " + primitive_type(in_type).to_string() + " accessor to a " + type().to_string() + diff --git a/src/core/legate_c.h b/src/core/legate_c.h index f34095a2cd..a48b380a0b 100644 --- a/src/core/legate_c.h +++ b/src/core/legate_c.h @@ -89,13 +89,16 @@ typedef enum legate_core_type_code_t { FLOAT64_LT = LEGION_TYPE_FLOAT64, COMPLEX64_LT = LEGION_TYPE_COMPLEX64, COMPLEX128_LT = LEGION_TYPE_COMPLEX128, + // Null type + NULL_LT, + // Opaque binary type + BINARY_LT, // Compound types FIXED_ARRAY_LT, STRUCT_LT, // Variable size types STRING_LT, LIST_LT, - INVALID_LT = -1, } legate_core_type_code_t; typedef enum legate_core_reduction_op_kind_t { diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index 40ea40ddb7..da335db841 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -54,8 +54,7 @@ namespace legate::detail { namespace { -constexpr uint32_t CUSTOM_TYPE_UID_BASE = 0x10000; -const char* TOPLEVEL_NAME = "Legate Core Toplevel Task"; +const char* TOPLEVEL_NAME = "Legate Core Toplevel Task"; } // namespace @@ -64,7 +63,6 @@ Runtime::Runtime() field_reuse_freq_( extract_env("LEGATE_FIELD_REUSE_FREQ", FIELD_REUSE_FREQ_DEFAULT, FIELD_REUSE_FREQ_TEST)), force_consensus_match_(extract_env("LEGATE_CONSENSUS", CONSENSUS_DEFAULT, CONSENSUS_TEST)), - next_type_uid_(CUSTOM_TYPE_UID_BASE), max_pending_exceptions_(extract_env( "LEGATE_MAX_PENDING_EXCEPTIONS", MAX_PENDING_EXCEPTIONS_DEFAULT, MAX_PENDING_EXCEPTIONS_TEST)) { @@ -111,8 +109,6 @@ Library* Runtime::find_or_create_library(const std::string& library_name, return result; } -uint32_t Runtime::get_type_uid() { return next_type_uid_++; } - void Runtime::record_reduction_operator(int32_t type_uid, int32_t op_kind, int32_t legion_op_id) { #ifdef DEBUG_LEGATE diff --git a/src/core/runtime/detail/runtime.h b/src/core/runtime/detail/runtime.h index 923a09b723..9898b625ff 100644 --- a/src/core/runtime/detail/runtime.h +++ b/src/core/runtime/detail/runtime.h @@ -63,7 +63,6 @@ class Runtime { bool* created); public: - uint32_t get_type_uid(); void record_reduction_operator(int32_t type_uid, int32_t op_kind, int32_t legion_op_id); int32_t find_reduction_operator(int32_t type_uid, int32_t op_kind) const; @@ -318,7 +317,6 @@ class Runtime { std::map libraries_{}; private: - uint32_t next_type_uid_; std::map, int32_t> reduction_ops_{}; private: diff --git a/src/core/type/detail/type_info.cc b/src/core/type/detail/type_info.cc index bb24fb155f..db478393bb 100644 --- a/src/core/type/detail/type_info.cc +++ b/src/core/type/detail/type_info.cc @@ -10,6 +10,7 @@ * its affiliates is strictly prohibited. */ +#include #include #include @@ -36,6 +37,7 @@ const std::unordered_map SIZEOF = { {Type::Code::FLOAT64, sizeof(legate_type_of)}, {Type::Code::COMPLEX64, sizeof(legate_type_of)}, {Type::Code::COMPLEX128, sizeof(legate_type_of)}, + {Type::Code::NIL, 0}, }; const std::unordered_map TYPE_NAMES = { @@ -54,10 +56,38 @@ const std::unordered_map TYPE_NAMES = { {Type::Code::COMPLEX64, "complex64"}, {Type::Code::COMPLEX128, "complex128"}, {Type::Code::STRING, "string"}, + {Type::Code::NIL, "null_type"}, }; const char* _VARIABLE_SIZE_ERROR_MESSAGE = "Variable-size element type cannot be used"; +// Some notes about these magic numbers: +// +// The numbers are chosen such that UIDs of types are truly unique even in the presence of types +// with static UIDs derived from their type codes and sizes. Here's the list of static UIDs that +// each kind of types can take (dynamic UIDs generated off of _BASE_CUSTOM_TYPE_UID are unique by +// construction): +// +// * Primitive types: [0x00, 0x0E] +// * Binary types: [0x000001, 0x0FFFFF] <+> [0x0F] +// * Fixed-size array types: [0x01, 0xFF] <+> [0x00, 0x0E] +// * Point types: [_BASE_POINT_TYPE_UID + 1, _BASE_POINT_TYPE_UID + LEGATE_MAX_DIM] +// * Rect types: [_BASE_RECT_TYPE_UID + 1, _BASE_RECT_TYPE_UID + LEGATE_MAX_DIM] +// +// where the <+> operator is a pairwise concatenation +constexpr uint32_t _TYPE_CODE_OFFSET = 8; +constexpr uint32_t _BASE_POINT_TYPE_UID = 0x10000000; +constexpr uint32_t _BASE_RECT_TYPE_UID = _BASE_POINT_TYPE_UID + LEGATE_MAX_DIM + 1; +constexpr uint32_t _BASE_CUSTOM_TYPE_UID = _BASE_RECT_TYPE_UID + LEGATE_MAX_DIM + 1; +// Last byte of a static UID is a type code +constexpr uint32_t _MAX_BINARY_TYPE_SIZE = 0x0FFFFF00 >> _TYPE_CODE_OFFSET; + +uint32_t get_next_uid() +{ + static std::atomic next_uid = _BASE_CUSTOM_TYPE_UID; + return next_uid++; +} + } // namespace Type::Type(Code c) : code(c) {} @@ -118,6 +148,21 @@ bool PrimitiveType::equal(const Type& other) const { return code == other.code; ExtensionType::ExtensionType(int32_t uid, Type::Code code) : Type(code), uid_(uid) {} +BinaryType::BinaryType(int32_t uid, uint32_t size) + : ExtensionType(uid, Type::Code::BINARY), size_(size) +{ +} + +std::string BinaryType::to_string() const { return "binary(" + std::to_string(size_) + ")"; } + +void BinaryType::pack(BufferBuilder& buffer) const +{ + buffer.pack(static_cast(code)); + buffer.pack(size_); +} + +bool BinaryType::equal(const Type& other) const { return uid_ == other.uid(); } + FixedArrayType::FixedArrayType(int32_t uid, std::shared_ptr element_type, uint32_t N) : ExtensionType(uid, Type::Code::FIXED_ARRAY), element_type_(std::move(element_type)), @@ -316,6 +361,19 @@ std::shared_ptr string_type() return type; } +std::shared_ptr binary_type(uint32_t size) +{ + if (size == 0) { + throw std::out_of_range("Size for an opaque binary type must be greater than 0"); + } + if (size > _MAX_BINARY_TYPE_SIZE) { + throw std::out_of_range("Maximum size for opaque binary types is " + + std::to_string(_MAX_BINARY_TYPE_SIZE)); + } + int32_t uid = static_cast(Type::Code::BINARY) | (size << _TYPE_CODE_OFFSET); + return std::make_shared(uid, size); +} + std::shared_ptr fixed_array_type(std::shared_ptr element_type, uint32_t N) { if (N == 0) { throw std::out_of_range("Size of array must be greater than 0"); } @@ -325,8 +383,8 @@ std::shared_ptr fixed_array_type(std::shared_ptr element_type, uint3 // | length | element type code | // +--------+-------------------+ int32_t uid = [&N](const Type& elem_type) { - if (!elem_type.is_primitive() || N > 0xFFU) return Runtime::get_runtime()->get_type_uid(); - return static_cast(elem_type.code) | N << 8; + if (!elem_type.is_primitive() || N > 0xFFU) return get_next_uid(); + return static_cast(elem_type.code) | (N << _TYPE_CODE_OFFSET); }(*element_type); return std::make_shared(uid, std::move(element_type), N); } @@ -334,19 +392,17 @@ std::shared_ptr fixed_array_type(std::shared_ptr element_type, uint3 std::shared_ptr struct_type(const std::vector>& field_types, bool align) { return std::make_shared( - Runtime::get_runtime()->get_type_uid(), std::vector>(field_types), align); + get_next_uid(), std::vector>(field_types), align); } std::shared_ptr struct_type(std::vector>&& field_types, bool align) { - return std::make_shared( - Runtime::get_runtime()->get_type_uid(), std::move(field_types), align); + return std::make_shared(get_next_uid(), std::move(field_types), align); } std::shared_ptr list_type(std::shared_ptr element_type) { - return std::make_shared(Runtime::get_runtime()->get_type_uid(), - std::move(element_type)); + return std::make_shared(get_next_uid(), std::move(element_type)); } std::shared_ptr bool_() @@ -433,13 +489,6 @@ std::shared_ptr complex128() return result; } -namespace { - -constexpr int32_t POINT_UID_BASE = static_cast(Type::Code::INVALID); -constexpr int32_t RECT_UID_BASE = POINT_UID_BASE + LEGATE_MAX_DIM + 1; - -} // namespace - std::shared_ptr point_type(int32_t ndim) { static std::shared_ptr cache[LEGATE_MAX_DIM + 1]; @@ -447,7 +496,8 @@ std::shared_ptr point_type(int32_t ndim) if (ndim <= 0 || ndim > LEGATE_MAX_DIM) throw std::out_of_range(std::to_string(ndim) + " is not a supported number of dimensions"); if (nullptr == cache[ndim]) { - cache[ndim] = std::make_shared(POINT_UID_BASE + ndim, int64(), ndim); + cache[ndim] = + std::make_shared(_BASE_POINT_TYPE_UID + ndim, int64(), ndim); } return cache[ndim]; } @@ -463,11 +513,17 @@ std::shared_ptr rect_type(int32_t ndim) auto pt_type = point_type(ndim); std::vector> field_types{pt_type, pt_type}; cache[ndim] = std::make_shared( - RECT_UID_BASE + ndim, std::move(field_types), true /*align*/); + _BASE_RECT_TYPE_UID + ndim, std::move(field_types), true /*align*/); } return cache[ndim]; } +std::shared_ptr null_type() +{ + static auto result = detail::primitive_type(Type::Code::NIL); + return result; +} + bool is_point_type(const std::shared_ptr& type, int32_t ndim) { switch (type->code) { diff --git a/src/core/type/detail/type_info.h b/src/core/type/detail/type_info.h index c5a7f6c54a..8d7577fa02 100644 --- a/src/core/type/detail/type_info.h +++ b/src/core/type/detail/type_info.h @@ -94,6 +94,22 @@ class ExtensionType : public Type { const uint32_t uid_; }; +class BinaryType : public ExtensionType { + public: + BinaryType(int32_t uid, uint32_t size); + uint32_t size() const override { return size_; } + uint32_t alignment() const override { return size_; } + bool variable_size() const override { return false; } + std::string to_string() const override; + void pack(BufferBuilder& buffer) const override; + + private: + bool equal(const Type& other) const override; + + private: + const uint32_t size_; +}; + class FixedArrayType : public ExtensionType { public: FixedArrayType(int32_t uid, std::shared_ptr element_type, uint32_t N); @@ -164,6 +180,8 @@ std::shared_ptr primitive_type(Type::Code code); std::shared_ptr string_type(); +std::shared_ptr binary_type(uint32_t size); + std::shared_ptr fixed_array_type(std::shared_ptr element_type, uint32_t N); std::shared_ptr struct_type(const std::vector>& field_types, @@ -189,6 +207,7 @@ std::shared_ptr complex64(); std::shared_ptr complex128(); std::shared_ptr point_type(int32_t ndim); std::shared_ptr rect_type(int32_t ndim); +std::shared_ptr null_type(); bool is_point_type(const std::shared_ptr& type, int32_t ndim); bool is_rect_type(const std::shared_ptr& type, int32_t ndim); diff --git a/src/core/type/type_info.cc b/src/core/type/type_info.cc index c91a0c673e..f5a7c994f7 100644 --- a/src/core/type/type_info.cc +++ b/src/core/type/type_info.cc @@ -55,6 +55,11 @@ void Type::record_reduction_operator(int32_t op_kind, int32_t global_op_id) cons impl_->record_reduction_operator(op_kind, global_op_id); } +void Type::record_reduction_operator(ReductionOpKind op_kind, int32_t global_op_id) const +{ + impl_->record_reduction_operator(static_cast(op_kind), global_op_id); +} + int32_t Type::find_reduction_operator(int32_t op_kind) const { return impl_->find_reduction_operator(op_kind); @@ -62,7 +67,7 @@ int32_t Type::find_reduction_operator(int32_t op_kind) const int32_t Type::find_reduction_operator(ReductionOpKind op_kind) const { - return impl_->find_reduction_operator(op_kind); + return impl_->find_reduction_operator(static_cast(op_kind)); } bool Type::operator==(const Type& other) const { return impl_->equal(*other.impl_); } @@ -169,10 +174,14 @@ Type complex64() { return Type(detail::complex64()); } Type complex128() { return Type(detail::complex128()); } +Type binary_type(uint32_t size) { return Type(detail::binary_type(size)); } + Type point_type(int32_t ndim) { return Type(detail::point_type(ndim)); } Type rect_type(int32_t ndim) { return Type(detail::rect_type(ndim)); } +Type null_type() { return Type(detail::null_type()); } + bool is_point_type(const Type& type, int32_t ndim) { return detail::is_point_type(type.impl(), ndim); diff --git a/src/core/type/type_info.h b/src/core/type/type_info.h index 528ce722b2..3cf6dd670c 100644 --- a/src/core/type/type_info.h +++ b/src/core/type/type_info.h @@ -57,6 +57,7 @@ class Type { * @brief Enum for type codes */ enum class Code : int32_t { + NIL = NULL_LT, /*!< Null type */ BOOL = BOOL_LT, /*!< Boolean type */ INT8 = INT8_LT, /*!< 8-bit signed integer type */ INT16 = INT16_LT, /*!< 16-bit signed integer type */ @@ -71,11 +72,11 @@ class Type { FLOAT64 = FLOAT64_LT, /*!< Double-precision floating point type */ COMPLEX64 = COMPLEX64_LT, /*!< Single-precision complex type */ COMPLEX128 = COMPLEX128_LT, /*!< Double-precision complex type */ + BINARY = BINARY_LT, /*!< Opaque binary type */ FIXED_ARRAY = FIXED_ARRAY_LT, /*!< Fixed-size array type */ STRUCT = STRUCT_LT, /*!< Struct type */ STRING = STRING_LT, /*!< String type */ LIST = LIST_LT, /*!< List type */ - INVALID = INVALID_LT, /*!< Invalid type */ }; public: @@ -157,6 +158,16 @@ class Type { * @param global_op_id Global reduction operator ID */ void record_reduction_operator(int32_t op_kind, int32_t global_op_id) const; + /** + * @brief Records a reduction operator. + * + * The global ID of the reduction operator is issued when that operator is registered + * to the runtime. + * + * @param op_kind Reduction operator kind + * @param global_op_id Global reduction operator ID + */ + void record_reduction_operator(ReductionOpKind op_kind, int32_t global_op_id) const; /** * @brief Finds the global operator ID for a given reduction operator kind. * @@ -302,6 +313,16 @@ Type primitive_type(Type::Code code); */ Type string_type(); +/** + * @ingroup types + * @brief Creates an opaque binary type of a given size + * + * @param size Element size + * + * @return Type object + */ +Type binary_type(uint32_t size); + /** * @ingroup types * @brief Creates a metadata object for a fixed-size array type @@ -491,6 +512,14 @@ Type point_type(int32_t ndim); */ Type rect_type(int32_t ndim); +/** + * @ingroup types + * @brief Creates a null type + * + * @return Type object + */ +Type null_type(); + /** * @ingroup types * @brief Checks if the type is a point type of the given dimensionality diff --git a/src/core/type/type_traits.h b/src/core/type/type_traits.h index 21162a76a3..354768add5 100644 --- a/src/core/type/type_traits.h +++ b/src/core/type/type_traits.h @@ -46,7 +46,7 @@ namespace legate { * @brief A template constexpr that converts types to type codes */ template -PREFIX constexpr Type::Code legate_type_code_of = Type::Code::INVALID; +PREFIX constexpr Type::Code legate_type_code_of = Type::Code::NIL; template <> PREFIX constexpr Type::Code legate_type_code_of<__half> = Type::Code::FLOAT16; template <> diff --git a/src/core/utilities/deserializer.h b/src/core/utilities/deserializer.h index 70d7967e3e..affdb72f06 100644 --- a/src/core/utilities/deserializer.h +++ b/src/core/utilities/deserializer.h @@ -49,7 +49,7 @@ class BaseDeserializer { } public: - template != Type::Code::INVALID>* = nullptr> + template != Type::Code::NIL>* = nullptr> void _unpack(T& value) { value = *reinterpret_cast(args_.ptr()); diff --git a/src/core/utilities/deserializer.inl b/src/core/utilities/deserializer.inl index d41a318b74..be250f1eb4 100644 --- a/src/core/utilities/deserializer.inl +++ b/src/core/utilities/deserializer.inl @@ -39,8 +39,9 @@ std::unique_ptr BaseDeserializer::unpack_scalar() // this unpack_type call must be in a separate line from the following one because they both // read and update the buffer location. auto type = unpack_type(); - auto result = std::make_unique(type, args_.ptr(), false /*copy*/); - args_ = args_.subspan(result->size()); + auto result = std::make_unique( + type, type->code == Type::Code::NIL ? nullptr : args_.ptr(), false /*copy*/); + args_ = args_.subspan(result->size()); return result; } @@ -156,21 +157,54 @@ std::shared_ptr BaseDeserializer::unpack_type() auto type = unpack_type(); return std::make_shared(uid, std::move(type)); } - case Type::Code::BOOL: - case Type::Code::INT8: - case Type::Code::INT16: - case Type::Code::INT32: - case Type::Code::INT64: - case Type::Code::UINT8: - case Type::Code::UINT16: - case Type::Code::UINT32: - case Type::Code::UINT64: - case Type::Code::FLOAT16: - case Type::Code::FLOAT32: - case Type::Code::FLOAT64: - case Type::Code::COMPLEX64: + case Type::Code::NIL: { + return detail::null_type(); + } + case Type::Code::BOOL: { + return detail::bool_(); + } + case Type::Code::INT8: { + return detail::int8(); + } + case Type::Code::INT16: { + return detail::int16(); + } + case Type::Code::INT32: { + return detail::int32(); + } + case Type::Code::INT64: { + return detail::int64(); + } + case Type::Code::UINT8: { + return detail::uint8(); + } + case Type::Code::UINT16: { + return detail::uint16(); + } + case Type::Code::UINT32: { + return detail::uint32(); + } + case Type::Code::UINT64: { + return detail::uint64(); + } + case Type::Code::FLOAT16: { + return detail::float16(); + } + case Type::Code::FLOAT32: { + return detail::float32(); + } + case Type::Code::FLOAT64: { + return detail::float64(); + } + case Type::Code::COMPLEX64: { + return detail::complex64(); + } case Type::Code::COMPLEX128: { - return std::make_shared(code); + return detail::complex128(); + } + case Type::Code::BINARY: { + auto size = unpack(); + return detail::binary_type(size); } case Type::Code::STRING: { return std::make_shared(); diff --git a/src/core/utilities/detail/buffer_builder.cc b/src/core/utilities/detail/buffer_builder.cc index 4b441e4143..5f444639e5 100644 --- a/src/core/utilities/detail/buffer_builder.cc +++ b/src/core/utilities/detail/buffer_builder.cc @@ -22,6 +22,7 @@ BufferBuilder::BufferBuilder() void BufferBuilder::pack_buffer(const void* src, size_t size) { + if (0 == size) return; auto off = buffer_.size(); buffer_.resize(buffer_.size() + size); auto tgt = buffer_.data() + off; diff --git a/tests/cpp/unit/scalar.cc b/tests/cpp/unit/scalar.cc index c95d4f0afb..4f37d06e80 100644 --- a/tests/cpp/unit/scalar.cc +++ b/tests/cpp/unit/scalar.cc @@ -39,12 +39,22 @@ struct PaddingStructData { bool bool_data; int32_t int32_data; uint64_t uint64_data; + bool operator==(const PaddingStructData& other) const + { + return bool_data == other.bool_data && int32_data == other.int32_data && + uint64_data == other.uint64_data; + } }; struct __attribute__((packed)) NoPaddingStructData { bool bool_data; int32_t int32_data; uint64_t uint64_data; + bool operator==(const NoPaddingStructData& other) const + { + return bool_data == other.bool_data && int32_data == other.int32_data && + uint64_data == other.uint64_data; + } }; template @@ -105,6 +115,19 @@ void check_type(T value, legate::Type type) } } +template +void check_binary_scalar(T& value) +{ + legate::Scalar scalar(value, legate::binary_type(sizeof(T))); + EXPECT_EQ(scalar.type().size(), sizeof(T)); + EXPECT_NE(scalar.ptr(), nullptr); + + EXPECT_EQ(value, scalar.value()); + auto actual = scalar.values(); + EXPECT_EQ(actual.size(), 1); + EXPECT_EQ(actual[0], value); +} + template void check_struct_type_scalar(T& struct_data, bool align) { @@ -295,17 +318,23 @@ TEST(ScalarUnit, CreateWithValue) check_type(COMPLEX_FLOAT_VALUE, legate::complex64()); check_type(COMPLEX_DOUBLE_VALUE, legate::complex128()); - // Invalid type - // Note: test fails - // Expected: legate::Scalar(BOOL_VALUE, legate::primitive_type(legate::Type::Code::INVALID)) - // throws an exception of type std::invalid_argument. Actual: it throws std::out_of_range with - // description "_Map_base::at". EXPECT_THROW(legate::Scalar(BOOL_VALUE, - // legate::primitive_type(legate::Type::Code::INVALID)), std::invalid_argument); - // Size mismatch EXPECT_THROW(legate::Scalar(FLOAT16_VALUE, legate::float32()), std::invalid_argument); } +TEST(ScalarUnit, CreateBinary) +{ + { + PaddingStructData value = {BOOL_VALUE, INT32_VALUE, UINT64_VALUE}; + check_binary_scalar(value); + } + + { + NoPaddingStructData value = {BOOL_VALUE, INT32_VALUE, UINT64_VALUE}; + check_binary_scalar(value); + } +} + TEST(ScalarUnit, CreateWithVector) { // constructor with arrays @@ -355,13 +384,13 @@ TEST(ScalarUnit, CreateWithStructType) // with struct padding { PaddingStructData struct_data = {BOOL_VALUE, INT32_VALUE, UINT64_VALUE}; - check_struct_type_scalar(struct_data, true); + check_struct_type_scalar(struct_data, true); } // without struct padding { NoPaddingStructData struct_data = {BOOL_VALUE, INT32_VALUE, UINT64_VALUE}; - check_struct_type_scalar(struct_data, false); + check_struct_type_scalar(struct_data, false); } } diff --git a/tests/cpp/unit/type.cc b/tests/cpp/unit/type.cc index f3760b8443..62bfd990b6 100644 --- a/tests/cpp/unit/type.cc +++ b/tests/cpp/unit/type.cc @@ -55,6 +55,18 @@ void test_string_type(const legate::Type& type) EXPECT_EQ(other, type); } +void test_binary_type(const legate::Type& type, uint32_t size) +{ + EXPECT_EQ(type.code(), legate::Type::Code::BINARY); + EXPECT_EQ(type.size(), size); + EXPECT_EQ(type.alignment(), size); + EXPECT_FALSE(type.variable_size()); + EXPECT_FALSE(type.is_primitive()); + EXPECT_EQ(type.to_string(), "binary(" + std::to_string(size) + ")"); + legate::Type other(type); + EXPECT_EQ(other, type); +} + void test_fixed_array_type(const legate::Type& type, const legate::Type& element_type, uint32_t N, @@ -165,15 +177,25 @@ TEST(TypeUnit, PrimitiveType) EXPECT_EQ(legate::complex64(), legate::primitive_type(legate::Type::Code::COMPLEX64)); EXPECT_EQ(legate::complex128(), legate::primitive_type(legate::Type::Code::COMPLEX128)); + EXPECT_THROW(legate::primitive_type(legate::Type::Code::BINARY), std::invalid_argument); EXPECT_THROW(legate::primitive_type(legate::Type::Code::FIXED_ARRAY), std::invalid_argument); EXPECT_THROW(legate::primitive_type(legate::Type::Code::STRUCT), std::invalid_argument); EXPECT_THROW(legate::primitive_type(legate::Type::Code::STRING), std::invalid_argument); EXPECT_THROW(legate::primitive_type(legate::Type::Code::LIST), std::invalid_argument); - EXPECT_THROW(legate::primitive_type(legate::Type::Code::INVALID), std::invalid_argument); } TEST(TypeUnit, StringType) { test_string_type(legate::string_type()); } +TEST(TypeUnit, BinaryType) +{ + test_binary_type(legate::binary_type(123), 123); + test_binary_type(legate::binary_type(45), 45); + EXPECT_EQ(legate::binary_type(678).uid(), legate::binary_type(678).uid()); + + EXPECT_THROW(legate::binary_type(0), std::out_of_range); + EXPECT_THROW(legate::binary_type(0xFFFFF + 1), std::out_of_range); +} + TEST(TypeUnit, FixedArrayType) { // element type is a primitive type @@ -440,7 +462,7 @@ TEST(TypeUnit, ReductionOperator) TEST(TypeUnit, TypeCodeOf) { - EXPECT_EQ(legate::legate_type_code_of, legate::Type::Code::INVALID); + EXPECT_EQ(legate::legate_type_code_of, legate::Type::Code::NIL); EXPECT_EQ(legate::legate_type_code_of, legate::Type::Code::BOOL); EXPECT_EQ(legate::legate_type_code_of, legate::Type::Code::INT8); EXPECT_EQ(legate::legate_type_code_of, legate::Type::Code::INT16); @@ -478,7 +500,7 @@ TEST(TypeUnit, TypeOf) (std::is_same_v, complex>)); EXPECT_TRUE((std::is_same_v, std::string>)); - EXPECT_TRUE((std::is_same_v, void>)); + EXPECT_TRUE((std::is_same_v, void>)); EXPECT_TRUE((std::is_same_v, void>)); EXPECT_TRUE((std::is_same_v, void>)); EXPECT_TRUE((std::is_same_v, void>)); @@ -503,7 +525,7 @@ TEST(TypeUnit, TypeUtils) EXPECT_FALSE(legate::is_integral::value); EXPECT_FALSE(legate::is_integral::value); - EXPECT_FALSE(legate::is_integral::value); + EXPECT_FALSE(legate::is_integral::value); EXPECT_FALSE(legate::is_integral::value); EXPECT_FALSE(legate::is_integral::value); EXPECT_FALSE(legate::is_integral::value); @@ -526,7 +548,7 @@ TEST(TypeUnit, TypeUtils) EXPECT_FALSE(legate::is_signed::value); EXPECT_FALSE(legate::is_signed::value); - EXPECT_FALSE(legate::is_signed::value); + EXPECT_FALSE(legate::is_signed::value); EXPECT_FALSE(legate::is_signed::value); EXPECT_FALSE(legate::is_signed::value); EXPECT_FALSE(legate::is_signed::value); @@ -549,7 +571,7 @@ TEST(TypeUnit, TypeUtils) EXPECT_FALSE(legate::is_unsigned::value); EXPECT_FALSE(legate::is_unsigned::value); - EXPECT_FALSE(legate::is_unsigned::value); + EXPECT_FALSE(legate::is_unsigned::value); EXPECT_FALSE(legate::is_unsigned::value); EXPECT_FALSE(legate::is_unsigned::value); EXPECT_FALSE(legate::is_unsigned::value); @@ -572,7 +594,7 @@ TEST(TypeUnit, TypeUtils) EXPECT_FALSE(legate::is_floating_point::value); EXPECT_FALSE(legate::is_floating_point::value); - EXPECT_FALSE(legate::is_floating_point::value); + EXPECT_FALSE(legate::is_floating_point::value); EXPECT_FALSE(legate::is_floating_point::value); EXPECT_FALSE(legate::is_floating_point::value); EXPECT_FALSE(legate::is_floating_point::value); @@ -595,7 +617,7 @@ TEST(TypeUnit, TypeUtils) EXPECT_FALSE(legate::is_complex::value); EXPECT_FALSE(legate::is_complex::value); - EXPECT_FALSE(legate::is_complex::value); + EXPECT_FALSE(legate::is_complex::value); EXPECT_FALSE(legate::is_complex::value); EXPECT_FALSE(legate::is_complex::value); EXPECT_FALSE(legate::is_complex::value); From 1760c8df3efc1accb86a3399d7c371ec0f2f258b Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Mon, 11 Sep 2023 18:10:24 -0700 Subject: [PATCH 0210/1425] Partition manager shouldn't throw up on empty stores (#64) --- src/core/runtime/detail/partition_manager.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/runtime/detail/partition_manager.cc b/src/core/runtime/detail/partition_manager.cc index 41e3055471..0bcef0ab50 100644 --- a/src/core/runtime/detail/partition_manager.cc +++ b/src/core/runtime/detail/partition_manager.cc @@ -78,9 +78,9 @@ Shape PartitionManager::compute_launch_shape(const mapping::Machine& machine, // Figure out how many shards we can make with this array int64_t max_pieces = (volume + min_shard_volume_ - 1) / min_shard_volume_; - assert(max_pieces > 0); + assert(volume == 0 || max_pieces > 0); // If we can only make one piece return that now - if (1 == max_pieces) return {}; + if (max_pieces <= 1) return {}; // Otherwise we need to compute it ourselves // TODO: a better heuristic here. From 633c6b72dadb53d3e38cf4c5df5da060ec85673c Mon Sep 17 00:00:00 2001 From: Mona Han <129724851+mona-nv@users.noreply.github.com> Date: Tue, 12 Sep 2023 09:17:20 +0800 Subject: [PATCH 0211/1425] Add unit test for constraint api (#14) * Add unit test for constraint api * Update test script based on review comments --- tests/cpp/unit/constraint.cc | 152 +++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 tests/cpp/unit/constraint.cc diff --git a/tests/cpp/unit/constraint.cc b/tests/cpp/unit/constraint.cc new file mode 100644 index 0000000000..094b769f1d --- /dev/null +++ b/tests/cpp/unit/constraint.cc @@ -0,0 +1,152 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include + +#include "core/partitioning/detail/constraint.h" +#include "legate.h" + +namespace unit { +static const char* library_name = "test_constraints"; + +enum TaskIDs { + INIT = 0, +}; + +// Dummy task to make the runtime think the store is initialized +struct Initializer : public legate::LegateTask { + static const int32_t TASK_ID = INIT; + static void cpu_variant(legate::TaskContext context) {} +}; + +void register_tasks() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + Initializer::register_variants(context); +} + +TEST(Variable, BasicMethods) +{ + legate::Core::perform_registration(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + auto task = runtime->create_task(context, INIT); + + // Test basic properties + auto part = task.declare_partition(); + auto part_imp = part.impl(); + EXPECT_FALSE(part_imp->closed()); + EXPECT_EQ(part_imp->kind(), legate::detail::Expr::Kind::VARIABLE); + EXPECT_EQ(part_imp->as_literal(), nullptr); + EXPECT_EQ(part_imp->as_variable(), part_imp); + EXPECT_TRUE(part_imp->operation() != nullptr); + + // Test equal + auto part1(part); + auto part1_imp = part1.impl(); + EXPECT_EQ(*part_imp, *part1_imp); + auto part2 = task.declare_partition(); + auto part2_imp = part2.impl(); + EXPECT_TRUE(*part_imp < *part2_imp); + + // Test find_partition_symbols + std::vector symbols = {}; + part_imp->find_partition_symbols(symbols); + part1_imp->find_partition_symbols(symbols); + part2_imp->find_partition_symbols(symbols); + EXPECT_EQ(symbols.size(), 3); + EXPECT_TRUE(std::find(symbols.begin(), symbols.end(), part_imp) != symbols.end()); + EXPECT_TRUE(std::find(symbols.begin(), symbols.end(), part1_imp) != symbols.end()); + EXPECT_TRUE(std::find(symbols.begin(), symbols.end(), part2_imp) != symbols.end()); +} + +TEST(Alignment, BasicMethods) +{ + legate::Core::perform_registration(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + auto task = runtime->create_task(context, INIT); + + auto part1 = task.declare_partition(); + auto part2 = task.declare_partition(); + + auto aligment = legate::detail::align(part1.impl(), part2.impl()); + EXPECT_EQ(aligment->kind(), legate::detail::Constraint::Kind::ALIGNMENT); + EXPECT_EQ(aligment->lhs(), part1.impl()); + EXPECT_EQ(aligment->rhs(), part2.impl()); + EXPECT_EQ(aligment->as_alignment(), aligment.get()); + EXPECT_EQ(aligment->as_broadcast(), nullptr); + EXPECT_EQ(aligment->as_image_constraint(), nullptr); + EXPECT_FALSE(aligment->is_trivial()); + + // Test find_partition_symbols + std::vector symbols = {}; + aligment->find_partition_symbols(symbols); + EXPECT_EQ(symbols.size(), 2); + EXPECT_TRUE(std::find(symbols.begin(), symbols.end(), part1.impl()) != symbols.end()); + EXPECT_TRUE(std::find(symbols.begin(), symbols.end(), part2.impl()) != symbols.end()); +} + +TEST(Broadcast, BasicMethods) +{ + legate::Core::perform_registration(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + auto task = runtime->create_task(context, INIT); + auto part1 = task.declare_partition(); + + auto dims = legate::from_range(3); + auto broadcast = legate::detail::broadcast(part1.impl(), dims); + EXPECT_EQ(broadcast->kind(), legate::detail::Constraint::Kind::BROADCAST); + EXPECT_EQ(broadcast->variable(), part1.impl()); + EXPECT_EQ(broadcast->axes(), dims); + EXPECT_EQ(broadcast->as_alignment(), nullptr); + EXPECT_EQ(broadcast->as_broadcast(), broadcast.get()); + EXPECT_EQ(broadcast->as_image_constraint(), nullptr); + + // Test find_partition_symbols + std::vector symbols = {}; + broadcast->find_partition_symbols(symbols); + EXPECT_EQ(symbols.size(), 1); + EXPECT_TRUE(std::find(symbols.begin(), symbols.end(), part1.impl()) != symbols.end()); +} + +TEST(ImageConstraint, BasicMethods) +{ + legate::Core::perform_registration(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + auto task = runtime->create_task(context, INIT); + auto part_func = task.declare_partition(); + auto part_range = task.declare_partition(); + + auto image_constraint = legate::detail::image(part_func.impl(), part_range.impl()); + EXPECT_EQ(image_constraint->kind(), legate::detail::Constraint::Kind::IMAGE); + EXPECT_EQ(image_constraint->var_function(), part_func.impl()); + EXPECT_EQ(image_constraint->var_range(), part_range.impl()); + EXPECT_EQ(image_constraint->as_alignment(), nullptr); + EXPECT_EQ(image_constraint->as_broadcast(), nullptr); + EXPECT_EQ(image_constraint->as_image_constraint(), image_constraint.get()); + + // Test find_partition_symbols + std::vector symbols = {}; + image_constraint->find_partition_symbols(symbols); + EXPECT_EQ(symbols.size(), 2); + EXPECT_TRUE(std::find(symbols.begin(), symbols.end(), part_func.impl()) != symbols.end()); + EXPECT_TRUE(std::find(symbols.begin(), symbols.end(), part_range.impl()) != symbols.end()); +} +} // namespace unit \ No newline at end of file From 58579108e2ed0bfe6bf0b236c04ddf5daa6c9b15 Mon Sep 17 00:00:00 2001 From: Mona Han <129724851+mona-nv@users.noreply.github.com> Date: Tue, 12 Sep 2023 09:19:45 +0800 Subject: [PATCH 0212/1425] Add more public API test in machine unit test (#8) --- tests/cpp/unit/machine.cc | 57 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/cpp/unit/machine.cc b/tests/cpp/unit/machine.cc index 7118296e8a..50d0a63fa3 100644 --- a/tests/cpp/unit/machine.cc +++ b/tests/cpp/unit/machine.cc @@ -48,6 +48,17 @@ TEST(Machine, ProcessorRange) EXPECT_THROW(range.get_node_range(), std::invalid_argument); } + // create another empty + { + legate::mapping::ProcessorRange range(3, 3, 0); + EXPECT_TRUE(range.empty()); + EXPECT_EQ(range.per_node_count, 1); + EXPECT_EQ(range.low, 0); + EXPECT_EQ(range.high, 0); + EXPECT_EQ(range.count(), 0); + EXPECT_THROW(range.get_node_range(), std::invalid_argument); + } + // check defaults { legate::mapping::ProcessorRange range; @@ -58,6 +69,23 @@ TEST(Machine, ProcessorRange) EXPECT_EQ(range.count(), 0); } + // test equal and comparison + { + legate::mapping::ProcessorRange range1(2, 6, 2); + legate::mapping::ProcessorRange range2(2, 6, 2); + EXPECT_EQ(range1, range2); + + legate::mapping::ProcessorRange range3(1, 6, 2); + EXPECT_NE(range1, range3); + EXPECT_TRUE(range3 < range1); + + legate::mapping::ProcessorRange range4(2, 5, 2); + EXPECT_TRUE(range4 < range1); + + legate::mapping::ProcessorRange range5(2, 6, 1); + EXPECT_TRUE(range5 < range1); + } + // get_node_range { legate::mapping::ProcessorRange range(0, 7, 2); @@ -93,6 +121,7 @@ TEST(Machine, ProcessorRange) legate::mapping::ProcessorRange range(1, 3, 1); EXPECT_EQ(range.slice(0, 0).count(), 0); EXPECT_EQ(range.slice(4, 6).count(), 0); + EXPECT_EQ(range.slice(1, 0).count(), 0); } // nonempty slice nonempty range @@ -117,6 +146,13 @@ TEST(Machine, MachineDesc) EXPECT_EQ(machine.slice(0, 1), legate::mapping::detail::Machine( {{legate::mapping::TaskTarget::CPU, legate::mapping::ProcessorRange()}})); + EXPECT_TRUE(machine.empty()); + EXPECT_EQ(machine.valid_targets().size(), 0); + EXPECT_EQ(machine.only(legate::mapping::TaskTarget::CPU), + legate::mapping::detail::Machine( + {{legate::mapping::TaskTarget::CPU, legate::mapping::ProcessorRange()}})); + std::map processor_ranges = {}; + EXPECT_EQ(machine.processor_ranges, processor_ranges); } legate::mapping::ProcessorRange cpu_range(1, 3, 4); @@ -265,11 +301,26 @@ TEST(Machine, MachineDesc) auto new_machine2 = machine2.slice(0, 2); EXPECT_EQ(new_machine2.preferred_target, legate::mapping::TaskTarget::GPU); EXPECT_EQ(new_machine2.processor_range().count(), 2); + + legate::mapping::detail::Machine machine3({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::OMP, omp_range}, + {legate::mapping::TaskTarget::GPU, gpu_range}}); + legate::mapping::detail::Machine expected1( + {{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::OMP, omp_range}, + {legate::mapping::TaskTarget::GPU, gpu_range.slice(0, 1)}}); + EXPECT_EQ(machine3.slice(0, 1, true), expected1); + legate::mapping::detail::Machine expected2( + {{legate::mapping::TaskTarget::CPU, cpu_range.slice(1, 2)}, + {legate::mapping::TaskTarget::OMP, omp_range}, + {legate::mapping::TaskTarget::GPU, gpu_range}}); + EXPECT_EQ(machine3.slice(1, 2, legate::mapping::TaskTarget::CPU, true), expected2); } // test operator[] { legate::mapping::detail::Machine machine({{legate::mapping::TaskTarget::CPU, cpu_range}, + {legate::mapping::TaskTarget::OMP, omp_range}, {legate::mapping::TaskTarget::GPU, gpu_range}}); auto machine1 = machine[legate::mapping::TaskTarget::GPU]; @@ -278,6 +329,12 @@ TEST(Machine, MachineDesc) EXPECT_EQ(machine1.count(), 3); EXPECT_EQ(machine1.preferred_target, legate::mapping::TaskTarget::GPU); EXPECT_EQ(machine1.processor_range().per_node_count, 3); + + auto machine2 = machine[{legate::mapping::TaskTarget::CPU, legate::mapping::TaskTarget::OMP}]; + auto valid_targets2 = machine2.valid_targets(); + EXPECT_EQ(valid_targets2.size(), 2); + EXPECT_EQ(machine2.preferred_target, legate::mapping::TaskTarget::OMP); + EXPECT_EQ(machine2.processor_range().per_node_count, 2); } // test intersection From a4fa0f5a2323a30c7059952ea95f1d7ff3af4394 Mon Sep 17 00:00:00 2001 From: Joy Shen <110752938+joyshennv@users.noreply.github.com> Date: Tue, 12 Sep 2023 09:28:56 +0800 Subject: [PATCH 0213/1425] Add unit tests for ScopedAllocator. (#11) * Add unit tests for ScopedAllocator. * Address review comments. * Fix broken test case where bytes are 0. * Address another review comments. --- tests/cpp/unit/scoped_allocator.cc | 139 +++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 tests/cpp/unit/scoped_allocator.cc diff --git a/tests/cpp/unit/scoped_allocator.cc b/tests/cpp/unit/scoped_allocator.cc new file mode 100644 index 0000000000..099ad9a366 --- /dev/null +++ b/tests/cpp/unit/scoped_allocator.cc @@ -0,0 +1,139 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include +#include "legate.h" + +namespace scoped_allocator_test { + +static const char* library_name = "legate.scopedallocator"; +constexpr int64_t ALLOCATOR_TASK_ID = 1; + +struct AllocatorParams { + uint32_t op_code; + uint64_t kind; + bool scoped; + uint64_t alignment; + uint64_t bytes; +}; + +enum class BufferOpCode : uint32_t { + DEALLOCATE = 0, + DOUBLE_DEALLOCATE = 1, + INVALID_DEALLOCATE = 2, +}; + +struct ScopedAllocatorTask : public legate::LegateTask { + static const int32_t TASK_ID = ALLOCATOR_TASK_ID; + static void cpu_variant(legate::TaskContext context); +}; + +/*static*/ void ScopedAllocatorTask::cpu_variant(legate::TaskContext context) +{ + auto allocator_params = context.scalar(0).value(); + auto allocator = legate::ScopedAllocator(static_cast(allocator_params.kind), + allocator_params.scoped, + allocator_params.alignment); + auto buffer = allocator.allocate(allocator_params.bytes); + if (0 == allocator_params.bytes) { + EXPECT_EQ(buffer, nullptr); + } else { + EXPECT_NE(buffer, nullptr); + } + + BufferOpCode code = static_cast(allocator_params.op_code); + switch (code) { + case BufferOpCode::DEALLOCATE: { + if (0 == allocator_params.bytes) { + EXPECT_THROW(allocator.deallocate(buffer), std::runtime_error); + } else { + allocator.deallocate(buffer); + } + break; + } + case BufferOpCode::DOUBLE_DEALLOCATE: { + allocator.deallocate(buffer); + EXPECT_THROW(allocator.deallocate(buffer), std::runtime_error); + break; + } + case BufferOpCode::INVALID_DEALLOCATE: { + EXPECT_THROW(allocator.deallocate(static_cast(static_cast(buffer) + 1)), + std::runtime_error); + break; + } + } +} + +void test_allocator( + BufferOpCode op_code, legate::Memory::Kind kind, bool scoped, size_t bytes, size_t alignment = 16) +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + auto task = runtime->create_task(context, ALLOCATOR_TASK_ID); + auto part = task.declare_partition(); + AllocatorParams struct_data = { + static_cast(op_code), static_cast(kind), scoped, alignment, bytes}; + task.add_scalar_arg(legate::Scalar(struct_data, + legate::struct_type(true, + legate::uint32(), + legate::uint64(), + legate::bool_(), + legate::uint64(), + legate::uint64()))); + runtime->submit(std::move(task)); +} + +void register_tasks() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + ScopedAllocatorTask::register_variants(context); +} + +TEST(ScopedAllocatorUnit, EmptyAllocate) +{ + auto allocator = legate::ScopedAllocator(legate::Memory::SYSTEM_MEM, true); + void* ptr = allocator.allocate(0); + EXPECT_EQ(ptr, nullptr); +} + +TEST(ScopedAllocatorUnit, Allocate) +{ + legate::Core::perform_registration(); + test_allocator(BufferOpCode::DEALLOCATE, legate::Memory::SYSTEM_MEM, true, 1000, 16); + test_allocator(BufferOpCode::DEALLOCATE, legate::Memory::NO_MEMKIND, true, 1000, 16); + test_allocator(BufferOpCode::DEALLOCATE, legate::Memory::SYSTEM_MEM, false, 0, 16); + test_allocator(BufferOpCode::DEALLOCATE, legate::Memory::SYSTEM_MEM, false, 1000, 0); + + // Note: test passes when bytes / alignment set to -1. + // Todo: Need to add negative test after issue #31 is fixed. + // test_allocator(BufferOpCode::DEALLOCATE, legate::Memory::SYSTEM_MEM, false, -1, 16); + // test_allocator(BufferOpCode::DEALLOCATE, legate::Memory::SYSTEM_MEM, false, 1000, -1); +} + +TEST(ScopedAllocatorUnit, DoubleDeallocate) +{ + legate::Core::perform_registration(); + test_allocator(BufferOpCode::DOUBLE_DEALLOCATE, legate::Memory::SYSTEM_MEM, true, 1000); +} + +TEST(ScopedAllocatorUnit, InvalidDeallocate) +{ + auto allocator = legate::ScopedAllocator(legate::Memory::SYSTEM_MEM, true); + std::vector data = {1, 2, 3}; + EXPECT_THROW(allocator.deallocate(data.data()), std::runtime_error); + + // invalid deallocate in task launch + legate::Core::perform_registration(); + test_allocator(BufferOpCode::INVALID_DEALLOCATE, legate::Memory::SYSTEM_MEM, true, 1000); +} +} // namespace scoped_allocator_test From b92073d354767932b79006e1da67c062fe4071ec Mon Sep 17 00:00:00 2001 From: Joy Shen <110752938+joyshennv@users.noreply.github.com> Date: Tue, 12 Sep 2023 09:29:50 +0800 Subject: [PATCH 0214/1425] Add unit tests for buffer.h (#12) * Add unit tests for buffer.h * Address review comments. * Address another review comments. --- tests/cpp/unit/buffer.cc | 116 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 tests/cpp/unit/buffer.cc diff --git a/tests/cpp/unit/buffer.cc b/tests/cpp/unit/buffer.cc new file mode 100644 index 0000000000..ad4736669f --- /dev/null +++ b/tests/cpp/unit/buffer.cc @@ -0,0 +1,116 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include +#include "legate.h" + +namespace buffer_test { + +static const char* library_name = "legate.buffer"; +constexpr int64_t BUFFER_TASK_ID = 0; + +struct BufferParams { + int32_t dim; + uint64_t bytes; + uint64_t kind; + uint64_t alignment; +}; + +struct buffer_fn { + template + void operator()(uint64_t bytes, uint64_t kind, uint64_t alignment) + { + // Todo: add check for memory size after API's ready. + switch (DIM) { + case 1: { + auto buffer = legate::create_buffer( + bytes, static_cast(kind), alignment); + auto memory = buffer.get_instance().get_location(); + EXPECT_TRUE(memory.exists()); + // NO_MEMKIND on a cpu is always mapped to SYSTEM_MEM + EXPECT_EQ(memory.kind(), + kind == legate::Memory::NO_MEMKIND ? legate::Memory::SYSTEM_MEM : kind); + break; + } + default: { + auto buffer = legate::create_buffer( + legate::Point(bytes), static_cast(kind), alignment); + auto memory = buffer.get_instance().get_location(); + EXPECT_TRUE(memory.exists()); + // NO_MEMKIND on a cpu is always mapped to SYSTEM_MEM + EXPECT_EQ(memory.kind(), + kind == legate::Memory::NO_MEMKIND ? legate::Memory::SYSTEM_MEM : kind); + break; + } + } + } +}; + +struct BufferTask : public legate::LegateTask { + static const int32_t TASK_ID = BUFFER_TASK_ID; + static void cpu_variant(legate::TaskContext context); +}; + +/*static*/ void BufferTask::cpu_variant(legate::TaskContext context) +{ + auto buffer_params = context.scalar(0).value(); + legate::dim_dispatch(buffer_params.dim, + buffer_fn{}, + buffer_params.bytes, + buffer_params.kind, + buffer_params.alignment); +} + +void test_buffer(int32_t dim, uint64_t bytes, legate::Memory::Kind kind, size_t alignment = 16) +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + auto task = runtime->create_task(context, BUFFER_TASK_ID); + BufferParams param = {dim, bytes, static_cast(kind), alignment}; + task.add_scalar_arg( + legate::Scalar(param, + legate::struct_type( + true, legate::int32(), legate::uint64(), legate::uint64(), legate::uint64()))); + runtime->submit(std::move(task)); +} + +void register_tasks() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + BufferTask::register_variants(context); +} + +TEST(BufferUnit, CreateBuffer) +{ + legate::Core::perform_registration(); + + // Todo: need to add tests for REGDMA_MEM + test_buffer(1, 10, legate::Memory::SYSTEM_MEM); + test_buffer(2, 10, legate::Memory::NO_MEMKIND); + test_buffer(3, 10, legate::Memory::SYSTEM_MEM); + test_buffer(4, 10, legate::Memory::SYSTEM_MEM); +} + +TEST(BufferUnit, NegativeTest) +{ + legate::Core::perform_registration(); + + test_buffer(1, 0, legate::Memory::SYSTEM_MEM); + test_buffer(2, 10, legate::Memory::SYSTEM_MEM, 0); + + // Note: test passes when bytes / alignment set to -1. + // Todo: Need to add negative test after issue #31 is fixed. + // test_buffer(3, -1, legate::Memory::NO_MEMKIND); + // test_buffer(4, 10, legate::Memory::SYSTEM_MEM, -1); +} +} // namespace buffer_test From 8290628006a01ee1efda76d8b7393f97aba23807 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Mon, 11 Sep 2023 21:30:01 -0700 Subject: [PATCH 0215/1425] Bloating constraints (#65) * Boilerplate for bloating constraints * Implement bloated partitions * Docstring for bloating constraints --- src/core/partitioning/constraint.cc | 9 +++ src/core/partitioning/constraint.h | 28 ++++++++- src/core/partitioning/detail/constraint.cc | 55 ++++++++++++++++ src/core/partitioning/detail/constraint.h | 54 +++++++++++++++- .../partitioning/detail/constraint_solver.cc | 16 +++++ src/core/partitioning/partition.cc | 62 +++++++++++++++++-- src/core/partitioning/partition.h | 20 +++++- 7 files changed, 237 insertions(+), 7 deletions(-) diff --git a/src/core/partitioning/constraint.cc b/src/core/partitioning/constraint.cc index a31bdeb9dd..e273443f60 100644 --- a/src/core/partitioning/constraint.cc +++ b/src/core/partitioning/constraint.cc @@ -45,4 +45,13 @@ Constraint scale(const Shape& factors, Variable var_smaller, Variable var_bigger return Constraint(detail::scale(factors, var_smaller.impl(), var_bigger.impl()).release()); } +Constraint bloat(Variable var_source, + Variable var_bloat, + const Shape& low_offsets, + const Shape& high_offsets) +{ + return Constraint( + detail::bloat(var_source.impl(), var_bloat.impl(), low_offsets, high_offsets).release()); +} + } // namespace legate diff --git a/src/core/partitioning/constraint.h b/src/core/partitioning/constraint.h index ce4575ceba..206a60d76c 100644 --- a/src/core/partitioning/constraint.h +++ b/src/core/partitioning/constraint.h @@ -122,10 +122,36 @@ Constraint image(Variable var_function, Variable var_range); * @param factors Scaling factors * @param var_smaller Partition symbol for the smaller store (i.e., the one whose extents are * scaled) - * @param var_bigger Partition symbol of the bigger store + * @param var_bigger Partition symbol for the bigger store * * @return Scaling constraint */ Constraint scale(const Shape& factors, Variable var_smaller, Variable var_bigger); +/** + * @ingroup partitioning + * @brief Creates a bloating constraint between partitions + * + * If two stores `A` and `B` are constrained by a bloating constraint + * + * `legate::bloat(pA, pB, L, H)` + * + * where `pA` and `pB ` are partition symbols for `A` and `B`, respectively, `A` and `B` will be + * partitioned such that each pair of sub-stores `Ak` and `Bk` satisfy the following property: + * + * @f$ \forall p \in \mathit{dom}(\mathtt{Ak}). \forall \delta \in [-\mathtt{L}, \mathtt{H}]. @f$ + * @f$ p + \delta \in \mathit{dom}(\mathtt{Bk}) \lor p + \delta \not \in \mathit{dom}(\mathtt{B})@f$ + * + * @param var_source Partition symbol for the source store + * @param var_bloat Partition symbol for the target store + * @param low_offsets Offsets to bloat towards the negative direction + * @param high_offsets Offsets to bloat towards the positive direction + * + * @return Bloating constraint + */ +Constraint bloat(Variable var_source, + Variable var_bloat, + const Shape& low_offsets, + const Shape& high_offsets); + } // namespace legate diff --git a/src/core/partitioning/detail/constraint.cc b/src/core/partitioning/detail/constraint.cc index 08cb501387..ce4c8988c0 100644 --- a/src/core/partitioning/detail/constraint.cc +++ b/src/core/partitioning/detail/constraint.cc @@ -192,6 +192,53 @@ std::unique_ptr ScaleConstraint::resolve(const detail::Strategy& stra return src_part->scale(factors_); } +BloatConstraint::BloatConstraint(const Variable* var_source, + const Variable* var_bloat, + const Shape& low_offsets, + const Shape& high_offsets) + : var_source_(var_source), + var_bloat_(var_bloat), + low_offsets_(low_offsets), + high_offsets_(high_offsets) +{ +} + +void BloatConstraint::find_partition_symbols(std::vector& partition_symbols) const +{ + partition_symbols.push_back(var_source_); + partition_symbols.push_back(var_bloat_); +} + +void BloatConstraint::validate() const +{ + auto source = var_source_->operation()->find_store(var_source_); + auto bloat = var_bloat_->operation()->find_store(var_bloat_); + + if (source->dim() != bloat->dim()) { + throw std::invalid_argument( + "Bloating constraint requires the stores to have the same number of dimensions"); + } + + if (source->dim() != low_offsets_.size() || source->dim() != high_offsets_.size()) { + throw std::invalid_argument( + "Bloating constraint requires the number of offsets to match the number of dimensions"); + } +} + +std::string BloatConstraint::to_string() const +{ + std::stringstream ss; + ss << "BloatConstraint(" << var_source_->to_string() << ", " << var_bloat_->to_string() + << ", low: " << low_offsets_ << ", high: " << high_offsets_ << ")"; + return std::move(ss).str(); +} + +std::unique_ptr BloatConstraint::resolve(const detail::Strategy& strategy) const +{ + auto src_part = strategy[var_source()]; + return src_part->bloat(low_offsets_, high_offsets_); +} + std::unique_ptr align(const Variable* lhs, const Variable* rhs) { return std::make_unique(lhs, rhs); @@ -215,6 +262,14 @@ std::unique_ptr scale(const Shape& factors, return std::make_unique(factors, var_smaller, var_bigger); } +std::unique_ptr bloat(const Variable* var_source, + const Variable* var_bloat, + const Shape& low_offsets, + const Shape& high_offsets) +{ + return std::make_unique(var_source, var_bloat, low_offsets, high_offsets); +} + } // namespace legate::detail // explicitly instantiate the deleter for std::unique_ptr diff --git a/src/core/partitioning/detail/constraint.h b/src/core/partitioning/detail/constraint.h index 71cb7d08b6..d9108a6360 100644 --- a/src/core/partitioning/detail/constraint.h +++ b/src/core/partitioning/detail/constraint.h @@ -26,6 +26,7 @@ class Operation; class Strategy; class Alignment; +class BloatConstraint; class Broadcast; struct Constraint; class ImageConstraint; @@ -118,6 +119,7 @@ struct Constraint { BROADCAST = 1, IMAGE = 2, SCALE = 3, + BLOAT = 4, }; virtual ~Constraint() {} virtual void find_partition_symbols(std::vector& partition_symbols) const = 0; @@ -128,6 +130,7 @@ struct Constraint { virtual const Broadcast* as_broadcast() const = 0; virtual const ImageConstraint* as_image_constraint() const = 0; virtual const ScaleConstraint* as_scale_constraint() const = 0; + virtual const BloatConstraint* as_bloat_constraint() const = 0; }; class Alignment : public Constraint { @@ -151,6 +154,7 @@ class Alignment : public Constraint { const Broadcast* as_broadcast() const override { return nullptr; } const ImageConstraint* as_image_constraint() const override { return nullptr; } const ScaleConstraint* as_scale_constraint() const override { return nullptr; } + const BloatConstraint* as_bloat_constraint() const override { return nullptr; } public: const Variable* lhs() const { return lhs_; } @@ -185,6 +189,7 @@ class Broadcast : public Constraint { const Broadcast* as_broadcast() const override { return this; } const ImageConstraint* as_image_constraint() const override { return nullptr; } const ScaleConstraint* as_scale_constraint() const override { return nullptr; } + const BloatConstraint* as_bloat_constraint() const override { return nullptr; } public: const Variable* variable() const { return variable_; } @@ -216,6 +221,7 @@ class ImageConstraint : public Constraint { const Broadcast* as_broadcast() const override { return nullptr; } const ImageConstraint* as_image_constraint() const override { return this; } const ScaleConstraint* as_scale_constraint() const override { return nullptr; } + const BloatConstraint* as_bloat_constraint() const override { return nullptr; } public: const Variable* var_function() const { return var_function_; } @@ -250,6 +256,7 @@ class ScaleConstraint : public Constraint { const Broadcast* as_broadcast() const override { return nullptr; } const ImageConstraint* as_image_constraint() const override { return nullptr; } const ScaleConstraint* as_scale_constraint() const override { return this; } + const BloatConstraint* as_bloat_constraint() const override { return nullptr; } public: const Variable* var_smaller() const { return var_smaller_; } @@ -259,11 +266,51 @@ class ScaleConstraint : public Constraint { std::unique_ptr resolve(const Strategy& strategy) const; private: - Shape factors_; + const Shape factors_; const Variable* var_smaller_; const Variable* var_bigger_; }; +class BloatConstraint : public Constraint { + public: + BloatConstraint(const Variable* var_source, + const Variable* var_bloat, + const Shape& low_offsets, + const Shape& high_offsets); + + public: + Kind kind() const override { return Kind::BLOAT; } + + public: + void find_partition_symbols(std::vector& partition_symbols) const override; + + public: + void validate() const override; + + public: + std::string to_string() const override; + + public: + const Alignment* as_alignment() const override { return nullptr; } + const Broadcast* as_broadcast() const override { return nullptr; } + const ImageConstraint* as_image_constraint() const override { return nullptr; } + const ScaleConstraint* as_scale_constraint() const override { return nullptr; } + const BloatConstraint* as_bloat_constraint() const override { return this; } + + public: + const Variable* var_source() const { return var_source_; } + const Variable* var_bloat() const { return var_bloat_; } + + public: + std::unique_ptr resolve(const Strategy& strategy) const; + + private: + const Variable* var_source_; + const Variable* var_bloat_; + const Shape low_offsets_; + const Shape high_offsets_; +}; + std::unique_ptr align(const Variable* lhs, const Variable* rhs); std::unique_ptr broadcast(const Variable* variable, const tuple& axes); @@ -274,4 +321,9 @@ std::unique_ptr scale(const Shape& factors, const Variable* var_smaller, const Variable* var_bigger); +std::unique_ptr bloat(const Variable* var_source, + const Variable* var_bloat, + const Shape& low_offsets, + const Shape& high_offsets); + } // namespace legate::detail diff --git a/src/core/partitioning/detail/constraint_solver.cc b/src/core/partitioning/detail/constraint_solver.cc index 06564ea325..844f1d6daa 100644 --- a/src/core/partitioning/detail/constraint_solver.cc +++ b/src/core/partitioning/detail/constraint_solver.cc @@ -162,6 +162,9 @@ void ConstraintSolver::solve_constraints() auto handle_scale_constraint = [&](const ScaleConstraint* scale_constraint) { is_dependent_[*scale_constraint->var_bigger()] = true; }; + auto handle_bloat_constraint = [&](const BloatConstraint* bloat_constraint) { + is_dependent_[*bloat_constraint->var_bloat()] = true; + }; // Reflect each constraint to the solver state for (auto& constraint : constraints_) { @@ -183,6 +186,10 @@ void ConstraintSolver::solve_constraints() handle_scale_constraint(constraint->as_scale_constraint()); break; } + case Constraint::Kind::BLOAT: { + handle_bloat_constraint(constraint->as_bloat_constraint()); + break; + } } } @@ -209,6 +216,11 @@ void ConstraintSolver::solve_dependent_constraints(Strategy& strategy) strategy.insert(scale_constraint->var_bigger(), std::move(scaled)); }; + auto solve_bloat_constraint = [&strategy](const BloatConstraint* bloat_constraint) { + auto bloated = bloat_constraint->resolve(strategy); + strategy.insert(bloat_constraint->var_bloat(), std::move(bloated)); + }; + for (auto& constraint : constraints_) { switch (constraint->kind()) { case Constraint::Kind::IMAGE: { @@ -219,6 +231,10 @@ void ConstraintSolver::solve_dependent_constraints(Strategy& strategy) solve_scale_constraint(constraint->as_scale_constraint()); break; } + case Constraint::Kind::BLOAT: { + solve_bloat_constraint(constraint->as_bloat_constraint()); + break; + } default: { continue; } diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index ee886f910c..5ec99353a6 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -39,6 +39,12 @@ std::unique_ptr NoPartition::scale(const Shape& factors) const return create_no_partition(); } +std::unique_ptr NoPartition::bloat(const Shape& low_offsts, + const Shape& high_offsets) const +{ + return create_no_partition(); +} + Legion::LogicalPartition NoPartition::construct(Legion::LogicalRegion region, bool complete) const { return Legion::LogicalPartition::NO_PART; @@ -58,9 +64,24 @@ std::string NoPartition::to_string() const { return "NoPartition"; } Tiling::Tiling(Shape&& tile_shape, Shape&& color_shape, tuple&& offsets) : Partition(), + disjoint_(true), tile_shape_(std::move(tile_shape)), color_shape_(std::move(color_shape)), - offsets_(std::move(offsets)) + offsets_(std::move(offsets)), + strides_(tile_shape_) +{ + if (offsets_.empty()) offsets_ = tuple(tile_shape_.size(), 0); + assert(tile_shape_.size() == color_shape_.size()); + assert(tile_shape_.size() == offsets_.size()); +} + +Tiling::Tiling(Shape&& tile_shape, Shape&& color_shape, tuple&& offsets, Shape&& strides) + : Partition(), + disjoint_(!(strides < tile_shape)), + tile_shape_(std::move(tile_shape)), + color_shape_(std::move(color_shape)), + offsets_(std::move(offsets)), + strides_(std::move(strides)) { if (offsets_.empty()) offsets_ = tuple(tile_shape_.size(), 0); assert(tile_shape_.size() == color_shape_.size()); @@ -115,7 +136,8 @@ bool Tiling::is_disjoint_for(const Domain* launch_domain) const { // TODO: The check really should be that every two points from the launch domain are mapped // to two different colors - return nullptr == launch_domain || launch_domain->get_volume() <= color_shape_.volume(); + return disjoint_ && + (nullptr == launch_domain || launch_domain->get_volume() <= color_shape_.volume()); } bool Tiling::satisfies_restrictions(const Restrictions& restrictions) const @@ -133,6 +155,17 @@ std::unique_ptr Tiling::scale(const Shape& factors) const return create_tiling(tile_shape_ * factors, Shape(color_shape_), std::move(new_offsets)); } +std::unique_ptr Tiling::bloat(const Shape& low_offsets, const Shape& high_offsets) const +{ + auto tile_shape = tile_shape_ + low_offsets + high_offsets; + auto offsets = apply([](int64_t off, size_t diff) { return off - static_cast(diff); }, + offsets_, + low_offsets); + + return create_tiling( + std::move(tile_shape), Shape(color_shape_), std::move(offsets), Shape(tile_shape_)); +} + Legion::LogicalPartition Tiling::construct(Legion::LogicalRegion region, bool complete) const { auto index_space = region.get_index_space(); @@ -148,7 +181,7 @@ Legion::LogicalPartition Tiling::construct(Legion::LogicalRegion region, bool co transform.m = ndim; transform.n = ndim; for (int32_t idx = 0; idx < ndim * ndim; ++idx) transform.matrix[idx] = 0; - for (int32_t idx = 0; idx < ndim; ++idx) transform.matrix[ndim * idx + idx] = tile_shape_[idx]; + for (int32_t idx = 0; idx < ndim; ++idx) transform.matrix[ndim * idx + idx] = strides_[idx]; auto extent = to_domain(tile_shape_); for (int32_t idx = 0; idx < ndim; ++idx) { @@ -177,7 +210,7 @@ std::string Tiling::to_string() const { std::stringstream ss; ss << "Tiling(tile:" << tile_shape_ << ",colors:" << color_shape_ << ",offset:" << offsets_ - << ")"; + << ",strides:" << strides_ << ")"; return std::move(ss).str(); } @@ -238,6 +271,12 @@ std::unique_ptr Weighted::scale(const Shape& factors) const return nullptr; } +std::unique_ptr Weighted::bloat(const Shape& low_offsts, const Shape& high_offsets) const +{ + throw std::runtime_error("Not implemented"); + return nullptr; +} + Legion::LogicalPartition Weighted::construct(Legion::LogicalRegion region, bool) const { auto runtime = detail::Runtime::get_runtime(); @@ -318,6 +357,12 @@ std::unique_ptr Image::scale(const Shape& factors) const return nullptr; } +std::unique_ptr Image::bloat(const Shape& low_offsts, const Shape& high_offsets) const +{ + throw std::runtime_error("Not implemented"); + return nullptr; +} + Legion::LogicalPartition Image::construct(Legion::LogicalRegion region, bool complete) const { if (!has_launch_domain()) { return Legion::LogicalPartition::NO_PART; } @@ -376,6 +421,15 @@ std::unique_ptr create_tiling(Shape&& tile_shape, std::move(tile_shape), std::move(color_shape), std::move(offsets)); } +std::unique_ptr create_tiling(Shape&& tile_shape, + Shape&& color_shape, + tuple&& offsets, + Shape&& strides) +{ + return std::make_unique( + std::move(tile_shape), std::move(color_shape), std::move(offsets), std::move(strides)); +} + std::unique_ptr create_weighted(const Legion::FutureMap& weights, const Domain& color_domain) { diff --git a/src/core/partitioning/partition.h b/src/core/partitioning/partition.h index 47534bc239..d8c725c1ff 100644 --- a/src/core/partitioning/partition.h +++ b/src/core/partitioning/partition.h @@ -54,7 +54,9 @@ struct Partition { virtual bool is_convertible() const = 0; public: - virtual std::unique_ptr scale(const Shape& factors) const = 0; + virtual std::unique_ptr scale(const Shape& factors) const = 0; + virtual std::unique_ptr bloat(const Shape& low_offsets, + const Shape& high_offsets) const = 0; public: virtual Legion::LogicalPartition construct(Legion::LogicalRegion region, @@ -89,6 +91,8 @@ class NoPartition : public Partition { public: std::unique_ptr scale(const Shape& factors) const override; + std::unique_ptr bloat(const Shape& low_offsets, + const Shape& high_offsets) const override; public: Legion::LogicalPartition construct(Legion::LogicalRegion region, bool complete) const override; @@ -114,6 +118,7 @@ class NoPartition : public Partition { class Tiling : public Partition { public: Tiling(Shape&& tile_shape, Shape&& color_shape, tuple&& offsets); + Tiling(Shape&& tile_shape, Shape&& color_shape, tuple&& offsets, Shape&& strides); public: Tiling(const Tiling&) = default; @@ -133,6 +138,8 @@ class Tiling : public Partition { public: std::unique_ptr scale(const Shape& factors) const override; + std::unique_ptr bloat(const Shape& low_offsets, + const Shape& high_offsets) const override; public: Legion::LogicalPartition construct(Legion::LogicalRegion region, bool complete) const override; @@ -157,9 +164,11 @@ class Tiling : public Partition { Shape get_child_offsets(const Shape& color); private: + bool disjoint_; Shape tile_shape_; Shape color_shape_; tuple offsets_; + Shape strides_; }; class Weighted : public Partition { @@ -184,6 +193,8 @@ class Weighted : public Partition { public: std::unique_ptr scale(const Shape& factors) const override; + std::unique_ptr bloat(const Shape& low_offsets, + const Shape& high_offsets) const override; public: Legion::LogicalPartition construct(Legion::LogicalRegion region, bool complete) const override; @@ -229,6 +240,8 @@ class Image : public Partition { public: std::unique_ptr scale(const Shape& factors) const override; + std::unique_ptr bloat(const Shape& low_offsets, + const Shape& high_offsets) const override; public: Legion::LogicalPartition construct(Legion::LogicalRegion region, bool complete) const override; @@ -257,6 +270,11 @@ std::unique_ptr create_tiling(Shape&& tile_shape, Shape&& color_shape, tuple&& offsets = {}); +std::unique_ptr create_tiling(Shape&& tile_shape, + Shape&& color_shape, + tuple&& offsets, + Shape&& strides); + std::unique_ptr create_weighted(const Legion::FutureMap& weights, const Domain& color_domain); From ecad494c607d85c8eda6db6e2f4b055394918a3b Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Tue, 12 Sep 2023 00:33:37 -0700 Subject: [PATCH 0216/1425] Fix for overlapped tiling (#66) * Integration test for bloating constraints * Scalar constructor for legate::Shape * Minor fix for overlapped tiling --- src/core/data/scalar.h | 14 +- src/core/data/scalar.inl | 8 ++ src/core/partitioning/partition.cc | 19 ++- src/core/partitioning/partition.h | 2 +- tests/cpp/integration/bloat_constraints.cc | 160 +++++++++++++++++++++ 5 files changed, 193 insertions(+), 10 deletions(-) create mode 100644 tests/cpp/integration/bloat_constraints.cc diff --git a/src/core/data/scalar.h b/src/core/data/scalar.h index 0132282638..3dbd67f3d5 100644 --- a/src/core/data/scalar.h +++ b/src/core/data/scalar.h @@ -14,6 +14,7 @@ #include "core/type/type_traits.h" #include "core/utilities/span.h" +#include "core/utilities/tuple.h" #include "core/utilities/typedefs.h" /** @@ -87,15 +88,22 @@ class Scalar { * @param string A string to create a `Scalar` with */ Scalar(const std::string& string); - /** - * @brief Creates an owned scalar from a tuple of scalars. The values in the input vector + * @brief Creates an owned scalar from a vector of scalars. The values in the input vector * will be copied. * - * @param values A vector that contains elements of a tuple + * @param values Values to create a scalar with in a vector */ template Scalar(const std::vector& values); + /** + * @brief Creates an owned scalar from a tuple of scalars. The values in the input tuple + * will be copied. + * + * @param values Values to create a scalar with in a tuple + */ + template + Scalar(const tuple& values); public: /** diff --git a/src/core/data/scalar.inl b/src/core/data/scalar.inl index a471d52ec1..175d962b19 100644 --- a/src/core/data/scalar.inl +++ b/src/core/data/scalar.inl @@ -81,6 +81,14 @@ Scalar::Scalar(const std::vector& values) { } +template +Scalar::Scalar(const tuple& values) + : impl_(create_impl(fixed_array_type(primitive_type(legate_type_code_of), values.size()), + values.data().data(), + true)) +{ +} + template Scalar::Scalar(const Point& point) : impl_(create_impl(point_type(DIM), &point, true)) { diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index 5ec99353a6..1c00a8f2d2 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -64,7 +64,7 @@ std::string NoPartition::to_string() const { return "NoPartition"; } Tiling::Tiling(Shape&& tile_shape, Shape&& color_shape, tuple&& offsets) : Partition(), - disjoint_(true), + overlapped_(false), tile_shape_(std::move(tile_shape)), color_shape_(std::move(color_shape)), offsets_(std::move(offsets)), @@ -77,12 +77,15 @@ Tiling::Tiling(Shape&& tile_shape, Shape&& color_shape, tuple&& offsets Tiling::Tiling(Shape&& tile_shape, Shape&& color_shape, tuple&& offsets, Shape&& strides) : Partition(), - disjoint_(!(strides < tile_shape)), + overlapped_(strides < tile_shape), tile_shape_(std::move(tile_shape)), color_shape_(std::move(color_shape)), offsets_(std::move(offsets)), strides_(std::move(strides)) { + if (!overlapped_) { + throw std::invalid_argument("This constructor must be called only for overlapped tiling"); + } if (offsets_.empty()) offsets_ = tuple(tile_shape_.size(), 0); assert(tile_shape_.size() == color_shape_.size()); assert(tile_shape_.size() == offsets_.size()); @@ -91,7 +94,7 @@ Tiling::Tiling(Shape&& tile_shape, Shape&& color_shape, tuple&& offsets bool Tiling::operator==(const Tiling& other) const { return tile_shape_ == other.tile_shape_ && color_shape_ == other.color_shape_ && - offsets_ == other.offsets_; + offsets_ == other.offsets_ && strides_ == other.strides_; } bool Tiling::operator<(const Tiling& other) const @@ -106,6 +109,10 @@ bool Tiling::operator<(const Tiling& other) const return false; if (offsets_ < other.offsets_) return true; + else if (other.offsets_ < offsets_) + return false; + if (strides_ < other.strides_) + return true; else return false; } @@ -124,7 +131,7 @@ bool Tiling::is_complete_for(const detail::Storage* storage) const for (uint32_t dim = 0; dim < ndim; ++dim) { int64_t my_lo = offsets_[dim]; - int64_t my_hi = my_lo + static_cast(tile_shape_[dim] * color_shape_[dim]); + int64_t my_hi = my_lo + static_cast(strides_[dim] * color_shape_[dim]); if (static_cast(storage_offs[dim]) < my_lo && my_hi < static_cast(storage_offs[dim] + storage_exts[dim])) return false; @@ -136,7 +143,7 @@ bool Tiling::is_disjoint_for(const Domain* launch_domain) const { // TODO: The check really should be that every two points from the launch domain are mapped // to two different colors - return disjoint_ && + return !overlapped_ && (nullptr == launch_domain || launch_domain->get_volume() <= color_shape_.volume()); } @@ -226,7 +233,7 @@ Shape Tiling::get_child_extents(const Shape& extents, const Shape& color) Shape Tiling::get_child_offsets(const Shape& color) { return apply([](size_t a, int64_t b) { return static_cast(static_cast(a) + b); }, - tile_shape_ * color, + strides_ * color, offsets_); } diff --git a/src/core/partitioning/partition.h b/src/core/partitioning/partition.h index d8c725c1ff..6d49fd4113 100644 --- a/src/core/partitioning/partition.h +++ b/src/core/partitioning/partition.h @@ -164,7 +164,7 @@ class Tiling : public Partition { Shape get_child_offsets(const Shape& color); private: - bool disjoint_; + bool overlapped_; Shape tile_shape_; Shape color_shape_; tuple offsets_; diff --git a/tests/cpp/integration/bloat_constraints.cc b/tests/cpp/integration/bloat_constraints.cc new file mode 100644 index 0000000000..2a4a0f387f --- /dev/null +++ b/tests/cpp/integration/bloat_constraints.cc @@ -0,0 +1,160 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include + +#include "legate.h" + +namespace bloat_constraints { + +static const char* library_name = "test_bloat_constraints"; + +static const int32_t TEST_MAX_DIM = 3; + +static legate::Logger logger(library_name); + +enum TaskIDs { + BLOAT_TESTER = 0, +}; + +template +struct BloatTester : public legate::LegateTask> { + static const int32_t TASK_ID = BLOAT_TESTER + DIM; + static void cpu_variant(legate::TaskContext context) + { + auto source = context.input(0); + auto bloated = context.input(1); + + auto extents = context.scalar(0).values(); + auto low_offsets = context.scalar(1).values(); + auto high_offsets = context.scalar(2).values(); + + auto source_shape = source.shape(); + auto bloated_shape = bloated.shape(); + + if (source_shape.empty()) return; + + for (int32_t idx = 0; idx < DIM; ++idx) { + auto low = + std::max(0, source_shape.lo[idx] - static_cast(low_offsets[idx])); + auto high = std::min(static_cast(extents[idx] - 1), + source_shape.hi[idx] + static_cast(high_offsets[idx])); + EXPECT_EQ(low, bloated_shape.lo[idx]); + EXPECT_EQ(high, bloated_shape.hi[idx]); + } + } +}; + +void prepare() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + BloatTester<1>::register_variants(context); + BloatTester<2>::register_variants(context); + BloatTester<3>::register_variants(context); +} + +struct BloatTestSpec { + legate::Shape extents; + legate::Shape low_offsets; + legate::Shape high_offsets; +}; + +void test_bloat(const BloatTestSpec& spec) +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + auto source = runtime->create_store(spec.extents, legate::int64()); + auto bloated = runtime->create_store(spec.extents, legate::int64()); + runtime->issue_fill(source, legate::Scalar(int64_t{0})); + runtime->issue_fill(bloated, legate::Scalar(int64_t{0})); + + auto task = runtime->create_task(context, BLOAT_TESTER + source.dim()); + auto part_source = task.add_input(source); + auto part_bloated = task.add_input(bloated); + task.add_constraint( + legate::bloat(part_source, part_bloated, spec.low_offsets, spec.high_offsets)); + task.add_scalar_arg(legate::Scalar(spec.extents.data())); + task.add_scalar_arg(legate::Scalar(spec.low_offsets.data())); + task.add_scalar_arg(legate::Scalar(spec.high_offsets.data())); + + runtime->submit(std::move(task)); +} + +void test_invalid() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + { + auto source = runtime->create_store({1, 2}, legate::float16()); + auto bloated = runtime->create_store({2, 3, 4}, legate::int64()); + + auto task = runtime->create_task(context, BLOAT_TESTER + 2); + auto part_source = task.add_output(source); + auto part_bloated = task.add_output(bloated); + task.add_constraint(legate::bloat(part_source, part_bloated, {2, 3}, {4, 5})); + + EXPECT_THROW(runtime->submit(std::move(task)), std::invalid_argument); + } + + { + auto source = runtime->create_store({1, 2}, legate::float16()); + auto bloated = runtime->create_store({2, 3}, legate::int64()); + + auto task = runtime->create_task(context, BLOAT_TESTER + 2); + auto part_source = task.add_output(source); + auto part_bloated = task.add_output(bloated); + task.add_constraint(legate::bloat(part_source, part_bloated, {2, 3, 3}, {4, 5})); + + EXPECT_THROW(runtime->submit(std::move(task)), std::invalid_argument); + } + + { + auto source = runtime->create_store({1, 2}, legate::float16()); + auto bloated = runtime->create_store({2, 3}, legate::int64()); + + auto task = runtime->create_task(context, BLOAT_TESTER + 2); + auto part_source = task.add_output(source); + auto part_bloated = task.add_output(bloated); + task.add_constraint(legate::bloat(part_source, part_bloated, {2, 3}, {4, 5, 3})); + + EXPECT_THROW(runtime->submit(std::move(task)), std::invalid_argument); + } +} + +TEST(BloatConstraint, 1D) +{ + legate::Core::perform_registration(); + test_bloat({{10}, {2}, {4}}); +} + +TEST(BloatConstraint, 2D) +{ + legate::Core::perform_registration(); + test_bloat({{9, 9}, {2, 3}, {3, 4}}); +} + +TEST(BloatConstraint, 3D) +{ + legate::Core::perform_registration(); + test_bloat({{10, 10, 10}, {2, 3, 4}, {4, 3, 2}}); +} + +TEST(BloatConstraint, Invalid) +{ + legate::Core::perform_registration(); + test_invalid(); +} + +} // namespace bloat_constraints From 96979aad25a0e377579e4f6d099fde44654378f4 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Tue, 12 Sep 2023 14:27:38 -0700 Subject: [PATCH 0217/1425] Should skip unification for variables in the same class (#67) --- .../partitioning/detail/constraint_solver.cc | 2 ++ .../cpp/integration/alignment_constraints.cc | 32 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/src/core/partitioning/detail/constraint_solver.cc b/src/core/partitioning/detail/constraint_solver.cc index 844f1d6daa..2d0a4f58ee 100644 --- a/src/core/partitioning/detail/constraint_solver.cc +++ b/src/core/partitioning/detail/constraint_solver.cc @@ -36,6 +36,8 @@ struct UnionFindEntry { UnionFindEntry* unify(UnionFindEntry* other) { + if (this == other) return this; + UnionFindEntry* self = this; if (self->size < other->size) std::swap(self, other); diff --git a/tests/cpp/integration/alignment_constraints.cc b/tests/cpp/integration/alignment_constraints.cc index a5e2728af6..eeac81fe8e 100644 --- a/tests/cpp/integration/alignment_constraints.cc +++ b/tests/cpp/integration/alignment_constraints.cc @@ -184,6 +184,32 @@ void test_alignment_transformed() launch_tester(store1.delinearize(0, {10, 10}), store4); } +void test_redundant_alignment() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + const std::vector extents = {10}; + auto store1 = runtime->create_store(extents, legate::int64()); + auto store2 = runtime->create_store(extents, legate::int64()); + auto store3 = runtime->create_store(extents, legate::int64()); + + auto task = runtime->create_task(context, ALIGNMENT_TESTER + extents.size()); + auto part1 = task.add_output(store1); + auto part2 = task.add_output(store2); + auto part3 = task.add_output(store3); + + task.add_constraint(legate::align(part1, part2)); + task.add_constraint(legate::align(part2, part3)); + + // Redundant alignments + task.add_constraint(legate::align(part1, part2)); + task.add_constraint(legate::align(part1, part3)); + task.add_constraint(legate::align(part2, part1)); + + runtime->submit(std::move(task)); +} + void test_invalid_alignment() { auto runtime = legate::Runtime::get_runtime(); @@ -217,6 +243,12 @@ TEST(Alignment, WithTransform) test_alignment_transformed(); } +TEST(Alignment, Redundant) +{ + legate::Core::perform_registration(); + test_redundant_alignment(); +} + TEST(Alignment, Invalid) { legate::Core::perform_registration(); From 3f138644e1ba8c8c3109a20f5a6a9feb5a69563c Mon Sep 17 00:00:00 2001 From: Jacob Faibussowitsch Date: Wed, 13 Sep 2023 15:07:32 -0400 Subject: [PATCH 0218/1425] Feature: `LegateDefined()` (#33) * Add LegateDefined() * src/legate_defines.h: "rename" DEBUG_LEGATE to LEGATE_USE_DEBUG (and define to 1) * Normalize DEBUG_LEGATE macro * Fix -Wformat-pedantic errors: * Add pre-commit hook --- .pre-commit-config.yaml | 10 + legate_core_cpp.cmake | 4 +- scripts/pre-commit/legate_defined.sh | 43 +++ src/core/comm/coll.cc | 20 +- src/core/comm/coll.h | 10 +- src/core/comm/comm.cc | 14 +- src/core/comm/comm_cpu.cc | 10 +- src/core/comm/local_comm.cc | 83 +++--- src/core/comm/mpi_comm.cc | 157 +++++----- src/core/cuda/cuda_help.h | 20 +- src/core/cuda/stream_pool.cu | 10 +- src/core/data/detail/array.cc | 14 +- src/core/data/detail/array_tasks.h | 12 +- src/core/data/detail/logical_array.cc | 6 +- src/core/data/detail/logical_region_field.cc | 8 +- src/core/data/detail/logical_store.cc | 72 ++--- src/core/data/detail/store.cc | 59 ++-- src/core/data/detail/transform.cc | 16 +- src/core/mapping/detail/array.cc | 6 +- src/core/mapping/detail/base_mapper.cc | 278 ++++++++---------- src/core/mapping/detail/core_mapper.cc | 11 +- src/core/mapping/detail/instance_manager.cc | 71 +++-- src/core/mapping/detail/machine.cc | 4 +- src/core/mapping/detail/mapping.cc | 26 +- src/core/mapping/detail/operation.cc | 12 +- src/core/mapping/detail/store.cc | 8 +- src/core/operation/detail/req_analyzer.cc | 20 +- src/core/operation/detail/task.cc | 12 +- src/core/operation/detail/task_launcher.cc | 4 +- .../partitioning/detail/constraint_solver.cc | 8 +- src/core/partitioning/detail/partitioner.cc | 50 ++-- src/core/partitioning/partition.cc | 8 +- src/core/runtime/detail/library.cc | 8 +- src/core/runtime/detail/machine_manager.cc | 4 +- src/core/runtime/detail/partition_manager.cc | 8 +- src/core/runtime/detail/projection.cc | 17 +- src/core/runtime/detail/provenance_manager.cc | 16 +- src/core/runtime/detail/runtime.cc | 94 +++--- src/core/runtime/detail/shard.cc | 6 +- src/core/runtime/library.inl | 9 +- src/core/runtime/runtime.cc | 49 +-- src/core/task/detail/return.cc | 32 +- src/core/task/detail/task_context.cc | 26 +- src/core/task/task.cc | 14 +- src/core/type/detail/type_info.cc | 52 ++-- src/core/utilities/debug.h | 47 ++- src/core/utilities/deserializer.cc | 16 +- src/core/utilities/nvtx_help.h | 10 +- src/env_defaults.h | 2 +- src/legate.mk | 4 +- src/legate_defines.h | 43 ++- tests/cpp/integration/machine_scope.cc | 4 +- tests/integration/scoping/src/library.cc | 4 +- 53 files changed, 745 insertions(+), 806 deletions(-) create mode 100755 scripts/pre-commit/legate_defined.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d4c48c6c5e..8f18375713 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,6 +24,16 @@ repos: pass_filenames: false args: ['legate', 'tests'] additional_dependencies: [numpy,pytest,pytest_mock] + - repo: local + hooks: + - id: legate-defined + name: legate-defined + description: 'Find uses of ifdef LEGATE_ that should be using LegateDefined()' + entry: ./scripts/pre-commit/legate_defined.sh + language: script + 'types_or': [c++, c, cuda] + require_serial: false + stages: [pre-commit] ci: autoupdate_schedule: quarterly diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index 007c303729..b737e3e443 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -133,8 +133,8 @@ if (legate_core_COLLECTIVE) endif() if(CMAKE_BUILD_TYPE STREQUAL "Debug") - list(APPEND legate_core_CXX_DEFS DEBUG_LEGATE) - list(APPEND legate_core_CUDA_DEFS DEBUG_LEGATE) + list(APPEND legate_core_CXX_DEFS LEGATE_USE_DEBUG) + list(APPEND legate_core_CUDA_DEFS LEGATE_USE_DEBUG) endif() if(Legion_USE_CUDA) diff --git a/scripts/pre-commit/legate_defined.sh b/scripts/pre-commit/legate_defined.sh new file mode 100755 index 0000000000..e31c006250 --- /dev/null +++ b/scripts/pre-commit/legate_defined.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +output=$( + grep -E \ + -n \ + -H \ + -C 1 \ + --color=always \ + -e '#\s*if[n]?def\s+LEGATE_\w+' \ + -e '#(\s*if\s+)?[!]?defined\s*\(\s*LEGATE_\w+' \ + -e '#.*defined\s*\(\s*LEGATE_\w+' \ + -e '#\s*elif\s+LEGATE_\w+' \ + -- \ + "$@" + ) +rc=$? +if [[ ${rc} -eq 1 ]]; then + # no matches found, that's a good thing + exit 0 +elif [[ ${rc} -eq 0 ]]; then + echo "x ===------------------------------------------------------------------=== x" + echo "${output}" + echo "" + echo "Instances of preprocessor ifdef/ifndef/if defined found, use" + echo "LegateDefined() instead:" + echo "" + echo "- #ifdef LEGATE_USE_FOO" + echo "- #include \"foo.h\"" + echo "- #endif" + echo "+ #if LegateDefined(LEGATE_USE_FOO)" + echo "+ #include \"foo.h\"" + echo "+ #endif" + echo "" + echo "- #ifdef LEGATE_USE_FOO" + echo "- x = 2;" + echo "- #endif" + echo "+ if (LegateDefined(LEGATE_USE_FOO)) {" + echo "+ x = 2;" + echo "+ }" + echo "x ===------------------------------------------------------------------=== x" + exit 1 +else + exit ${rc} +fi diff --git a/src/core/comm/coll.cc b/src/core/comm/coll.cc index aa23ba82e2..ab843dca15 100644 --- a/src/core/comm/coll.cc +++ b/src/core/comm/coll.cc @@ -109,18 +109,20 @@ int collAllgather( // called from main thread int collInit(int argc, char* argv[]) { -#ifdef LEGATE_USE_NETWORK - char* network = getenv("LEGATE_NEED_NETWORK"); - int need_network = 0; - if (network != nullptr) { need_network = atoi(network); } - if (need_network) { - backend_network = new MPINetwork(argc, argv); + if (LegateDefined(LEGATE_USE_NETWORK)) { + char* network = getenv("LEGATE_NEED_NETWORK"); + int need_network = 0; + if (network != nullptr) { need_network = atoi(network); } + if (need_network) { +#if LegateDefined(LEGATE_USE_NETWORK) + backend_network = new MPINetwork(argc, argv); +#endif + } else { + backend_network = new LocalNetwork(argc, argv); + } } else { backend_network = new LocalNetwork(argc, argv); } -#else - backend_network = new LocalNetwork(argc, argv); -#endif return CollSuccess; } diff --git a/src/core/comm/coll.h b/src/core/comm/coll.h index a917ff8974..af3efe5264 100644 --- a/src/core/comm/coll.h +++ b/src/core/comm/coll.h @@ -16,7 +16,9 @@ #include #include -#ifdef LEGATE_USE_NETWORK +#include "legate_defines.h" + +#if LegateDefined(LEGATE_USE_NETWORK) #include #endif @@ -33,7 +35,7 @@ namespace legate::comm::coll { -#ifdef LEGATE_USE_NETWORK +#if LegateDefined(LEGATE_USE_NETWORK) struct RankMappingTable { int* mpi_rank; int* global_rank; @@ -70,7 +72,7 @@ enum CollCommType : int { }; struct Coll_Comm { -#ifdef LEGATE_USE_NETWORK +#if LegateDefined(LEGATE_USE_NETWORK) MPI_Comm mpi_comm; RankMappingTable mapping_table; #endif @@ -129,7 +131,7 @@ class BackendNetwork { int current_unique_id; }; -#ifdef LEGATE_USE_NETWORK +#if LegateDefined(LEGATE_USE_NETWORK) class MPINetwork : public BackendNetwork { public: MPINetwork(int argc, char* argv[]); diff --git a/src/core/comm/comm.cc b/src/core/comm/comm.cc index 0d59b2c9fd..198e300b41 100644 --- a/src/core/comm/comm.cc +++ b/src/core/comm/comm.cc @@ -10,11 +10,11 @@ * its affiliates is strictly prohibited. */ +#include "legate_defines.h" +// #include "core/comm/comm.h" -#ifdef LEGATE_USE_CUDA -#include "core/comm/comm_nccl.h" -#endif #include "core/comm/comm_cpu.h" +#include "core/comm/comm_nccl.h" #include "core/runtime/runtime.h" #include "env_defaults.h" @@ -22,9 +22,7 @@ namespace legate::comm { void register_tasks(Legion::Runtime* runtime, const detail::Library* library) { -#ifdef LEGATE_USE_CUDA - nccl::register_tasks(runtime, library); -#endif + if (LegateDefined(LEGATE_USE_CUDA)) { nccl::register_tasks(runtime, library); } bool disable_mpi = static_cast(extract_env("LEGATE_DISABLE_MPI", DISABLE_MPI_DEFAULT, DISABLE_MPI_TEST)); if (!disable_mpi) { cpu::register_tasks(runtime, library); } @@ -32,9 +30,7 @@ void register_tasks(Legion::Runtime* runtime, const detail::Library* library) void register_builtin_communicator_factories(const detail::Library* library) { -#ifdef LEGATE_USE_CUDA - nccl::register_factory(library); -#endif + if (LegateDefined(LEGATE_USE_CUDA)) { nccl::register_factory(library); } cpu::register_factory(library); } diff --git a/src/core/comm/comm_cpu.cc b/src/core/comm/comm_cpu.cc index 7c0ac3cc06..60fb60d394 100644 --- a/src/core/comm/comm_cpu.cc +++ b/src/core/comm/comm_cpu.cc @@ -94,7 +94,7 @@ static int init_cpucoll_mapping(const Legion::Task* task, { Core::show_progress(task, context, runtime); int mpi_rank = 0; -#if defined(LEGATE_USE_NETWORK) +#if LegateDefined(LEGATE_USE_NETWORK) if (coll::backend_network->comm_type == coll::CollCommType::CollMPI) { MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); } @@ -118,8 +118,8 @@ static coll::CollComm init_cpucoll(const Legion::Task* task, coll::CollComm comm = (coll::CollComm)malloc(sizeof(coll::Coll_Comm)); -#ifdef LEGATE_USE_NETWORK - if (coll::backend_network->comm_type == coll::CollCommType::CollMPI) { + if (LegateDefined(LEGATE_USE_NETWORK) && + (coll::backend_network->comm_type == coll::CollCommType::CollMPI)) { int* mapping_table = (int*)malloc(sizeof(int) * num_ranks); for (int i = 0; i < num_ranks; i++) { const int mapping_table_element = task->futures[i + 1].get_result(); @@ -128,9 +128,7 @@ static coll::CollComm init_cpucoll(const Legion::Task* task, coll::collCommCreate(comm, num_ranks, point, unique_id, mapping_table); assert(mapping_table[point] == comm->mpi_rank); free(mapping_table); - } else -#endif - { + } else { coll::collCommCreate(comm, num_ranks, point, unique_id, nullptr); } diff --git a/src/core/comm/local_comm.cc b/src/core/comm/local_comm.cc index 1c6b273cf4..99af6092d9 100644 --- a/src/core/comm/local_comm.cc +++ b/src/core/comm/local_comm.cc @@ -154,23 +154,22 @@ int LocalNetwork::alltoallv(const void* sendbuf, static_cast(displs[recvfrom_seg_id]) * type_extent; char* dst = static_cast(recvbuf) + static_cast(rdispls[recvfrom_global_rank]) * type_extent; -#ifdef DEBUG_LEGATE - log_coll.debug( - "AlltoallvLocal i: %d === global_rank %d, dtype %d, copy rank %d (seg %d, sdispls %d, %p) to " - "rank %d (seg " - "%d, rdispls %d, %p)", - i, - global_rank, - type_extent, - recvfrom_global_rank, - recvfrom_seg_id, - sdispls[recvfrom_seg_id], - static_cast(src), - global_rank, - recvfrom_global_rank, - rdispls[recvfrom_global_rank], - static_cast(dst)); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + log_coll.debug( + "AlltoallvLocal i: %d === global_rank %d, dtype %d, copy rank %d (seg %d, sdispls %d, %p) " + "to rank %d (seg %d, rdispls %d, %p)", + i, + global_rank, + type_extent, + recvfrom_global_rank, + recvfrom_seg_id, + sdispls[recvfrom_seg_id], + static_cast(src), + global_rank, + recvfrom_global_rank, + rdispls[recvfrom_global_rank], + static_cast(dst)); + } memcpy(dst, src, recvcounts[recvfrom_global_rank] * type_extent); } @@ -208,20 +207,20 @@ int LocalNetwork::alltoall( static_cast(recvfrom_seg_id) * type_extent * count; char* dst = static_cast(recvbuf) + static_cast(recvfrom_global_rank) * type_extent * count; -#ifdef DEBUG_LEGATE - log_coll.debug( - "AlltoallLocal i: %d === global_rank %d, dtype %d, copy rank %d (seg %d, %p) to rank %d (seg " - "%d, %p)", - i, - global_rank, - type_extent, - recvfrom_global_rank, - recvfrom_seg_id, - static_cast(src), - global_rank, - recvfrom_global_rank, - static_cast(dst)); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + log_coll.debug( + "AlltoallLocal i: %d === global_rank %d, dtype %d, copy rank %d (seg %d, %p) to rank %d " + "(seg %d, %p)", + i, + global_rank, + type_extent, + recvfrom_global_rank, + recvfrom_seg_id, + static_cast(src), + global_rank, + recvfrom_global_rank, + static_cast(dst)); + } memcpy(dst, src, count * type_extent); } @@ -258,17 +257,17 @@ int LocalNetwork::allgather( const void* src = global_comm->local_comm->buffers[recvfrom_global_rank]; char* dst = static_cast(recvbuf) + static_cast(recvfrom_global_rank) * type_extent * count; -#ifdef DEBUG_LEGATE - log_coll.debug( - "AllgatherLocal i: %d === global_rank %d, dtype %d, copy rank %d (%p) to rank %d (%p)", - recvfrom_global_rank, - global_rank, - type_extent, - recvfrom_global_rank, - src, - global_rank, - static_cast(dst)); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + log_coll.debug( + "AllgatherLocal i: %d === global_rank %d, dtype %d, copy rank %d (%p) to rank %d (%p)", + recvfrom_global_rank, + global_rank, + type_extent, + recvfrom_global_rank, + src, + global_rank, + static_cast(dst)); + } memcpy(dst, src, count * type_extent); } diff --git a/src/core/comm/mpi_comm.cc b/src/core/comm/mpi_comm.cc index 400963203a..ada56025db 100644 --- a/src/core/comm/mpi_comm.cc +++ b/src/core/comm/mpi_comm.cc @@ -34,11 +34,11 @@ inline void check_mpi(int error, const char* file, int line) if (error != MPI_SUCCESS) { fprintf( stderr, "Internal MPI failure with error code %d in file %s at line %d\n", error, file, line); -#ifdef DEBUG_LEGATE - assert(false); -#else - exit(error); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + assert(false); + } else { + exit(error); + } } } @@ -103,13 +103,13 @@ int MPINetwork::init_comm() { int id = 0; collGetUniqueId(&id); -#ifdef DEBUG_LEGATE - int mpi_rank; - int send_id = id; - // check if all ranks get the same unique id - CHECK_MPI(MPI_Bcast(&send_id, 1, MPI_INT, 0, MPI_COMM_WORLD)); - assert(send_id == id); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + int mpi_rank; + int send_id = id; + // check if all ranks get the same unique id + CHECK_MPI(MPI_Bcast(&send_id, 1, MPI_INT, 0, MPI_COMM_WORLD)); + assert(send_id == id); + } assert(mpi_comms.size() == id); // create mpi comm MPI_Comm mpi_comm; @@ -202,21 +202,21 @@ int MPINetwork::alltoallv(const void* sendbuf, // tag: seg idx + rank_idx + tag int send_tag = generateAlltoallvTag(sendto_global_rank, global_rank, global_comm); int recv_tag = generateAlltoallvTag(global_rank, recvfrom_global_rank, global_comm); -#ifdef DEBUG_LEGATE - log_coll.debug( - "AlltoallvMPI i: %d === global_rank %d, mpi rank %d, send to %d (%d), send_tag %d, " - "recv from %d (%d), " - "recv_tag %d", - i, - global_rank, - global_comm->mpi_rank, - sendto_global_rank, - sendto_mpi_rank, - send_tag, - recvfrom_global_rank, - recvfrom_mpi_rank, - recv_tag); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + log_coll.debug( + "AlltoallvMPI i: %d === global_rank %d, mpi rank %d, send to %d (%d), send_tag %d, " + "recv from %d (%d), " + "recv_tag %d", + i, + global_rank, + global_comm->mpi_rank, + sendto_global_rank, + sendto_mpi_rank, + send_tag, + recvfrom_global_rank, + recvfrom_mpi_rank, + recv_tag); + } CHECK_MPI(MPI_Sendrecv(src, scount, mpi_type, @@ -262,21 +262,21 @@ int MPINetwork::alltoall( // tag: seg idx + rank_idx + tag int send_tag = generateAlltoallTag(sendto_global_rank, global_rank, global_comm); int recv_tag = generateAlltoallTag(global_rank, recvfrom_global_rank, global_comm); -#ifdef DEBUG_LEGATE - log_coll.debug( - "AlltoallMPI i: %d === global_rank %d, mpi rank %d, send to %d (%d), send_tag %d, " - "recv from %d (%d), " - "recv_tag %d", - i, - global_rank, - global_comm->mpi_rank, - sendto_global_rank, - sendto_mpi_rank, - send_tag, - recvfrom_global_rank, - recvfrom_mpi_rank, - recv_tag); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + log_coll.debug( + "AlltoallMPI i: %d === global_rank %d, mpi rank %d, send to %d (%d), send_tag %d, " + "recv from %d (%d), " + "recv_tag %d", + i, + global_rank, + global_comm->mpi_rank, + sendto_global_rank, + sendto_mpi_rank, + send_tag, + recvfrom_global_rank, + recvfrom_mpi_rank, + recv_tag); + } CHECK_MPI(MPI_Sendrecv(src, count, mpi_type, @@ -340,14 +340,15 @@ int MPINetwork::gather( // non-root if (global_rank != root) { tag = generateGatherTag(global_rank, global_comm); -#ifdef DEBUG_LEGATE - log_coll.debug("GatherMPI: non-root send global_rank %d, mpi rank %d, send to %d (%d), tag %d", - global_rank, - global_comm->mpi_rank, - root, - root_mpi_rank, - tag); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + log_coll.debug( + "GatherMPI: non-root send global_rank %d, mpi rank %d, send to %d (%d), tag %d", + global_rank, + global_comm->mpi_rank, + root, + root_mpi_rank, + tag); + } CHECK_MPI(MPI_Send(sendbuf, count, mpi_type, root_mpi_rank, tag, global_comm->mpi_comm)); return CollSuccess; } @@ -362,17 +363,17 @@ int MPINetwork::gather( recvfrom_mpi_rank = global_comm->mapping_table.mpi_rank[i]; assert(i == global_comm->mapping_table.global_rank[i]); tag = generateGatherTag(i, global_comm); -#ifdef DEBUG_LEGATE - log_coll.debug( - "GatherMPI: root i %d === global_rank %d, mpi rank %d, recv %p, from %d (%d), tag %d", - i, - global_rank, - global_comm->mpi_rank, - dst, - i, - recvfrom_mpi_rank, - tag); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + log_coll.debug( + "GatherMPI: root i %d === global_rank %d, mpi rank %d, recv %p, from %d (%d), tag %d", + i, + global_rank, + global_comm->mpi_rank, + dst, + i, + recvfrom_mpi_rank, + tag); + } assert(dst != nullptr); if (global_rank == i) { memcpy(dst, sendbuf, incr); @@ -402,14 +403,14 @@ int MPINetwork::bcast(void* buf, int count, CollDataType type, int root, CollCom // non-root if (global_rank != root) { tag = generateBcastTag(global_rank, global_comm); -#ifdef DEBUG_LEGATE - log_coll.debug("BcastMPI: non-root recv global_rank %d, mpi rank %d, send to %d (%d), tag %d", - global_rank, - global_comm->mpi_rank, - root, - root_mpi_rank, - tag); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + log_coll.debug("BcastMPI: non-root recv global_rank %d, mpi rank %d, send to %d (%d), tag %d", + global_rank, + global_comm->mpi_rank, + root, + root_mpi_rank, + tag); + } CHECK_MPI(MPI_Recv(buf, count, mpi_type, root_mpi_rank, tag, global_comm->mpi_comm, &status)); return CollSuccess; } @@ -420,15 +421,15 @@ int MPINetwork::bcast(void* buf, int count, CollDataType type, int root, CollCom sendto_mpi_rank = global_comm->mapping_table.mpi_rank[i]; assert(i == global_comm->mapping_table.global_rank[i]); tag = generateBcastTag(i, global_comm); -#ifdef DEBUG_LEGATE - log_coll.debug("BcastMPI: root i %d === global_rank %d, mpi rank %d, send to %d (%d), tag %d", - i, - global_rank, - global_comm->mpi_rank, - i, - sendto_mpi_rank, - tag); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + log_coll.debug("BcastMPI: root i %d === global_rank %d, mpi rank %d, send to %d (%d), tag %d", + i, + global_rank, + global_comm->mpi_rank, + i, + sendto_mpi_rank, + tag); + } if (global_rank != i) { CHECK_MPI(MPI_Send(buf, count, mpi_type, sendto_mpi_rank, tag, global_comm->mpi_comm)); } diff --git a/src/core/cuda/cuda_help.h b/src/core/cuda/cuda_help.h index fe81370871..d5af1df9a7 100644 --- a/src/core/cuda/cuda_help.h +++ b/src/core/cuda/cuda_help.h @@ -16,6 +16,10 @@ #include #include +#include "legion.h" + +#include "legate_defines.h" + #define THREADS_PER_BLOCK 128 #define CHECK_CUDA(expr) \ do { \ @@ -23,18 +27,14 @@ legate::cuda::check_cuda(__result__, __FILE__, __LINE__); \ } while (false) -#ifdef DEBUG_LEGATE - +#if LegateDefined(LEGATE_USE_DEBUG) #define CHECK_CUDA_STREAM(stream) \ do { \ CHECK_CUDA(cudaStreamSynchronize(stream)); \ CHECK_CUDA(cudaPeekAtLastError()); \ } while (false) - #else - #define CHECK_CUDA_STREAM(stream) - #endif namespace legate::cuda { @@ -48,11 +48,11 @@ __host__ inline void check_cuda(cudaError_t error, const char* file, int line) cudaGetErrorName(error), file, line); -#ifdef DEBUG_LEGATE - assert(false); -#else - exit(error); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + assert(false); + } else { + exit(error); + } } } diff --git a/src/core/cuda/stream_pool.cu b/src/core/cuda/stream_pool.cu index 5a1c920e76..5bde41d062 100644 --- a/src/core/cuda/stream_pool.cu +++ b/src/core/cuda/stream_pool.cu @@ -20,11 +20,11 @@ namespace legate::cuda { StreamView::~StreamView() { if (valid_ && Core::synchronize_stream_view) { -#ifdef DEBUG_LEGATE - CHECK_CUDA_STREAM(stream_); -#else - CHECK_CUDA(cudaStreamSynchronize(stream_)); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + CHECK_CUDA_STREAM(stream_); + } else { + CHECK_CUDA(cudaStreamSynchronize(stream_)); + } } } diff --git a/src/core/data/detail/array.cc b/src/core/data/detail/array.cc index 37e73e35a7..cd91d25cd0 100644 --- a/src/core/data/detail/array.cc +++ b/src/core/data/detail/array.cc @@ -27,9 +27,9 @@ BaseArray::BaseArray(std::shared_ptr data, std::shared_ptr null_ma bool BaseArray::unbound() const { -#ifdef DEBUG_LEGATE - assert(!nullable() || data_->is_unbound_store() == null_mask_->is_unbound_store()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + assert(!nullable() || data_->is_unbound_store() == null_mask_->is_unbound_store()); + } return data_->is_unbound_store(); } @@ -75,9 +75,7 @@ bool ListArray::unbound() const { return descriptor_->unbound() || vardata_->unb bool ListArray::valid() const { -#ifdef DEBUG_LEGATE - assert(descriptor_->valid() == vardata_->valid()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(descriptor_->valid() == vardata_->valid()); } return descriptor_->valid(); } @@ -125,9 +123,7 @@ bool StructArray::valid() const { auto result = std::all_of(fields_.begin(), fields_.end(), [](auto& field) { return field->valid(); }); -#ifdef DEBUG_LEGATE - assert(null_mask_->valid() == result); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(null_mask_->valid() == result); } return result; } diff --git a/src/core/data/detail/array_tasks.h b/src/core/data/detail/array_tasks.h index ac0ed45db3..0d232e196f 100644 --- a/src/core/data/detail/array_tasks.h +++ b/src/core/data/detail/array_tasks.h @@ -18,30 +18,30 @@ class Library; struct FixupRanges : public LegateTask { static void cpu_variant(legate::TaskContext context); -#ifdef LEGATE_USE_OPENMP +#if LegateDefined(LEGATE_USE_OPENMP) static void omp_variant(legate::TaskContext context); #endif -#ifdef LEGATE_USE_CUDA +#if LegateDefined(LEGATE_USE_CUDA) static void gpu_variant(legate::TaskContext context); #endif }; struct OffsetsToRanges : public LegateTask { static void cpu_variant(legate::TaskContext context); -#ifdef LEGATE_USE_OPENMP +#if LegateDefined(LEGATE_USE_OPENMP) static void omp_variant(legate::TaskContext context); #endif -#ifdef LEGATE_USE_CUDA +#if LegateDefined(LEGATE_USE_CUDA) static void gpu_variant(legate::TaskContext context); #endif }; struct RangesToOffsets : public LegateTask { static void cpu_variant(legate::TaskContext context); -#ifdef LEGATE_USE_OPENMP +#if LegateDefined(LEGATE_USE_OPENMP) static void omp_variant(legate::TaskContext context); #endif -#ifdef LEGATE_USE_CUDA +#if LegateDefined(LEGATE_USE_CUDA) static void gpu_variant(legate::TaskContext context); #endif }; diff --git a/src/core/data/detail/logical_array.cc b/src/core/data/detail/logical_array.cc index 48566984c7..c84dd35d5d 100644 --- a/src/core/data/detail/logical_array.cc +++ b/src/core/data/detail/logical_array.cc @@ -34,9 +34,9 @@ BaseLogicalArray::BaseLogicalArray(std::shared_ptr data, bool BaseLogicalArray::unbound() const { -#ifdef DEBUG_LEGATE - assert(!nullable() || data_->unbound() == null_mask_->unbound()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + assert(!nullable() || data_->unbound() == null_mask_->unbound()); + } return data_->unbound(); } diff --git a/src/core/data/detail/logical_region_field.cc b/src/core/data/detail/logical_region_field.cc index 70f3c9f694..0e2e3be643 100644 --- a/src/core/data/detail/logical_region_field.cc +++ b/src/core/data/detail/logical_region_field.cc @@ -85,10 +85,10 @@ void LogicalRegionField::add_invalidation_callback(std::function callbac void LogicalRegionField::perform_invalidation_callbacks() { if (parent_ != nullptr) { -#ifdef DEBUG_LEGATE - // Callbacks should exist only in the root - assert(callbacks_.empty()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + // Callbacks should exist only in the root + assert(callbacks_.empty()); + } parent_->perform_invalidation_callbacks(); } else { for (auto& callback : callbacks_) { callback(); } diff --git a/src/core/data/detail/logical_store.cc b/src/core/data/detail/logical_store.cc index 66b7bd32cb..97581e09da 100644 --- a/src/core/data/detail/logical_store.cc +++ b/src/core/data/detail/logical_store.cc @@ -42,9 +42,7 @@ Storage::Storage(int32_t dim, std::shared_ptr type) type_(std::move(type)), offsets_(dim_, 0) { -#ifdef DEBUG_LEGATE - log_legate.debug() << "Create " << to_string(); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { log_legate.debug() << "Create " << to_string(); } } Storage::Storage(const Shape& extents, std::shared_ptr type, bool optimize_scalar) @@ -56,9 +54,7 @@ Storage::Storage(const Shape& extents, std::shared_ptr type, bool optimize offsets_(dim_, 0) { if (optimize_scalar && volume_ == 1) kind_ = Kind::FUTURE; -#ifdef DEBUG_LEGATE - log_legate.debug() << "Create " << to_string(); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { log_legate.debug() << "Create " << to_string(); } } Storage::Storage(const Shape& extents, std::shared_ptr type, const Legion::Future& future) @@ -71,9 +67,7 @@ Storage::Storage(const Shape& extents, std::shared_ptr type, const Legion: future_(future), offsets_(dim_, 0) { -#ifdef DEBUG_LEGATE - log_legate.debug() << "Create " << to_string(); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { log_legate.debug() << "Create " << to_string(); } } Storage::Storage(Shape&& extents, @@ -91,9 +85,7 @@ Storage::Storage(Shape&& extents, color_(std::move(color)), offsets_(std::move(offsets)) { -#ifdef DEBUG_LEGATE - log_legate.debug() << "Create " << to_string(); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { log_legate.debug() << "Create " << to_string(); } } const Shape& Storage::extents() const @@ -107,9 +99,7 @@ const Shape& Storage::extents() const const Shape& Storage::offsets() const { -#ifdef DEBUG_LEGATE - assert(!unbound_); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(!unbound_); } return offsets_; } @@ -156,9 +146,7 @@ std::shared_ptr Storage::get_root() LogicalRegionField* Storage::get_region_field() { -#ifdef DEBUG_LEGATE - assert(Kind::REGION_FIELD == kind_); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(Kind::REGION_FIELD == kind_); } if (region_field_ != nullptr) return region_field_.get(); if (nullptr == parent_) { @@ -172,9 +160,7 @@ LogicalRegionField* Storage::get_region_field() Legion::Future Storage::get_future() const { -#ifdef DEBUG_LEGATE - assert(Kind::FUTURE == kind_); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(Kind::FUTURE == kind_); } return future_; } @@ -200,9 +186,7 @@ void Storage::set_future(Legion::Future future) { future_ = future; } RegionField Storage::map() { -#ifdef DEBUG_LEGATE - assert(Kind::REGION_FIELD == kind_); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(Kind::REGION_FIELD == kind_); } return Runtime::get_runtime()->map_region_field(get_region_field()); } @@ -337,11 +321,11 @@ LogicalStore::LogicalStore(std::shared_ptr&& storage) transform_(std::make_shared()) { if (!unbound()) extents_ = storage_->extents(); -#ifdef DEBUG_LEGATE - assert(transform_ != nullptr); + if (LegateDefined(LEGATE_USE_DEBUG)) { + assert(transform_ != nullptr); - log_legate.debug() << "Create " << to_string(); -#endif + log_legate.debug() << "Create " << to_string(); + } } LogicalStore::LogicalStore(Shape&& extents, @@ -352,11 +336,11 @@ LogicalStore::LogicalStore(Shape&& extents, storage_(storage), transform_(std::move(transform)) { -#ifdef DEBUG_LEGATE - assert(transform_ != nullptr); + if (LegateDefined(LEGATE_USE_DEBUG)) { + assert(transform_ != nullptr); - log_legate.debug() << "Create " << to_string(); -#endif + log_legate.debug() << "Create " << to_string(); + } } LogicalStore::~LogicalStore() @@ -400,18 +384,14 @@ Legion::Future LogicalStore::get_future() { return storage_->get_future(); } void LogicalStore::set_region_field(std::shared_ptr&& region_field) { -#ifdef DEBUG_LEGATE - assert(!has_scalar_storage()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(!has_scalar_storage()); } storage_->set_region_field(std::move(region_field)); extents_ = storage_->extents(); } void LogicalStore::set_future(Legion::Future future) { -#ifdef DEBUG_LEGATE - assert(has_scalar_storage()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(has_scalar_storage()); } storage_->set_future(future); } @@ -571,9 +551,7 @@ std::shared_ptr LogicalStore::get_physical_store() return std::make_shared(dim(), type(), -1, future, transform_); } -#ifdef DEBUG_LEGATE - assert(storage_->kind() == Storage::Kind::REGION_FIELD); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(storage_->kind() == Storage::Kind::REGION_FIELD); } auto region_field = storage_->map(); mapped_ = std::make_shared(dim(), type(), -1, std::move(region_field), transform_); return mapped_; @@ -606,9 +584,7 @@ Legion::ProjectionID LogicalStore::compute_projection( auto point = transform_->invert(proj::create_symbolic_point(ndim)); // TODO: We can't currently mix affine projections with delinearizing projections -#ifdef DEBUG_LEGATE - assert(ndim == launch_ndim); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(ndim == launch_ndim); } return Runtime::get_runtime()->get_projection(ndim, point); } @@ -638,9 +614,7 @@ std::shared_ptr LogicalStore::find_or_create_key_partition( } } else store_part = transform_->convert(storage_part); -#ifdef DEBUG_LEGATE - assert(store_part != nullptr); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(store_part != nullptr); } return store_part; } @@ -725,9 +699,7 @@ std::unique_ptr LogicalStore::to_launcher_arg(const Variable* variab std::unique_ptr LogicalStore::to_launcher_arg_for_fixup(const Domain* launch_domain, Legion::PrivilegeMode privilege) { -#ifdef DEBUG_LEGATE - assert(key_partition_ != nullptr); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(key_partition_ != nullptr); } auto store_partition = create_partition(key_partition_); auto proj_info = store_partition->create_projection_info(launch_domain); return std::make_unique(this, privilege, std::move(proj_info)); diff --git a/src/core/data/detail/store.cc b/src/core/data/detail/store.cc index 1456823603..118d5fb052 100644 --- a/src/core/data/detail/store.cc +++ b/src/core/data/detail/store.cc @@ -17,7 +17,7 @@ #include "core/utilities/dispatch.h" #include "core/utilities/machine.h" -#ifdef LEGATE_USE_CUDA +#if LegateDefined(LEGATE_USE_CUDA) #include "core/cuda/cuda_help.h" #include "core/cuda/stream_pool.h" #endif @@ -120,14 +120,14 @@ void UnboundRegionField::bind_empty_data(int32_t ndim) ReturnValue UnboundRegionField::pack_weight() const { -#ifdef DEBUG_LEGATE - if (!bound_) { - legate::log_legate.error( - "Found an uninitialized unbound store. Please make sure you return buffers to all unbound " - "stores in the task"); - LEGATE_ABORT; + if (LegateDefined(LEGATE_USE_DEBUG)) { + if (!bound_) { + legate::log_legate.error( + "Found an uninitialized unbound store. Please make sure you return buffers to all unbound " + "stores in the task"); + LEGATE_ABORT; + } } -#endif return ReturnValue(num_elements_, sizeof(size_t)); } @@ -145,19 +145,14 @@ FutureWrapper::FutureWrapper(bool read_only, : read_only_(read_only), field_size_(field_size), domain_(domain), future_(future) { if (!read_only) { -#ifdef DEBUG_LEGATE - assert(!initialize || future_.get_untyped_size() == field_size); -#endif - auto mem_kind = find_memory_kind_for_executing_processor( -#ifdef LEGATE_NO_FUTURES_ON_FB - true -#else - false -#endif - ); + if (LegateDefined(LEGATE_USE_DEBUG)) { + assert(!initialize || future_.get_untyped_size() == field_size); + } + auto mem_kind = + find_memory_kind_for_executing_processor(LegateDefined(LEGATE_NO_FUTURES_ON_FB)); if (initialize) { auto p_init_value = future_.get_buffer(mem_kind); -#ifdef LEGATE_USE_CUDA +#if LegateDefined(LEGATE_USE_CUDA) if (mem_kind == Memory::Kind::GPU_FB_MEM) { // TODO: This should be done by Legion buffer_ = Legion::UntypedDeferredValue(field_size, mem_kind); @@ -200,11 +195,9 @@ void FutureWrapper::initialize_with_identity(int32_t redop_id) auto ptr = untyped_acc.ptr(0); auto redop = Legion::Runtime::get_reduction_op(redop_id); -#ifdef DEBUG_LEGATE - assert(redop->sizeof_lhs == field_size_); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(redop->sizeof_lhs == field_size_); } auto identity = redop->identity; -#ifdef LEGATE_USE_CUDA +#if LegateDefined(LEGATE_USE_CUDA) if (buffer_.get_instance().get_location().kind() == Memory::Kind::GPU_FB_MEM) { auto stream = cuda::StreamPool::get_stream_pool().get_stream(); CHECK_CUDA(cudaMemcpyAsync(ptr, identity, field_size_, cudaMemcpyHostToDevice, stream)); @@ -343,9 +336,7 @@ Domain Store::domain() const throw std::invalid_argument("Invalid to retrieve the domain of an unbound store"); auto result = is_future_ ? future_.domain() : region_field_.domain(); if (!transform_->identity()) result = transform_->transform(result); -#ifdef DEBUG_LEGATE - assert(result.dim == dim_ || dim_ == 0); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(result.dim == dim_ || dim_ == 0); } return result; } @@ -408,34 +399,26 @@ bool Store::is_read_only_future() const { return future_.is_read_only(); } void Store::get_region_field(Legion::PhysicalRegion& pr, Legion::FieldID& fid) const { -#ifdef DEBUG_LEGATE - assert(!(is_future() || is_unbound_store())); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(!(is_future() || is_unbound_store())); } pr = region_field_.get_physical_region(); fid = region_field_.get_field_id(); } Legion::Future Store::get_future() const { -#ifdef DEBUG_LEGATE - assert(is_future()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(is_future()); } return future_.get_future(); } Legion::UntypedDeferredValue Store::get_buffer() const { -#ifdef DEBUG_LEGATE - assert(is_future()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(is_future()); } return future_.get_buffer(); } void Store::get_output_field(Legion::OutputRegion& out, Legion::FieldID& fid) { -#ifdef DEBUG_LEGATE - assert(is_unbound_store()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(is_unbound_store()); } out = unbound_field_.get_output_region(); fid = unbound_field_.get_field_id(); } diff --git a/src/core/data/detail/transform.cc b/src/core/data/detail/transform.cc index 6832084f66..5d29428764 100644 --- a/src/core/data/detail/transform.cc +++ b/src/core/data/detail/transform.cc @@ -119,17 +119,13 @@ void TransformStack::pack(BufferBuilder& buffer) const Legion::Domain TransformStack::transform(const Legion::Domain& input) const { -#ifdef DEBUG_LEGATE - assert(transform_ != nullptr); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(transform_ != nullptr); } return transform_->transform(parent_->identity() ? input : parent_->transform(input)); } Legion::DomainAffineTransform TransformStack::inverse_transform(int32_t in_dim) const { -#ifdef DEBUG_LEGATE - assert(transform_ != nullptr); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(transform_ != nullptr); } auto result = transform_->inverse_transform(in_dim); auto out_dim = transform_->target_ndim(in_dim); @@ -157,9 +153,7 @@ void TransformStack::print(std::ostream& out) const std::unique_ptr TransformStack::pop() { -#ifdef DEBUG_LEGATE - assert(transform_ != nullptr); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(transform_ != nullptr); } auto result = std::move(transform_); if (parent_ != nullptr) { transform_ = std::move(parent_->transform_); @@ -706,9 +700,7 @@ void Transpose::find_imaginary_dims(std::vector& dims) const // e.g. X.promoted = [0] => X.transpose((1,2,0)).promoted = [2] for (auto& promoted : dims) { auto finder = std::find(axes_.begin(), axes_.end(), promoted); -#ifdef DEBUG_LEGATE - assert(finder != axes_.end()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(finder != axes_.end()); } promoted = finder - axes_.begin(); } } diff --git a/src/core/mapping/detail/array.cc b/src/core/mapping/detail/array.cc index b931cfa979..eb326dc37e 100644 --- a/src/core/mapping/detail/array.cc +++ b/src/core/mapping/detail/array.cc @@ -27,9 +27,9 @@ BaseArray::BaseArray(std::shared_ptr data, std::shared_ptr null_ma bool BaseArray::unbound() const { -#ifdef DEBUG_LEGATE - assert(!nullable() || data_->unbound() == null_mask_->unbound()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + assert(!nullable() || data_->unbound() == null_mask_->unbound()); + } return data_->unbound(); } diff --git a/src/core/mapping/detail/base_mapper.cc b/src/core/mapping/detail/base_mapper.cc index abd8db7408..001e0e41ae 100644 --- a/src/core/mapping/detail/base_mapper.cc +++ b/src/core/mapping/detail/base_mapper.cc @@ -48,9 +48,7 @@ std::string log_mappable(const Legion::Mappable& mappable, bool prefix_only = fa {LEGION_PARTITION_MAPPABLE, "Partition "}, }; auto finder = prefixes.find(mappable.get_mappable_type()); -#ifdef DEBUG_LEGATE - assert(finder != prefixes.end()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(finder != prefixes.end()); } if (prefix_only) return finder->second; std::stringstream ss; @@ -126,37 +124,36 @@ void BaseMapper::select_task_options(const Legion::Mapping::MapperContext ctx, TaskOptions& output) { Task legate_task(&task, library, runtime, ctx); -#ifdef LEGATE_USE_COLLECTIVE - auto hi = task.index_domain.hi(); - auto lo = task.index_domain.lo(); - for (auto& array : legate_task.inputs()) { - auto stores = array->stores(); - for (auto& store : stores) { - if (store->is_future()) continue; - std::vector promoted_dims = store->find_imaginary_dims(); - for (auto& d : promoted_dims) { - if ((hi[d] - lo[d]) >= 1) { - output.check_collective_regions.insert(store->requirement_index()); - break; + if (LegateDefined(LEGATE_USE_COLLECTIVE)) { + auto hi = task.index_domain.hi(); + auto lo = task.index_domain.lo(); + for (auto& array : legate_task.inputs()) { + auto stores = array->stores(); + for (auto& store : stores) { + if (store->is_future()) continue; + std::vector promoted_dims = store->find_imaginary_dims(); + for (auto& d : promoted_dims) { + if ((hi[d] - lo[d]) >= 1) { + output.check_collective_regions.insert(store->requirement_index()); + break; + } } } } - } - for (auto& array : legate_task.reductions()) { - auto stores = array->stores(); - for (auto& store : stores) { - if (store->is_future()) continue; - auto idx = store->requirement_index(); - auto req = task.regions[idx]; - if (req.privilege & LEGION_WRITE_PRIV) continue; - if (req.handle_type == LEGION_SINGULAR_PROJECTION || req.projection != 0) { - output.check_collective_regions.insert(idx); + for (auto& array : legate_task.reductions()) { + auto stores = array->stores(); + for (auto& store : stores) { + if (store->is_future()) continue; + auto idx = store->requirement_index(); + auto req = task.regions[idx]; + if (req.privilege & LEGION_WRITE_PRIV) continue; + if (req.handle_type == LEGION_SINGULAR_PROJECTION || req.projection != 0) { + output.check_collective_regions.insert(idx); + } } } } -#endif - auto& machine_desc = legate_task.machine(); auto all_targets = machine_desc.valid_targets(); @@ -244,9 +241,7 @@ std::optional BaseMapper::find_variant(const Legion::Mapping: runtime->find_valid_variants(ctx, key.first, avail_variants, key.second); std::optional result; for (auto vid : avail_variants) { -#ifdef DEBUG_LEGATE - assert(vid > 0); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(vid > 0); } switch (vid) { case LEGATE_CPU_VARIANT: case LEGATE_OMP_VARIANT: @@ -266,19 +261,17 @@ void BaseMapper::map_task(const Legion::Mapping::MapperContext ctx, const MapTaskInput& input, MapTaskOutput& output) { -#ifdef DEBUG_LEGATE - logger.debug() << "Entering map_task for " - << Legion::Mapping::Utilities::to_string(runtime, ctx, task); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + logger.debug() << "Entering map_task for " + << Legion::Mapping::Utilities::to_string(runtime, ctx, task); + } // Should never be mapping the top-level task here assert(task.get_depth() > 0); // Let's populate easy outputs first auto variant = find_variant(ctx, task, task.target_proc.kind()); -#ifdef DEBUG_LEGATE - assert(variant.has_value()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(variant.has_value()); } output.chosen_variant = *variant; Task legate_task(&task, library, runtime, ctx); @@ -291,9 +284,7 @@ void BaseMapper::map_task(const Legion::Mapping::MapperContext ctx, // If this is a single task, here is the right place to compute the final target processor auto local_range = local_machine.slice(legate_task.target(), legate_task.machine(), task.local_function); -#ifdef DEBUG_LEGATE - assert(!local_range.empty()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(!local_range.empty()); } output.target_procs.push_back(local_range.first()); } @@ -312,9 +303,9 @@ void BaseMapper::map_task(const Legion::Mapping::MapperContext ctx, assert(!(mapping->for_future() || mapping->for_unbound_store()) || mapping->stores.size() == 1); }; -#ifdef DEBUG_LEGATE - for (auto& client_mapping : client_mappings) validate_colocation(client_mapping.impl()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + for (auto& client_mapping : client_mappings) validate_colocation(client_mapping.impl()); + } std::vector> for_futures; std::vector> for_unbound_stores; @@ -359,9 +350,7 @@ void BaseMapper::map_task(const Legion::Mapping::MapperContext ctx, } } }; -#ifdef DEBUG_LEGATE - check_consistency(for_stores); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { check_consistency(for_stores); } // Generate default mappings for stores that are not yet mapped by the client mapper auto default_option = options.front(); @@ -393,23 +382,23 @@ void BaseMapper::map_task(const Legion::Mapping::MapperContext ctx, generate_default_mappings(legate_task.inputs(), false); generate_default_mappings(legate_task.outputs(), false); generate_default_mappings(legate_task.reductions(), false); -#ifdef DEBUG_LEGATE - assert(mapped_futures.size() <= task.futures.size()); - // The launching code should be packing all Store-backing Futures first. - if (mapped_futures.size() > 0) { - uint32_t max_mapped_fut = *mapped_futures.rbegin(); - assert(mapped_futures.size() == max_mapped_fut + 1); + if (LegateDefined(LEGATE_USE_DEBUG)) { + assert(mapped_futures.size() <= task.futures.size()); + // The launching code should be packing all Store-backing Futures first. + if (mapped_futures.size() > 0) { + uint32_t max_mapped_fut = *mapped_futures.rbegin(); + assert(mapped_futures.size() == max_mapped_fut + 1); + } } -#endif // Map future-backed stores output.future_locations.resize(mapped_futures.size()); for (auto& mapping : for_futures) { auto fut_idx = mapping->store()->future_index(); StoreTarget target = mapping->policy.target; -#ifdef LEGATE_NO_FUTURES_ON_FB - if (target == StoreTarget::FBMEM) target = StoreTarget::ZCMEM; -#endif + if (LegateDefined(LEGATE_NO_FUTURES_ON_FB)) { + if (target == StoreTarget::FBMEM) target = StoreTarget::ZCMEM; + } output.future_locations[fut_idx] = local_machine.get_memory(task.target_proc, target); } @@ -464,27 +453,25 @@ void BaseMapper::map_legate_stores(const Legion::Mapping::MapperContext ctx, auto reqs = mapping->requirements(); while (map_legate_store(ctx, mappable, *mapping, reqs, target_proc, result, can_fail)) { if (NO_INST == result) { -#ifdef DEBUG_LEGATE - assert(can_fail); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(can_fail); } for (auto& instance : instances) runtime->release_instance(ctx, instance); return false; } -#ifdef DEBUG_LEGATE std::stringstream reqs_ss; - for (auto req_idx : mapping->requirement_indices()) reqs_ss << " " << req_idx; -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + for (auto req_idx : mapping->requirement_indices()) reqs_ss << " " << req_idx; + } if (runtime->acquire_instance(ctx, result)) { -#ifdef DEBUG_LEGATE - logger.debug() << log_mappable(mappable) << ": acquired instance " << result - << " for reqs:" << reqs_ss.str(); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + logger.debug() << log_mappable(mappable) << ": acquired instance " << result + << " for reqs:" << reqs_ss.str(); + } break; } -#ifdef DEBUG_LEGATE - logger.debug() << log_mappable(mappable) << ": failed to acquire instance " << result - << " for reqs:" << reqs_ss.str(); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + logger.debug() << log_mappable(mappable) << ": failed to acquire instance " << result + << " for reqs:" << reqs_ss.str(); + } if ((*reqs.begin())->redop != 0) { Legion::Mapping::AutoLock lock(ctx, reduction_instances->manager_lock()); reduction_instances->erase(result); @@ -512,10 +499,10 @@ void BaseMapper::map_legate_stores(const Legion::Mapping::MapperContext ctx, for (auto& mapping : mappings) can_fail = can_fail || !mapping->policy.exact; if (!try_mapping(can_fail)) { -#ifdef DEBUG_LEGATE - logger.debug() << log_mappable(mappable) << " failed to map all stores, retrying with " - << "tighter policies"; -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + logger.debug() << log_mappable(mappable) << " failed to map all stores, retrying with " + << "tighter policies"; + } // If instance creation failed we try mapping all stores again, but request tight instances for // write requirements. The hope is that these write requirements cover the entire region (i.e. // they use a complete partition), so the new tight instances will invalidate any pre-existing @@ -537,12 +524,12 @@ void BaseMapper::tighten_write_policies(const Legion::Mappable& mappable, // We tighten only write requirements if (!(priv & LEGION_WRITE_PRIV)) continue; -#ifdef DEBUG_LEGATE - std::stringstream reqs_ss; - for (auto req_idx : mapping->requirement_indices()) reqs_ss << " " << req_idx; - logger.debug() << log_mappable(mappable) - << ": tightened mapping policy for reqs:" << reqs_ss.str(); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + std::stringstream reqs_ss; + for (auto req_idx : mapping->requirement_indices()) reqs_ss << " " << req_idx; + logger.debug() << log_mappable(mappable) + << ": tightened mapping policy for reqs:" << reqs_ss.str(); + } mapping->policy.exact = true; } } @@ -568,16 +555,16 @@ bool BaseMapper::map_legate_store(const Legion::Mapping::MapperContext ctx, auto target_memory = local_machine.get_memory(target_proc, policy.target); auto redop = (*reqs.begin())->redop; -#ifdef DEBUG_LEGATE - for (auto* req : reqs) { - if (redop != req->redop) { - logger.error( - "Colocated stores should be either non-reduction arguments " - "or reductions with the same reduction operator."); - LEGATE_ABORT; + if (LegateDefined(LEGATE_USE_DEBUG)) { + for (auto* req : reqs) { + if (redop != req->redop) { + logger.error( + "Colocated stores should be either non-reduction arguments " + "or reductions with the same reduction operator."); + LEGATE_ABORT; + } } } -#endif // Targets of reduction copies should be mapped to normal instances if (mappable.get_mappable_type() == Legion::Mappable::COPY_MAPPABLE) redop = 0; @@ -601,11 +588,11 @@ bool BaseMapper::map_legate_store(const Legion::Mapping::MapperContext ctx, if (fields.size() == 1 && regions.size() == 1 && reduction_instances->find_instance( redop, regions.front(), fields.front(), target_memory, result, policy)) { -#ifdef DEBUG_LEGATE - logger.debug() << "Operation " << mappable.get_unique_id() - << ": reused cached reduction instance " << result << " for " - << regions.front(); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + logger.debug() << "Operation " << mappable.get_unique_id() + << ": reused cached reduction instance " << result << " for " + << regions.front(); + } runtime->enable_reentrant(ctx); // Needs acquire to keep the runtime happy return true; @@ -625,13 +612,13 @@ bool BaseMapper::map_legate_store(const Legion::Mapping::MapperContext ctx, LEGION_GC_DEFAULT_PRIORITY, false /*tight bounds*/, &footprint)) { -#ifdef DEBUG_LEGATE - Realm::LoggerMessage msg = logger.debug(); - msg << "Operation " << mappable.get_unique_id() << ": created reduction instance " << result - << " for"; - for (auto& r : regions) msg << " " << r; - msg << " (size: " << footprint << " bytes, memory: " << target_memory << ")"; -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + Realm::LoggerMessage msg = logger.debug(); + msg << "Operation " << mappable.get_unique_id() << ": created reduction instance " << result + << " for"; + for (auto& r : regions) msg << " " << r; + msg << " (size: " << footprint << " bytes, memory: " << target_memory << ")"; + } if (target_proc.kind() == Processor::TOC_PROC) { // store reduction instance if (fields.size() == 1 && regions.size() == 1) { @@ -654,10 +641,10 @@ bool BaseMapper::map_legate_store(const Legion::Mapping::MapperContext ctx, if (fields.size() == 1 && regions.size() == 1 && local_instances->find_instance( regions.front(), fields.front(), target_memory, result, policy)) { -#ifdef DEBUG_LEGATE - logger.debug() << "Operation " << mappable.get_unique_id() << ": reused cached instance " - << result << " for " << regions.front(); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + logger.debug() << "Operation " << mappable.get_unique_id() << ": reused cached instance " + << result << " for " << regions.front(); + } runtime->enable_reentrant(ctx); // Needs acquire to keep the runtime happy return true; @@ -717,16 +704,16 @@ bool BaseMapper::map_legate_store(const Legion::Mapping::MapperContext ctx, if (success) { // We succeeded in making the instance where we want it assert(result.exists()); -#ifdef DEBUG_LEGATE - if (created) { - logger.debug() << "Operation " << mappable.get_unique_id() << ": created instance " << result - << " for " << *group << " (size: " << footprint - << " bytes, memory: " << target_memory << ")"; - } else { - logger.debug() << "Operation " << mappable.get_unique_id() << ": found instance " << result - << " for " << *group; + if (LegateDefined(LEGATE_USE_DEBUG)) { + if (created) { + logger.debug() << "Operation " << mappable.get_unique_id() << ": created instance " + << result << " for " << *group << " (size: " << footprint + << " bytes, memory: " << target_memory << ")"; + } else { + logger.debug() << "Operation " << mappable.get_unique_id() << ": found instance " << result + << " for " << *group; + } } -#endif // Only save the result for future use if it is not an external instance if (!result.is_external_instance() && group != nullptr) { assert(fields.size() == 1); @@ -811,9 +798,7 @@ void BaseMapper::select_task_variant(const Legion::Mapping::MapperContext ctx, SelectVariantOutput& output) { auto variant = find_variant(ctx, task, input.processor.kind()); -#ifdef DEBUG_LEGATE - assert(variant.has_value()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(variant.has_value()); } output.chosen_variant = *variant; } @@ -868,9 +853,7 @@ void find_source_instance_bandwidth( // and target memories, in which case we assign the smallest bandwidth // TODO: Not all multi-hop copies are equal if (!affinities.empty()) { -#ifdef DEBUG_LEGATE - assert(affinities.size() == 1); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(affinities.size() == 1); } bandwidth = affinities.front().bandwidth; } source_memory_bandwidths[source_memory] = bandwidth; @@ -904,10 +887,10 @@ void BaseMapper::legate_select_sources( for (uint32_t idx = 0; idx < collective_sources.size(); idx++) { std::vector source_instances; collective_sources[idx].find_instances_nearest_memory(target_memory, source_instances); -#ifdef DEBUG_LEGATE - // there must exist at least one instance in the collective view - assert(!source_instances.empty()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + // there must exist at least one instance in the collective view + assert(!source_instances.empty()); + } // we need only first instance if there are several find_source_instance_bandwidth(all_sources, source_memory_bandwidths, @@ -915,9 +898,7 @@ void BaseMapper::legate_select_sources( target_memory, legion_machine); } -#ifdef DEBUG_LEGATE - assert(!all_sources.empty()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(!all_sources.empty()); } if (all_sources.size() > 1) // Sort source instances by their bandwidths std::sort(all_sources.begin(), all_sources.end(), [](const auto& lhs, const auto& rhs) { @@ -962,9 +943,9 @@ void BaseMapper::map_inline(const Legion::Mapping::MapperContext ctx, auto store_target = default_store_targets(target_proc.kind()).front(); -#ifdef DEBUG_LEGATE - assert(inline_op.requirement.instance_fields.size() == 1); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + assert(inline_op.requirement.instance_fields.size() == 1); + } Store store(mapper_runtime, ctx, &inline_op.requirement); std::vector> mappings; @@ -1012,9 +993,7 @@ void BaseMapper::map_copy(const Legion::Mapping::MapperContext ctx, // However, if the machine in the scope doesn't have any CPU or OMP as a fallback for // indirect copies, we have no choice but using GPUs if (valid_targets.empty()) { -#ifdef DEBUG_LEGATE - assert(indirect); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(indirect); } valid_targets = machine_desc.valid_targets(); } return valid_targets.front(); @@ -1050,10 +1029,10 @@ void BaseMapper::map_copy(const Legion::Mapping::MapperContext ctx, add_to_output_map(copy.src_requirements, output.src_instances); add_to_output_map(copy.dst_requirements, output.dst_instances); -#ifdef DEBUG_LEGATE - assert(copy.src_indirect_requirements.size() <= 1); - assert(copy.dst_indirect_requirements.size() <= 1); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + assert(copy.src_indirect_requirements.size() <= 1); + assert(copy.dst_indirect_requirements.size() <= 1); + } if (!copy.src_indirect_requirements.empty()) { // This is to make the push_back call later add the isntance to the right place output.src_indirect_instances.clear(); @@ -1212,9 +1191,9 @@ void BaseMapper::map_partition(const Legion::Mapping::MapperContext ctx, auto store_target = default_store_targets(target_proc.kind()).front(); -#ifdef DEBUG_LEGATE - assert(partition.requirement.instance_fields.size() == 1); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + assert(partition.requirement.instance_fields.size() == 1); + } Store store(mapper_runtime, ctx, &partition.requirement); std::vector> mappings; @@ -1277,18 +1256,17 @@ void BaseMapper::map_future_map_reduction(const Legion::Mapping::MapperContext c // TODO: It's been reported that blindly mapping target instances of future map reductions // to framebuffers hurts performance. Until we find a better mapping policy, we guard // the current policy with a macro. -#ifdef LEGATE_MAP_FUTURE_MAP_REDUCTIONS_TO_GPU - - // If this was joining exceptions, we should put instances on a host-visible memory - // because they need serdez - if (input.tag == LEGATE_CORE_JOIN_EXCEPTION_TAG) + if (LegateDefined(LEGATE_MAP_FUTURE_MAP_REDUCTIONS_TO_GPU)) { + // If this was joining exceptions, we should put instances on a host-visible memory + // because they need serdez + if (input.tag == LEGATE_CORE_JOIN_EXCEPTION_TAG) + output.destination_memories.push_back(local_machine.zerocopy_memory()); + else + for (auto& pair : local_machine.frame_buffers()) + output.destination_memories.push_back(pair.second); + } else { output.destination_memories.push_back(local_machine.zerocopy_memory()); - else - for (auto& pair : local_machine.frame_buffers()) - output.destination_memories.push_back(pair.second); -#else - output.destination_memories.push_back(local_machine.zerocopy_memory()); -#endif + } } else if (local_machine.has_socket_memory()) for (auto& pair : local_machine.socket_memories()) output.destination_memories.push_back(pair.second); diff --git a/src/core/mapping/detail/core_mapper.cc b/src/core/mapping/detail/core_mapper.cc index e51569be26..fcbbc03f95 100644 --- a/src/core/mapping/detail/core_mapper.cc +++ b/src/core/mapping/detail/core_mapper.cc @@ -14,10 +14,8 @@ #include "core/mapping/detail/core_mapper.h" -#include "core/mapping/detail/machine.h" -#ifdef LEGATE_USE_CUDA #include "core/comm/comm_nccl.h" -#endif +#include "core/mapping/detail/machine.h" namespace legate { @@ -157,11 +155,8 @@ Scalar CoreMapper::tunable_value(TunableID tunable_id) return max_lru_length; } case LEGATE_CORE_TUNABLE_NCCL_NEEDS_BARRIER: { -#ifdef LEGATE_USE_CUDA - return machine.has_gpus() && comm::nccl::needs_barrier(); -#else - return false; -#endif + return LegateDefined(LEGATE_USE_CUDA) ? machine.has_gpus() && comm::nccl::needs_barrier() + : false; } } // Illegal tunable variable diff --git a/src/core/mapping/detail/instance_manager.cc b/src/core/mapping/detail/instance_manager.cc index 2dfb4fa9c8..6a609e759d 100644 --- a/src/core/mapping/detail/instance_manager.cc +++ b/src/core/mapping/detail/instance_manager.cc @@ -103,22 +103,21 @@ struct construct_overlapping_region_group_fn { size_t bound_vol = bound.volume(); std::set regions{region}; -#ifdef DEBUG_LEGATE - log_instmgr.debug() << "construct_overlapping_region_group( " << region << "," << domain << ")"; -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + log_instmgr.debug() << "construct_overlapping_region_group( " << region << "," << domain + << ")"; + } for (const auto& pair : instances) { auto& group = pair.first; Rect group_bbox = group->bounding_box.bounds(); -#ifdef DEBUG_LEGATE - log_instmgr.debug() << " check intersection with " << group_bbox; -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + log_instmgr.debug() << " check intersection with " << group_bbox; + } auto intersect = bound.intersection(group_bbox); if (intersect.empty()) { -#ifdef DEBUG_LEGATE - log_instmgr.debug() << " no intersection"; -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { log_instmgr.debug() << " no intersection"; } continue; } @@ -128,11 +127,11 @@ struct construct_overlapping_region_group_fn { size_t group_vol = group_bbox.volume(); size_t intersect_vol = intersect.volume(); if (too_big(union_vol, bound_vol, group_vol, intersect_vol)) { -#ifdef DEBUG_LEGATE - log_instmgr.debug() << " too big to merge (union:" << union_bbox - << ",bound:" << bound_vol << ",group:" << group_vol - << ",intersect:" << intersect_vol << ")"; -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + log_instmgr.debug() << " too big to merge (union:" << union_bbox + << ",bound:" << bound_vol << ",group:" << group_vol + << ",intersect:" << intersect_vol << ")"; + } continue; } @@ -141,9 +140,9 @@ struct construct_overlapping_region_group_fn { // thus there's at least one shared_ptr remaining that points to it. Otherwise we run the risk // that a group pointer stored on the instances_ table points to a group that's been collected regions.insert(group->regions.begin(), group->regions.end()); -#ifdef DEBUG_LEGATE - log_instmgr.debug() << " bounds updated: " << bound << " ~> " << union_bbox; -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + log_instmgr.debug() << " bounds updated: " << bound << " ~> " << union_bbox; + } bound = union_bbox; bound_vol = union_vol; @@ -171,13 +170,13 @@ std::set InstanceSet::record_instance(RegionGroupP group, Instance instance, const InstanceMappingPolicy& policy) { -#ifdef DEBUG_LEGATE + if (LegateDefined(LEGATE_USE_DEBUG)) { #ifdef DEBUG_INSTANCE_MANAGER - log_instmgr.debug() << "===== before adding an entry " << *group << " ~> " << instance - << " ====="; + log_instmgr.debug() << "===== before adding an entry " << *group << " ~> " << instance + << " ====="; #endif + } dump_and_sanity_check(); -#endif std::set replaced; std::set removed_groups; @@ -221,25 +220,25 @@ std::set InstanceSet::record_instance(RegionGroupP group, replaced.erase(instance); -#ifdef DEBUG_LEGATE + if (LegateDefined(LEGATE_USE_DEBUG)) { #ifdef DEBUG_INSTANCE_MANAGER - log_instmgr.debug() << "===== after adding an entry " << *group << " ~> " << instance << " ====="; + log_instmgr.debug() << "===== after adding an entry " << *group << " ~> " << instance + << " ====="; #endif - dump_and_sanity_check(); -#endif - + dump_and_sanity_check(); + } return replaced; } bool InstanceSet::erase(Instance inst) { std::set filtered_groups; -#ifdef DEBUG_LEGATE + if (LegateDefined(LEGATE_USE_DEBUG)) { #ifdef DEBUG_INSTANCE_MANAGER - log_instmgr.debug() << "===== before erasing an instance " << inst << " ====="; + log_instmgr.debug() << "===== before erasing an instance " << inst << " ====="; #endif + } dump_and_sanity_check(); -#endif for (auto it = instances_.begin(); it != instances_.end(); /*nothing*/) { if (it->second.instance == inst) { @@ -259,12 +258,12 @@ bool InstanceSet::erase(Instance inst) filtered_regions.insert(region); for (Region region : filtered_regions) groups_.erase(region); -#ifdef DEBUG_LEGATE + if (LegateDefined(LEGATE_USE_DEBUG)) { #ifdef DEBUG_INSTANCE_MANAGER - log_instmgr.debug() << "===== after erasing an instance " << inst << " ====="; + log_instmgr.debug() << "===== after erasing an instance " << inst << " ====="; #endif + } dump_and_sanity_check(); -#endif return instances_.empty(); } @@ -362,10 +361,10 @@ RegionGroupP InstanceManager::find_region_group(const Region& region, else result = finder->second.construct_overlapping_region_group(region, domain, exact); -#ifdef DEBUG_LEGATE - log_instmgr.debug() << "find_region_group(" << region << "," << domain << "," << field_id << "," - << memory << "," << exact << ") ~> " << *result; -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + log_instmgr.debug() << "find_region_group(" << region << "," << domain << "," << field_id << "," + << memory << "," << exact << ") ~> " << *result; + } return result; } diff --git a/src/core/mapping/detail/machine.cc b/src/core/mapping/detail/machine.cc index e99f8b74e1..3ab5bbb14d 100644 --- a/src/core/mapping/detail/machine.cc +++ b/src/core/mapping/detail/machine.cc @@ -178,9 +178,7 @@ LocalProcessorRange::LocalProcessorRange(uint32_t offset, const Processor& LocalProcessorRange::operator[](uint32_t idx) const { auto local_idx = (idx % total_proc_count_) - offset_; -#ifdef DEBUG_LEGATE - assert(local_idx < procs_.size()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(local_idx < procs_.size()); } return procs_[local_idx]; } diff --git a/src/core/mapping/detail/mapping.cc b/src/core/mapping/detail/mapping.cc index f00e232f66..1df78c0c2a 100644 --- a/src/core/mapping/detail/mapping.cc +++ b/src/core/mapping/detail/mapping.cc @@ -103,20 +103,20 @@ const Store* StoreMapping::store() const { return stores.front(); } uint32_t StoreMapping::requirement_index() const { -#ifdef DEBUG_LEGATE - assert(stores.size() > 0); - uint32_t result = -1U; - for (auto& store : stores) { - auto idx = store->requirement_index(); - assert(result == -1U || result == idx); - result = idx; + if (LegateDefined(LEGATE_USE_DEBUG)) { + assert(stores.size() > 0); + uint32_t result = -1U; + for (auto& store : stores) { + auto idx = store->requirement_index(); + assert(result == -1U || result == idx); + result = idx; + } + return result; + } else { + static constexpr uint32_t invalid = -1U; + if (stores.empty()) return invalid; + return stores.front()->requirement_index(); } - return result; -#else - static constexpr uint32_t invalid = -1U; - if (stores.empty()) return invalid; - return stores.front()->requirement_index(); -#endif } std::set StoreMapping::requirement_indices() const diff --git a/src/core/mapping/detail/operation.cc b/src/core/mapping/detail/operation.cc index 7351a7adec..6a421338ec 100644 --- a/src/core/mapping/detail/operation.cc +++ b/src/core/mapping/detail/operation.cc @@ -75,12 +75,12 @@ Copy::Copy(const Legion::Copy* copy, input_indirections_ = dez.unpack(); dez.next_requirement_list(); output_indirections_ = dez.unpack(); -#ifdef DEBUG_LEGATE - for (auto& input : inputs_) assert(!input.is_future()); - for (auto& output : outputs_) assert(!output.is_future()); - for (auto& input_indirection : input_indirections_) assert(!input_indirection.is_future()); - for (auto& output_indirection : output_indirections_) assert(!output_indirection.is_future()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + for (auto& input : inputs_) assert(!input.is_future()); + for (auto& output : outputs_) assert(!output.is_future()); + for (auto& input_indirection : input_indirections_) assert(!input_indirection.is_future()); + for (auto& output_indirection : output_indirections_) assert(!output_indirection.is_future()); + } } } // namespace legate::mapping::detail diff --git a/src/core/mapping/detail/store.cc b/src/core/mapping/detail/store.cc index f14db69470..369e30b823 100644 --- a/src/core/mapping/detail/store.cc +++ b/src/core/mapping/detail/store.cc @@ -104,17 +104,13 @@ bool Store::can_colocate_with(const Store& other) const const RegionField& Store::region_field() const { -#ifdef DEBUG_LEGATE - assert(!is_future()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(!is_future()); } return region_field_; } const FutureWrapper& Store::future() const { -#ifdef DEBUG_LEGATE - assert(is_future()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(is_future()); } return future_; } diff --git a/src/core/operation/detail/req_analyzer.cc b/src/core/operation/detail/req_analyzer.cc index a96ade3908..a4094ddd6a 100644 --- a/src/core/operation/detail/req_analyzer.cc +++ b/src/core/operation/detail/req_analyzer.cc @@ -51,9 +51,7 @@ uint32_t FieldSet::get_requirement_index(Legion::PrivilegeMode privilege, { auto finder = req_indices_.find(Key(privilege, proj_info)); if (req_indices_.end() == finder) finder = req_indices_.find(Key(LEGION_READ_WRITE, proj_info)); -#ifdef DEBUG_LEGATE - assert(finder != req_indices_.end()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(finder != req_indices_.end()); } return finder->second; } @@ -107,9 +105,7 @@ uint32_t RequirementAnalyzer::get_requirement_index(const Legion::LogicalRegion& const ProjectionInfo& proj_info) const { auto finder = field_sets_.find(region); -#ifdef DEBUG_LEGATE - assert(finder != field_sets_.end()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(finder != field_sets_.end()); } auto& [field_set, req_offset] = finder->second; return req_offset + field_set.get_requirement_index(privilege, proj_info); } @@ -150,10 +146,10 @@ void OutputRequirementAnalyzer::insert(int32_t dim, Legion::FieldID field_id) { auto& req_info = req_infos_[field_space]; -#ifdef DEBUG_LEGATE - // TODO: This should be checked when alignment constraints are set on unbound stores - assert(-1 == req_info.dim || req_info.dim == dim); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + // TODO: This should be checked when alignment constraints are set on unbound stores + assert(-1 == req_info.dim || req_info.dim == dim); + } req_info.dim = dim; field_groups_[field_space].insert(field_id); } @@ -162,9 +158,7 @@ uint32_t OutputRequirementAnalyzer::get_requirement_index(const Legion::FieldSpa Legion::FieldID field_id) const { auto finder = req_infos_.find(field_space); -#ifdef DEBUG_LEGATE - assert(finder != req_infos_.end()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(finder != req_infos_.end()); } return finder->second.req_idx; } diff --git a/src/core/operation/detail/task.cc b/src/core/operation/detail/task.cc index f8b1415500..5e62be8e42 100644 --- a/src/core/operation/detail/task.cc +++ b/src/core/operation/detail/task.cc @@ -146,7 +146,7 @@ void Task::demux_scalar_stores(const Legion::Future& result) auto* runtime = detail::Runtime::get_runtime(); runtime->record_pending_exception(result); } -#ifdef DEBUG_LEGATE +#if LegateDefined(LEGATE_USE_DEBUG) else { assert(1 == num_unbound_outs); } @@ -185,7 +185,7 @@ void Task::demux_scalar_stores(const Legion::FutureMap& result, const Domain& la auto* runtime = detail::Runtime::get_runtime(); runtime->record_pending_exception(runtime->reduce_exception_future_map(result)); } -#ifdef DEBUG_LEGATE +#if LegateDefined(LEGATE_USE_DEBUG) else { assert(1 == num_unbound_outs); } @@ -365,10 +365,10 @@ void ManualTask::add_output(std::shared_ptr store) void ManualTask::add_output(std::shared_ptr store_partition) { -#ifdef DEBUG_LEGATE - // TODO: We need to raise an exception for the user error in this case - assert(!store_partition->store()->unbound()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + // TODO: We need to raise an exception for the user error in this case + assert(!store_partition->store()->unbound()); + } if (store_partition->store()->has_scalar_storage()) { record_scalar_output(store_partition->store()); } diff --git a/src/core/operation/detail/task_launcher.cc b/src/core/operation/detail/task_launcher.cc index 8c365ea78f..d0c20507b0 100644 --- a/src/core/operation/detail/task_launcher.cc +++ b/src/core/operation/detail/task_launcher.cc @@ -232,9 +232,7 @@ void TaskLauncher::post_process_unbound_stores( auto no_part = create_no_partition(); for (auto& arg : unbound_stores_) { -#ifdef DEBUG_LEGATE - assert(arg->requirement_index() != -1U); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(arg->requirement_index() != -1U); } auto* store = arg->store(); auto& req = output_requirements[arg->requirement_index()]; auto region_field = diff --git a/src/core/partitioning/detail/constraint_solver.cc b/src/core/partitioning/detail/constraint_solver.cc index 2d0a4f58ee..4656174a9d 100644 --- a/src/core/partitioning/detail/constraint_solver.cc +++ b/src/core/partitioning/detail/constraint_solver.cc @@ -125,15 +125,11 @@ void ConstraintSolver::solve_constraints() std::vector part_symbs_to_unify; alignment->find_partition_symbols(part_symbs_to_unify); -#ifdef DEBUG_LEGATE - assert(!part_symbs_to_unify.empty()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(!part_symbs_to_unify.empty()); } auto it = part_symbs_to_unify.begin(); auto* equiv_class = table[**it++]; -#ifdef DEBUG_LEGATE - assert(equiv_class != nullptr); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(equiv_class != nullptr); } for (; it != part_symbs_to_unify.end(); ++it) { auto* to_unify = table[**it]; auto* result = equiv_class->unify(to_unify); diff --git a/src/core/partitioning/detail/partitioner.cc b/src/core/partitioning/detail/partitioner.cc index 3c854de9b2..c24ce893cf 100644 --- a/src/core/partitioning/detail/partitioner.cc +++ b/src/core/partitioning/detail/partitioner.cc @@ -69,16 +69,12 @@ std::unique_ptr LaunchDomainResolver::resolve_launch_domain() const if (unbound_dim_ != UNSET && unbound_dim_ > 1) return nullptr; else { -#ifdef DEBUG_LEGATE - assert(launch_volumes_.size() == 1); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(launch_volumes_.size() == 1); } int64_t volume = *launch_volumes_.begin(); return std::make_unique(0, volume - 1); } } else { -#ifdef DEBUG_LEGATE - assert(launch_domains_.size() == 1); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(launch_domains_.size() == 1); } auto& launch_domain = *launch_domains_.begin(); if (unbound_dim_ != UNSET && launch_domain.dim != unbound_dim_) { int64_t volume = *launch_volumes_.begin(); @@ -108,17 +104,17 @@ const Domain* Strategy::launch_domain(const Operation* op) const void Strategy::set_launch_shape(const Operation* op, const Shape& shape) { -#ifdef DEBUG_LEGATE - assert(launch_domains_.find(op) == launch_domains_.end()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + assert(launch_domains_.find(op) == launch_domains_.end()); + } launch_domains_.insert({op, std::make_unique(to_domain(shape))}); } void Strategy::insert(const Variable* partition_symbol, std::shared_ptr partition) { -#ifdef DEBUG_LEGATE - assert(assignments_.find(*partition_symbol) == assignments_.end()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + assert(assignments_.find(*partition_symbol) == assignments_.end()); + } assignments_.insert({*partition_symbol, std::move(partition)}); } @@ -126,9 +122,9 @@ void Strategy::insert(const Variable* partition_symbol, std::shared_ptr partition, Legion::FieldSpace field_space) { -#ifdef DEBUG_LEGATE - assert(field_spaces_.find(*partition_symbol) == field_spaces_.end()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + assert(field_spaces_.find(*partition_symbol) == field_spaces_.end()); + } field_spaces_.insert({*partition_symbol, field_space}); insert(partition_symbol, std::move(partition)); } @@ -141,18 +137,14 @@ bool Strategy::has_assignment(const Variable* partition_symbol) const std::shared_ptr Strategy::operator[](const Variable* partition_symbol) const { auto finder = assignments_.find(*partition_symbol); -#ifdef DEBUG_LEGATE - assert(finder != assignments_.end()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(finder != assignments_.end()); } return finder->second; } const Legion::FieldSpace& Strategy::find_field_space(const Variable* partition_symbol) const { auto finder = field_spaces_.find(*partition_symbol); -#ifdef DEBUG_LEGATE - assert(finder != field_spaces_.end()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(finder != field_spaces_.end()); } return finder->second; } @@ -222,9 +214,7 @@ std::unique_ptr Partitioner::partition_stores() solver.solve_constraints(); -#ifdef DEBUG_LEGATE - solver.dump(); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { solver.dump(); } auto strategy = std::make_unique(); @@ -238,9 +228,7 @@ std::unique_ptr Partitioner::partition_stores() auto store = op->find_store(part_symb); auto has_key_part = store->has_key_partition(op->machine(), solver.find_restrictions(part_symb)); -#ifdef DEBUG_LEGATE - assert(!store->unbound()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(!store->unbound()); } return std::make_pair(store->storage_size(), has_key_part); }; @@ -263,9 +251,7 @@ std::unique_ptr Partitioner::partition_stores() auto store = op->find_store(part_symb); auto partition = store->find_or_create_key_partition(op->machine(), restrictions); strategy->record_key_partition(part_symb); -#ifdef DEBUG_LEGATE - assert(partition != nullptr); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(partition != nullptr); } for (auto symb : equiv_class) strategy->insert(symb, partition); } @@ -274,9 +260,7 @@ std::unique_ptr Partitioner::partition_stores() strategy->compute_launch_domains(solver); -#ifdef DEBUG_LEGATE - strategy->dump(); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { strategy->dump(); } return strategy; } diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index 1c00a8f2d2..6859b2dcbb 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -122,10 +122,10 @@ bool Tiling::is_complete_for(const detail::Storage* storage) const const auto& storage_exts = storage->extents(); const auto& storage_offs = storage->offsets(); -#ifdef DEBUG_LEGATE - assert(storage_exts.size() == storage_offs.size()); - assert(storage_offs.size() == offsets_.size()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + assert(storage_exts.size() == storage_offs.size()); + assert(storage_offs.size() == offsets_.size()); + } uint32_t ndim = storage_exts.size(); diff --git a/src/core/runtime/detail/library.cc b/src/core/runtime/detail/library.cc index 0a442e69df..6f74dc63b0 100644 --- a/src/core/runtime/detail/library.cc +++ b/src/core/runtime/detail/library.cc @@ -145,10 +145,10 @@ void Library::register_task(int64_t local_task_id, std::unique_ptr tas throw std::out_of_range(std::move(ss).str()); } -#ifdef DEBUG_LEGATE - log_legate.debug() << "[" << library_name_ << "] task " << local_task_id - << " (global id: " << task_id << "), " << *task_info; -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + log_legate.debug() << "[" << library_name_ << "] task " << local_task_id + << " (global id: " << task_id << "), " << *task_info; + } if (tasks_.find(local_task_id) != tasks_.end()) throw std::invalid_argument("Task " + std::to_string(local_task_id) + " already exists in library " + library_name_); diff --git a/src/core/runtime/detail/machine_manager.cc b/src/core/runtime/detail/machine_manager.cc index 6a1363f9cf..3cf51f963e 100644 --- a/src/core/runtime/detail/machine_manager.cc +++ b/src/core/runtime/detail/machine_manager.cc @@ -19,9 +19,7 @@ namespace legate::detail { //////////////////////////////////////////// const mapping::detail::Machine& MachineManager::get_machine() const { -#ifdef DEBUG_LEGATE - assert(machines_.size() > 0); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(machines_.size() > 0); } return machines_.back(); } diff --git a/src/core/runtime/detail/partition_manager.cc b/src/core/runtime/detail/partition_manager.cc index 0bcef0ab50..94e74ac8fc 100644 --- a/src/core/runtime/detail/partition_manager.cc +++ b/src/core/runtime/detail/partition_manager.cc @@ -26,9 +26,7 @@ PartitionManager::PartitionManager(Runtime* runtime) min_shard_volume_ = runtime->get_tunable(mapper_id, LEGATE_CORE_TUNABLE_MIN_SHARD_VOLUME); -#ifdef DEBUG_LEGATE - assert(min_shard_volume_ > 0); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(min_shard_volume_ > 0); } } const std::vector& PartitionManager::get_factors(const mapping::Machine& machine) @@ -272,9 +270,7 @@ void PartitionManager::invalidate_image_partition(const Legion::IndexSpace& inde Legion::FieldID field_id) { auto finder = image_cache_.find(std::tie(index_space, func_partition, field_id)); -#ifdef DEBUG_LEGATE - assert(finder != image_cache_.end()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(finder != image_cache_.end()); } image_cache_.erase(finder); } diff --git a/src/core/runtime/detail/projection.cc b/src/core/runtime/detail/projection.cc index 2fdb9ffd14..7c3076f1d8 100644 --- a/src/core/runtime/detail/projection.cc +++ b/src/core/runtime/detail/projection.cc @@ -297,14 +297,15 @@ struct create_affine_functor_fn { Legion::ProjectionID proj_id) { auto functor = new AffineFunctor(runtime, dims, weights, offsets); -#ifdef DEBUG_LEGATE - std::stringstream ss; - ss << "Register projection functor: functor: " << functor << ", id: " << proj_id << ", "; - spec_to_string(ss, SRC_DIM, TGT_DIM, dims, weights, offsets); - log_legate.debug() << ss.str(); -#else - log_legate.debug("Register projection functor: functor: %p, id: %d", functor, proj_id); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + std::stringstream ss; + ss << "Register projection functor: functor: " << functor << ", id: " << proj_id << ", "; + spec_to_string(ss, SRC_DIM, TGT_DIM, dims, weights, offsets); + log_legate.debug() << ss.str(); + } else { + log_legate.debug( + "Register projection functor: functor: %p, id: %d", static_cast(functor), proj_id); + } runtime->register_projection_functor(proj_id, functor, true /*silence warnings*/); const std::lock_guard lock(functor_table_lock); diff --git a/src/core/runtime/detail/provenance_manager.cc b/src/core/runtime/detail/provenance_manager.cc index 721a1da750..3f72866f4d 100644 --- a/src/core/runtime/detail/provenance_manager.cc +++ b/src/core/runtime/detail/provenance_manager.cc @@ -12,6 +12,10 @@ #include "core/runtime/detail/provenance_manager.h" +#include "legion.h" + +#include "legate_defines.h" + #include #include @@ -23,25 +27,19 @@ ProvenanceManager::ProvenanceManager() { provenance_.push_back(BOTTOM); } const std::string& ProvenanceManager::get_provenance() { -#ifdef DEBUG_LEGATE - assert(provenance_.size() > 0); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(provenance_.size() > 0); } return provenance_.back(); } void ProvenanceManager::set_provenance(const std::string& p) { -#ifdef DEBUG_LEGATE - assert(provenance_.size() > 0); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(provenance_.size() > 0); } provenance_.back() = p; } void ProvenanceManager::reset_provenance() { -#ifdef DEBUG_LEGATE - assert(provenance_.size() > 0); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(provenance_.size() > 0); } provenance_.back() = BOTTOM; } diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index da335db841..9793713416 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -111,12 +111,12 @@ Library* Runtime::find_or_create_library(const std::string& library_name, void Runtime::record_reduction_operator(int32_t type_uid, int32_t op_kind, int32_t legion_op_id) { -#ifdef DEBUG_LEGATE - log_legate.debug("Record reduction op (type_uid: %d, op_kind: %d, legion_op_id: %d)", - type_uid, - op_kind, - legion_op_id); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + log_legate.debug("Record reduction op (type_uid: %d, op_kind: %d, legion_op_id: %d)", + type_uid, + op_kind, + legion_op_id); + } auto key = std::make_pair(type_uid, op_kind); auto finder = reduction_ops_.find(key); if (finder != reduction_ops_.end()) { @@ -132,17 +132,17 @@ int32_t Runtime::find_reduction_operator(int32_t type_uid, int32_t op_kind) cons auto key = std::make_pair(type_uid, op_kind); auto finder = reduction_ops_.find(key); if (reduction_ops_.end() == finder) { -#ifdef DEBUG_LEGATE - log_legate.debug("Can't find reduction op (type_uid: %d, op_kind: %d)", type_uid, op_kind); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + log_legate.debug("Can't find reduction op (type_uid: %d, op_kind: %d)", type_uid, op_kind); + } std::stringstream ss; ss << "Reduction op " << op_kind << " does not exist for type " << type_uid; throw std::invalid_argument(std::move(ss).str()); } -#ifdef DEBUG_LEGATE - log_legate.debug( - "Found reduction op %d (type_uid: %d, op_kind: %d)", finder->second, type_uid, op_kind); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + log_legate.debug( + "Found reduction op %d (type_uid: %d, op_kind: %d)", finder->second, type_uid, op_kind); + } return finder->second; } @@ -636,11 +636,11 @@ Legion::IndexPartition Runtime::create_image_partition( Legion::FieldID func_field_id, bool is_range) { -#ifdef DEBUG_LEGATE - log_legate.debug() << "Create image partition {index_space: " << index_space - << ", func_partition: " << func_partition - << ", func_field_id: " << func_field_id << ", is_range: " << is_range << "}"; -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + log_legate.debug() << "Create image partition {index_space: " << index_space + << ", func_partition: " << func_partition + << ", func_field_id: " << func_field_id << ", is_range: " << is_range << "}"; + } if (is_range) return legion_runtime_->create_partition_by_image_range(legion_context_, index_space, @@ -894,32 +894,28 @@ void Runtime::initialize_toplevel_machine() mapping::detail::Machine machine({{mapping::TaskTarget::GPU, create_range(num_gpus)}, {mapping::TaskTarget::OMP, create_range(num_omps)}, {mapping::TaskTarget::CPU, create_range(num_cpus)}}); -#ifdef DEBUG_LEGATE - assert(machine_manager_ != nullptr); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(machine_manager_ != nullptr); } machine_manager_->push_machine(std::move(machine)); } const mapping::detail::Machine& Runtime::get_machine() const { -#ifdef DEBUG_LEGATE - assert(machine_manager_ != nullptr); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(machine_manager_ != nullptr); } return machine_manager_->get_machine(); } Legion::ProjectionID Runtime::get_projection(int32_t src_ndim, const proj::SymbolicPoint& point) { -#ifdef DEBUG_LEGATE - log_legate.debug() << "Query projection {src_ndim: " << src_ndim << ", point: " << point << "}"; -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + log_legate.debug() << "Query projection {src_ndim: " << src_ndim << ", point: " << point << "}"; + } if (is_identity(src_ndim, point)) { -#ifdef DEBUG_LEGATE - log_legate.debug() << "Identity projection {src_ndim: " << src_ndim << ", point: " << point - << "}"; -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + log_legate.debug() << "Identity projection {src_ndim: " << src_ndim << ", point: " << point + << "}"; + } return 0; } @@ -942,10 +938,10 @@ Legion::ProjectionID Runtime::get_projection(int32_t src_ndim, const proj::Symbo src_ndim, ndim, dims.data(), weights.data(), offsets.data(), proj_id); registered_projections_[key] = proj_id; -#ifdef DEBUG_LEGATE - log_legate.debug() << "Register projection " << proj_id << " {src_ndim: " << src_ndim - << ", point: " << point << "}"; -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + log_legate.debug() << "Register projection " << proj_id << " {src_ndim: " << src_ndim + << ", point: " << point << "}"; + } return proj_id; } @@ -966,26 +962,24 @@ Legion::ShardingID Runtime::get_sharding(const mapping::detail::Machine& machine auto offset = proc_range.low % proc_range.per_node_count; ShardingDesc key{proj_id, low, high, offset, proc_range.per_node_count}; -#ifdef DEBUG_LEGATE - log_legate.debug() << "Query sharding {proj_id: " << proj_id - << ", processor range: " << proc_range - << ", processor type: " << machine.preferred_target << "}"; -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + log_legate.debug() << "Query sharding {proj_id: " << proj_id + << ", processor range: " << proc_range + << ", processor type: " << machine.preferred_target << "}"; + } auto finder = registered_shardings_.find(key); if (finder != registered_shardings_.end()) { -#ifdef DEBUG_LEGATE - log_legate.debug() << "Found sharding " << finder->second; -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + log_legate.debug() << "Found sharding " << finder->second; + } return finder->second; } auto sharding_id = core_library_->get_sharding_id(next_sharding_id_++); registered_shardings_.insert({key, sharding_id}); -#ifdef DEBUG_LEGATE - log_legate.debug() << "Create sharding " << sharding_id; -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { log_legate.debug() << "Create sharding " << sharding_id; } legate_create_sharding_functor_using_projection( sharding_id, proj_id, low, high, offset, proc_range.per_node_count); @@ -1133,12 +1127,8 @@ void register_legate_core_tasks(Legion::Machine machine, task_info->add_variant(variant_id, nullptr, desc, VariantOptions{}); }; register_extract_scalar_variant(LEGATE_CPU_VARIANT); -#ifdef LEGATE_USE_CUDA - register_extract_scalar_variant(LEGATE_GPU_VARIANT); -#endif -#ifdef LEGATE_USE_OPENMP - register_extract_scalar_variant(LEGATE_OMP_VARIANT); -#endif + if (LegateDefined(LEGATE_USE_CUDA)) register_extract_scalar_variant(LEGATE_GPU_VARIANT); + if (LegateDefined(LEGATE_USE_OPENMP)) register_extract_scalar_variant(LEGATE_OMP_VARIANT); core_lib->register_task(LEGATE_CORE_EXTRACT_SCALAR_TASK_ID, std::move(task_info)); register_array_tasks(core_lib); diff --git a/src/core/runtime/detail/shard.cc b/src/core/runtime/detail/shard.cc index cfe10a6000..298fc0d424 100644 --- a/src/core/runtime/detail/shard.cc +++ b/src/core/runtime/detail/shard.cc @@ -120,9 +120,9 @@ class LegateShardingFunctor : public Legion::ShardingFunctor { auto point = proj_functor_->project_point(p, launch_space); auto shard_id = (linearize(lo, hi, point) + offset_) / per_node_count_ + start_node_id_; -#ifdef DEBUG_LEGATE - assert(start_node_id_ <= shard_id && shard_id < end_node_id_); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + assert(start_node_id_ <= shard_id && shard_id < end_node_id_); + } return shard_id; } diff --git a/src/core/runtime/library.inl b/src/core/runtime/library.inl index 41c3aff67e..bf9ace4f9b 100644 --- a/src/core/runtime/library.inl +++ b/src/core/runtime/library.inl @@ -68,10 +68,11 @@ template int32_t Library::register_reduction_operator(int32_t redop_id) { int32_t legion_redop_id = get_reduction_op_id(redop_id); -#if defined(LEGATE_USE_CUDA) && !defined(REALM_COMPILER_IS_NVCC) - extern Logger log_legate; - log_legate.error("Reduction operators must be registered in a .cu file when CUDA is enabled"); - LEGATE_ABORT; +#if !defined(REALM_COMPILER_IS_NVCC) + if (LegateDefined(LEGATE_USE_CUDA)) { + log_legate.error("Reduction operators must be registered in a .cu file when CUDA is enabled"); + LEGATE_ABORT; + } #endif perform_callback(detail::register_reduction_callback, Legion::UntypedBuffer(&legion_redop_id, sizeof(int32_t))); diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index a9f238106c..1f2b4559ae 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -36,33 +36,34 @@ extern Logger log_legate; /*static*/ void Core::parse_config(void) { -#ifndef LEGATE_USE_CUDA - const char* need_cuda = getenv("LEGATE_NEED_CUDA"); - if (need_cuda != nullptr) { - fprintf(stderr, - "Legate was run with GPUs but was not built with GPU support. " - "Please install Legate again with the \"--cuda\" flag.\n"); - exit(1); + if (!LegateDefined(LEGATE_USE_CUDA)) { + const char* need_cuda = getenv("LEGATE_NEED_CUDA"); + if (need_cuda != nullptr) { + fprintf(stderr, + "Legate was run with GPUs but was not built with GPU support. " + "Please install Legate again with the \"--cuda\" flag.\n"); + exit(1); + } } -#endif -#ifndef LEGATE_USE_OPENMP - const char* need_openmp = getenv("LEGATE_NEED_OPENMP"); - if (need_openmp != nullptr) { - fprintf(stderr, - "Legate was run with OpenMP processors but was not built with " - "OpenMP support. Please install Legate again with the \"--openmp\" flag.\n"); - exit(1); + if (!LegateDefined(LEGATE_USE_OPENMP)) { + const char* need_openmp = getenv("LEGATE_NEED_OPENMP"); + if (need_openmp != nullptr) { + fprintf(stderr, + "Legate was run on multiple nodes but was not built with networking " + "support. Please install Legate again with \"--network\".\n"); + exit(1); + } } -#endif -#ifndef LEGATE_USE_NETWORK - const char* need_network = getenv("LEGATE_NEED_NETWORK"); - if (need_network != nullptr) { - fprintf(stderr, - "Legate was run on multiple nodes but was not built with networking " - "support. Please install Legate again with \"--network\".\n"); - exit(1); + if (!LegateDefined(LEGATE_USE_NETWORK)) { + const char* need_network = getenv("LEGATE_NEED_NETWORK"); + if (need_network != nullptr) { + fprintf(stderr, + "Legate was run on multiple nodes but was not built with networking " + "support. Please install Legate again with \"--network\".\n"); + exit(1); + } } -#endif + auto parse_variable = [](const char* variable, bool& result) { const char* value = getenv(variable); if (value != nullptr && atoi(value) > 0) result = true; diff --git a/src/core/task/detail/return.cc b/src/core/task/detail/return.cc index 463c60dc5a..4f4a42bd8b 100644 --- a/src/core/task/detail/return.cc +++ b/src/core/task/detail/return.cc @@ -15,7 +15,7 @@ #include "core/runtime/detail/library.h" #include "core/utilities/machine.h" #include "core/utilities/typedefs.h" -#ifdef LEGATE_USE_CUDA +#if LegateDefined(LEGATE_USE_CUDA) #include "core/cuda/cuda_help.h" #include "core/cuda/stream_pool.h" #endif @@ -31,9 +31,7 @@ ReturnValue::ReturnValue(Legion::UntypedDeferredValue value, size_t size) /*static*/ ReturnValue ReturnValue::unpack(const void* ptr, size_t size, Memory::Kind memory_kind) { ReturnValue result(Legion::UntypedDeferredValue(size, memory_kind), size); -#ifdef DEBUG_LEGATE - assert(!result.is_device_value()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(!result.is_device_value()); } memcpy(result.ptr(), ptr, size); return result; @@ -149,19 +147,21 @@ void ReturnValues::legion_serialize(void* buffer) const // Special case with a single scalar if (return_values_.size() == 1) { auto& ret = return_values_.front(); -#ifdef LEGATE_USE_CUDA + if (ret.is_device_value()) { -#ifdef DEBUG_LEGATE - assert(Processor::get_executing_processor().kind() == Processor::Kind::TOC_PROC); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + assert(Processor::get_executing_processor().kind() == Processor::Kind::TOC_PROC); + } +#if LegateDefined(LEGATE_USE_CUDA) // TODO expose cudaMemcpyAsync() as a stub instead CHECK_CUDA(cudaMemcpyAsync(buffer, ret.ptr(), ret.size(), cudaMemcpyDeviceToHost, cuda::StreamPool::get_stream_pool().get_stream())); - } else #endif - memcpy(buffer, ret.ptr(), ret.size()); + return; + } + memcpy(buffer, ret.ptr(), ret.size()); return; } @@ -175,7 +175,7 @@ void ReturnValues::legion_serialize(void* buffer) const ptr = ptr + sizeof(uint32_t); } -#ifdef LEGATE_USE_CUDA +#if LegateDefined(LEGATE_USE_CUDA) if (Processor::get_executing_processor().kind() == Processor::Kind::TOC_PROC) { auto stream = cuda::StreamPool::get_stream_pool().get_stream(); for (auto ret : return_values_) { @@ -244,7 +244,7 @@ void ReturnValues::finalize(Legion::Context legion_context) const return; } -#ifdef LEGATE_USE_CUDA +#if LegateDefined(LEGATE_USE_CUDA) auto kind = Processor::get_executing_processor().kind(); // FIXME: We don't currently have a good way to defer the return value packing on GPUs, // as doing so would require the packing to be chained up with all preceding kernels, @@ -270,9 +270,7 @@ struct JoinReturnedException { template static void apply(LHS& lhs, RHS rhs) { -#ifdef DEBUG_LEGATE - assert(EXCLUSIVE); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(EXCLUSIVE); } if (lhs.raised() || !rhs.raised()) return; lhs = rhs; } @@ -280,9 +278,7 @@ struct JoinReturnedException { template static void fold(RHS& rhs1, RHS rhs2) { -#ifdef DEBUG_LEGATE - assert(EXCLUSIVE); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(EXCLUSIVE); } if (rhs1.raised() || !rhs2.raised()) return; rhs1 = rhs2; } diff --git a/src/core/task/detail/task_context.cc b/src/core/task/detail/task_context.cc index e77b8b2964..1cf91358ad 100644 --- a/src/core/task/detail/task_context.cc +++ b/src/core/task/detail/task_context.cc @@ -15,7 +15,7 @@ #include "core/data/detail/store.h" #include "core/utilities/deserializer.h" -#ifdef LEGATE_USE_CUDA +#if LegateDefined(LEGATE_USE_CUDA) #include "core/cuda/cuda_help.h" #endif @@ -86,18 +86,20 @@ TaskContext::TaskContext(const Legion::Task* task, arrival.arrive(); wait.wait(); } -#ifdef LEGATE_USE_CUDA - // If the task is running on a GPU and there is at least one scalar store for reduction, - // we need to wait for all the host-to-device copies for initialization to finish - if (Processor::get_executing_processor().kind() == Processor::Kind::TOC_PROC) - for (auto& reduction : reductions_) { - auto reduction_store = reduction->data(); - if (reduction_store->is_future()) { - CHECK_CUDA(cudaDeviceSynchronize()); - break; - } - } + if (LegateDefined(LEGATE_USE_CUDA)) { + // If the task is running on a GPU and there is at least one scalar store for reduction, + // we need to wait for all the host-to-device copies for initialization to finish + if (Processor::get_executing_processor().kind() == Processor::Kind::TOC_PROC) + for (auto& reduction : reductions_) { + auto reduction_store = reduction->data(); + if (reduction_store->is_future()) { +#if LegateDefined(LEGATE_USE_CUDA) + CHECK_CUDA(cudaDeviceSynchronize()); #endif + break; + } + } + } } void TaskContext::make_all_unbound_stores_empty() diff --git a/src/core/task/task.cc b/src/core/task/task.cc index 66c629ea0c..d67eade3e9 100644 --- a/src/core/task/task.cc +++ b/src/core/task/task.cc @@ -54,13 +54,13 @@ void task_wrapper(VariantImpl variant_impl, Legion::Runtime* runtime; Legion::Runtime::legion_task_preamble(args, arglen, p, task, regions, legion_context, runtime); -#ifdef LEGATE_USE_CUDA - std::stringstream ss; - ss << task_name; - if (!task->get_provenance_string().empty()) ss << " : " + task->get_provenance_string(); - std::string msg = ss.str(); - nvtx::Range auto_range(msg.c_str()); -#endif + if (LegateDefined(LEGATE_USE_CUDA)) { + std::stringstream ss; + ss << task_name; + if (!task->get_provenance_string().empty()) ss << " : " + task->get_provenance_string(); + std::string msg = ss.str(); + nvtx::Range auto_range(msg.c_str()); + } Core::show_progress(task, legion_context, runtime); diff --git a/src/core/type/detail/type_info.cc b/src/core/type/detail/type_info.cc index db478393bb..10715aee10 100644 --- a/src/core/type/detail/type_info.cc +++ b/src/core/type/detail/type_info.cc @@ -194,13 +194,13 @@ bool FixedArrayType::equal(const Type& other) const if (code != other.code) return false; auto& casted = static_cast(other); -#ifdef DEBUG_LEGATE - // Do a structural check in debug mode - return uid_ == casted.uid_ && N_ == casted.N_ && element_type_ == casted.element_type_; -#else - // Each type is uniquely identified by the uid, so it's sufficient to compare between uids - return uid_ == casted.uid_; -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + // Do a structural check in debug mode + return uid_ == casted.uid_ && N_ == casted.N_ && element_type_ == casted.element_type_; + } else { + // Each type is uniquely identified by the uid, so it's sufficient to compare between uids + return uid_ == casted.uid_; + } } StructType::StructType(int32_t uid, std::vector>&& field_types, bool align) @@ -271,18 +271,18 @@ bool StructType::equal(const Type& other) const if (code != other.code) return false; auto& casted = static_cast(other); -#ifdef DEBUG_LEGATE - // Do a structural check in debug mode - if (uid_ != casted.uid_) return false; - uint32_t nf = num_fields(); - if (nf != casted.num_fields()) return false; - for (uint32_t idx = 0; idx < nf; ++idx) - if (field_type(idx) != casted.field_type(idx)) return false; - return true; -#else - // Each type is uniquely identified by the uid, so it's sufficient to compare between uids - return uid_ == casted.uid_; -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + // Do a structural check in debug mode + if (uid_ != casted.uid_) return false; + uint32_t nf = num_fields(); + if (nf != casted.num_fields()) return false; + for (uint32_t idx = 0; idx < nf; ++idx) + if (field_type(idx) != casted.field_type(idx)) return false; + return true; + } else { + // Each type is uniquely identified by the uid, so it's sufficient to compare between uids + return uid_ == casted.uid_; + } } std::shared_ptr StructType::field_type(uint32_t field_idx) const @@ -346,13 +346,13 @@ bool ListType::equal(const Type& other) const if (code != other.code) return false; auto& casted = static_cast(other); -#ifdef DEBUG_LEGATE - // Do a structural check in debug mode - return uid_ == casted.uid_ && element_type_ == casted.element_type_; -#else - // Each type is uniquely identified by the uid, so it's sufficient to compare between uids - return uid_ == casted.uid_; -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { + // Do a structural check in debug mode + return uid_ == casted.uid_ && element_type_ == casted.element_type_; + } else { + // Each type is uniquely identified by the uid, so it's sufficient to compare between uids + return uid_ == casted.uid_; + } } std::shared_ptr string_type() diff --git a/src/core/utilities/debug.h b/src/core/utilities/debug.h index 3d72c7bd39..c5997555b1 100644 --- a/src/core/utilities/debug.h +++ b/src/core/utilities/debug.h @@ -15,7 +15,7 @@ #include "core/data/store.h" #include "core/utilities/typedefs.h" -#ifdef LEGATE_USE_CUDA +#if LegateDefined(LEGATE_USE_CUDA) #include #endif @@ -30,21 +30,6 @@ */ namespace legate { -#ifdef LEGATE_USE_CUDA - -#ifndef MAX -#define MAX(x, y) (((x) > (y)) ? (x) : (y)) -#endif - -inline bool is_device_only_ptr(const void* ptr) -{ - cudaPointerAttributes attrs; - cudaError_t res = cudaPointerGetAttributes(&attrs, ptr); - return res == cudaSuccess && attrs.type == cudaMemoryTypeDevice; -} - -#endif // LEGATE_USE_CUDA - /** * @ingroup util * @brief Converts the dense array into a string @@ -58,19 +43,33 @@ inline bool is_device_only_ptr(const void* ptr) template std::string print_dense_array(const T* base, const Point& extents, size_t strides[DIM]) { -#ifdef LEGATE_USE_CUDA - T* buf = nullptr; + T* buf = nullptr; + const auto is_device_only_ptr = [](const void* ptr) { +#if LegateDefined(LEGATE_USE_CUDA) + cudaPointerAttributes attrs; + auto res = cudaPointerGetAttributes(&attrs, ptr); + return res == cudaSuccess && attrs.type == cudaMemoryTypeDevice; +#else + static_cast(ptr); + return false; +#endif + }; + if (is_device_only_ptr(base)) { + const auto max_different_types = [](const auto& lhs, const auto& rhs) { + return lhs < rhs ? rhs : lhs; + }; size_t num_elems = 0; for (size_t dim = 0; dim < DIM; ++dim) { - num_elems = MAX(num_elems, strides[dim] * extents[dim]); + num_elems = max_different_types(num_elems, strides[dim] * extents[dim]); } - buf = new T[num_elems]; - cudaError_t res = cudaMemcpy(buf, base, num_elems * sizeof(T), cudaMemcpyDeviceToHost); + buf = new T[num_elems]; +#if LegateDefined(LEGATE_USE_CUDA) + auto res = cudaMemcpy(buf, base, num_elems * sizeof(T), cudaMemcpyDeviceToHost); assert(res == cudaSuccess); +#endif base = buf; } -#endif // LEGATE_USE_CUDA std::stringstream ss; for (int dim = 0; dim < DIM; ++dim) { if (strides[dim] == 0) continue; @@ -100,9 +99,7 @@ std::string print_dense_array(const T* base, const Point& extents, size_t s } } } while (dim >= 0); -#ifdef LEGATE_USE_CUDA - if (buf != nullptr) delete buf; -#endif // LEGATE_USE_CUDA + if (LegateDefined(LEGATE_USE_CUDA)) { delete buf; } // LEGATE_USE_CUDA return ss.str(); } diff --git a/src/core/utilities/deserializer.cc b/src/core/utilities/deserializer.cc index 8d61085e55..7f1426812b 100644 --- a/src/core/utilities/deserializer.cc +++ b/src/core/utilities/deserializer.cc @@ -77,9 +77,7 @@ std::shared_ptr TaskDeserializer::unpack_list_array() std::shared_ptr TaskDeserializer::unpack_struct_array() { auto type = unpack_type(); -#ifdef DEBUG_LEGATE - assert(type->code == Type::Code::STRUCT); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(type->code == Type::Code::STRUCT); } std::vector> fields; const auto& st_type = type->as_struct_type(); auto nullable = unpack(); @@ -219,9 +217,7 @@ std::shared_ptr TaskDeserializer::unpack_list_array() std::shared_ptr TaskDeserializer::unpack_struct_array() { auto type = unpack_type(); -#ifdef DEBUG_LEGATE - assert(type->code == Type::Code::STRUCT); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(type->code == Type::Code::STRUCT); } std::vector> fields; const auto& st_type = type->as_struct_type(); auto nullable = unpack(); @@ -288,9 +284,7 @@ CopyDeserializer::CopyDeserializer(const Legion::Copy* copy, void CopyDeserializer::next_requirement_list() { -#ifdef DEBUG_LEGATE - assert(curr_reqs_ != all_reqs_.end()); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(curr_reqs_ != all_reqs_.end()); } req_index_offset_ += curr_reqs_->get().size(); ++curr_reqs_; } @@ -304,9 +298,7 @@ void CopyDeserializer::_unpack(detail::Store& store) auto transform = unpack_transform(); -#ifdef DEBUG_LEGATE - assert(!is_future && !is_output_region); -#endif + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(!is_future && !is_output_region); } auto redop_id = unpack(); detail::RegionField rf; _unpack(rf); diff --git a/src/core/utilities/nvtx_help.h b/src/core/utilities/nvtx_help.h index e550c0670d..04271001b0 100644 --- a/src/core/utilities/nvtx_help.h +++ b/src/core/utilities/nvtx_help.h @@ -14,9 +14,13 @@ #include "legate_defines.h" -#ifdef LEGATE_USE_CUDA - +#if LegateDefined(LEGATE_USE_CUDA) #include +#else +using nvtxRangeId_t = char; +inline constexpr nvtxRangeId_t nvtxRangeStartA(const char*) noexcept { return 0; } +inline constexpr void nvtxRangeEnd(nvtxRangeId_t) noexcept {} +#endif namespace legate::nvtx { @@ -30,5 +34,3 @@ class Range { }; } // namespace legate::nvtx - -#endif diff --git a/src/env_defaults.h b/src/env_defaults.h index dfda055c5d..7c272cead4 100644 --- a/src/env_defaults.h +++ b/src/env_defaults.h @@ -30,7 +30,7 @@ #define WINDOW_SIZE_DEFAULT 1 #define WINDOW_SIZE_TEST 1 -#ifdef DEBUG_LEGATE +#ifdef LEGATE_USE_DEBUG // In debug mode, the default is always block on tasks that can throw exceptions #define MAX_PENDING_EXCEPTIONS_DEFAULT 1 #else diff --git a/src/legate.mk b/src/legate.mk index 9f643fa22b..3f4bf2e29f 100644 --- a/src/legate.mk +++ b/src/legate.mk @@ -116,9 +116,9 @@ endif ifeq ($(strip $(DEBUG)),1) -CC_FLAGS += -DDEBUG_LEGATE +CC_FLAGS += -DLEGATE_USE_DEBUG=1 -NVCC_FLAGS += -DDEBUG_LEGATE +NVCC_FLAGS += -DLEGATE_USE_DEBUG=1 endif diff --git a/src/legate_defines.h b/src/legate_defines.h index 1d3fe3c306..17830f3085 100644 --- a/src/legate_defines.h +++ b/src/legate_defines.h @@ -12,6 +12,8 @@ #pragma once +#include "legion.h" + #ifdef __cplusplus #if __cplusplus <= 199711L #ifdef _MSC_VER @@ -61,28 +63,59 @@ #ifndef LEGATE_USE_CUDA #ifdef LEGION_USE_CUDA -#define LEGATE_USE_CUDA +#define LEGATE_USE_CUDA 1 #endif #endif #ifndef LEGATE_USE_OPENMP #ifdef REALM_USE_OPENMP -#define LEGATE_USE_OPENMP +#define LEGATE_USE_OPENMP 1 #endif #endif #ifndef LEGATE_USE_NETWORK #if defined(REALM_USE_GASNET1) || defined(REALM_USE_GASNETEX) || defined(REALM_USE_MPI) || \ defined(REALM_USE_UCX) -#define LEGATE_USE_NETWORK +#define LEGATE_USE_NETWORK 1 #endif #endif #ifdef LEGION_BOUNDS_CHECKS -#define LEGATE_BOUNDS_CHECKS +#define LEGATE_BOUNDS_CHECKS 1 #endif #define LEGATE_MAX_DIM LEGION_MAX_DIM +// backwards compatibility +#if defined(DEBUG_LEGATE) && !defined(LEGATE_USE_DEBUG) +#define LEGATE_USE_DEBUG 1 +#endif + // TODO: 2022-10-04: Work around a Legion bug, by not instantiating futures on framebuffer. -#define LEGATE_NO_FUTURES_ON_FB +#define LEGATE_NO_FUTURES_ON_FB 1 + +#define LegateConcat_(x, y) x##y +#define LegateConcat(x, y) LegateConcat_(x, y) + +// Each suffix defines an additional "enabled" state for LegateDefined(LEGATE_), i.e. if you define +// +// #define LegateDefinedEnabledForm_FOO ignored, +// ^^^~~~~~~~~~~~ note suffix +// Results in +// +// #define LEGATE_HAVE_BAR FOO +// LegateDefined(LEGATE_HAVE_BAR) // now evalues to 1 +#define LegateDefinedEnabledForm_1 ignored, +#define LegateDefinedEnabledForm_ ignored, + +// arguments are either +// - (0, 1, 0, dummy) +// - (1, 0, dummy) +// this final step cherry-picks the middle +#define LegateDefinedPrivate___(ignored, val, ...) val +// the following 2 steps are needed purely for MSVC since it has a nonconforming preprocessor +// and does not expand __VA_ARGS__ in a single step +#define LegateDefinedPrivate__(args) LegateDefinedPrivate___ args +#define LegateDefinedPrivate_(...) LegateDefinedPrivate__((__VA_ARGS__)) +#define LegateDefinedPrivate(x) LegateDefinedPrivate_(x 1, 0, dummy) +#define LegateDefined(x) LegateDefinedPrivate(LegateConcat_(LegateDefinedEnabledForm_, x)) diff --git a/tests/cpp/integration/machine_scope.cc b/tests/cpp/integration/machine_scope.cc index 7414ab4d8b..3257cb1d36 100644 --- a/tests/cpp/integration/machine_scope.cc +++ b/tests/cpp/integration/machine_scope.cc @@ -37,10 +37,10 @@ struct MultiVariantTask : public legate::LegateTask { static const int32_t TASK_ID = MULTI_VARIANT; static void cpu_variant(legate::TaskContext context) { validate(context); } -#ifdef LEGATE_USE_OPENMP +#if LegateDefined(USE_OPENMP) static void omp_variant(legate::TaskContext context) { validate(context); } #endif -#ifdef LEGATE_USE_CUDA +#if LegateDefined(USE_CUDA) static void gpu_variant(legate::TaskContext context) { validate(context); } #endif }; diff --git a/tests/integration/scoping/src/library.cc b/tests/integration/scoping/src/library.cc index f067100e56..da5cd57b65 100644 --- a/tests/integration/scoping/src/library.cc +++ b/tests/integration/scoping/src/library.cc @@ -44,10 +44,10 @@ void validate(legate::TaskContext context) class MultiVariantTask : public Task { public: static void cpu_variant(legate::TaskContext context) { validate(context); } -#ifdef LEGATE_USE_OPENMP +#if LegateDefined(USE_OPENMP) static void omp_variant(legate::TaskContext context) { validate(context); } #endif -#ifdef LEGATE_USE_CUDA +#if LegateDefined(USE_CUDA) static void gpu_variant(legate::TaskContext context) { validate(context); } #endif }; From cc6332ff9959dfeb9243282743b78191eaef5ea5 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 13 Sep 2023 16:27:08 -0700 Subject: [PATCH 0219/1425] Move the timing code to C++ using Cython (#69) * Move the timing code to C++ using Cython * Address review comments * Add missing file & lines * Docstring for the C++ timing code * Use make_shared --- legate/timing/__init__.py | 2 +- legate/timing/_lib/CMakeLists.txt | 25 +++++ legate/timing/_lib/timing.pxd | 26 +++++ legate/timing/_lib/timing.pyx | 79 +++++++++++++++ legate/timing/timing.py | 137 -------------------------- legate_core_cpp.cmake | 2 + legate_core_python.cmake | 1 + src/timing/timing.cc | 59 +++++++++++ src/timing/timing.h | 79 +++++++++++++++ typings/legate/timing/_lib/timing.pyi | 31 ++++++ 10 files changed, 303 insertions(+), 138 deletions(-) create mode 100644 legate/timing/_lib/CMakeLists.txt create mode 100644 legate/timing/_lib/timing.pxd create mode 100644 legate/timing/_lib/timing.pyx delete mode 100644 legate/timing/timing.py create mode 100644 src/timing/timing.cc create mode 100644 src/timing/timing.h create mode 100644 typings/legate/timing/_lib/timing.pyi diff --git a/legate/timing/__init__.py b/legate/timing/__init__.py index 4103d6320a..07b29a9914 100644 --- a/legate/timing/__init__.py +++ b/legate/timing/__init__.py @@ -10,4 +10,4 @@ from __future__ import annotations -from legate.timing.timing import time +from legate.timing._lib.timing import time diff --git a/legate/timing/_lib/CMakeLists.txt b/legate/timing/_lib/CMakeLists.txt new file mode 100644 index 0000000000..1533af37bc --- /dev/null +++ b/legate/timing/_lib/CMakeLists.txt @@ -0,0 +1,25 @@ +#============================================================================= +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. +#============================================================================= + +set(cython_sources timing.pyx) +set(linked_libraries legate::core) + +rapids_cython_create_modules( + CXX + ASSOCIATED_TARGETS legate_core + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${linked_libraries}" +) + +foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + target_compile_features(${target} PRIVATE cxx_std_17) +endforeach() diff --git a/legate/timing/_lib/timing.pxd b/legate/timing/_lib/timing.pxd new file mode 100644 index 0000000000..c3663239c1 --- /dev/null +++ b/legate/timing/_lib/timing.pxd @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +from libcpp.utility cimport move + +import cython + + +cdef extern from "timing/timing.h" namespace "legate::timing" nogil: + cdef cppclass Time: + int value() + + cdef Time measure_microseconds() + + cdef Time measure_nanoseconds() + + +cdef class PyTime: + cdef Time _time diff --git a/legate/timing/_lib/timing.pyx b/legate/timing/_lib/timing.pyx new file mode 100644 index 0000000000..788e9b45dd --- /dev/null +++ b/legate/timing/_lib/timing.pyx @@ -0,0 +1,79 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +from libcpp.utility cimport move + +from typing import Union + +import cython + +from legate.core import get_legate_runtime + + +cdef class PyTime: + @staticmethod + def measure_microseconds() -> PyTime: + cdef PyTime result = PyTime.__new__(PyTime) + result._time = move(measure_microseconds()) + return result + + @staticmethod + def measure_nanoseconds() -> PyTime: + cdef PyTime result = PyTime.__new__(PyTime) + result._time = move(measure_nanoseconds()) + return result + + def value(self) -> int: + return self._time.value() + + def __int__(self) -> int: + return self.value() + + def __str__(self) -> str: + return str(self.value()) + + def __float__(self) -> float: + return float(self.value()) + + def __add__(self, rhs: Union[int, PyTime]) -> int: + return self.value() + int(rhs) + + def __radd__(self, lhs: Union[int, PyTime]) -> int: + return int(lhs) + self.value() + + def __sub__(self, rhs: Union[int, PyTime]) -> int: + return self.value() - int(rhs) + + def __rsub__(self, lhs: Union[int, PyTime]) -> int: + return int(lhs) - self.value() + + def __mul__(self, rhs: Union[int, PyTime]) -> int: + return self.value() * int(rhs) + + def __rmul__(self, lhs: Union[int, PyTime]) -> int: + return int(lhs) * self.value() + + def __div__(self, rhs: Union[int, PyTime]) -> float: + return self.value() / int(rhs) + + def __rdiv__(self, lhs: Union[int, PyTime]) -> float: + return int(lhs) / self.value() + + +def time(units: str = "us") -> PyTime: + # Issue a Legion execution fence and then perform a timing operation + # immediately after it + get_legate_runtime().issue_execution_fence() + if units == "us": + return PyTime.measure_microseconds() + elif units == "ns": + return PyTime.measure_nanoseconds() + else: + raise ValueError("time units must be 'us' or 'ns'") diff --git a/legate/timing/timing.py b/legate/timing/timing.py deleted file mode 100644 index 73078e5bc7..0000000000 --- a/legate/timing/timing.py +++ /dev/null @@ -1,137 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. -# All rights reserved. -# SPDX-License-Identifier: LicenseRef-NvidiaProprietary -# -# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual -# property and proprietary rights in and to this material, related -# documentation and any modifications thereto. Any use, reproduction, -# disclosure or distribution of this material and related documentation -# without an express license agreement from NVIDIA CORPORATION or -# its affiliates is strictly prohibited. - -from __future__ import annotations - -import struct -from typing import Any, Type, Union - -import legate.core.types as ty - -from ..core import Future, get_legion_context, get_legion_runtime, legion - - -class TimingRuntime: - def __init__(self) -> None: - self.runtime = get_legion_runtime() - self.context = get_legion_context() - - def issue_execution_fence(self) -> None: - legion.legion_runtime_issue_execution_fence(self.runtime, self.context) - - def measure_seconds(self) -> Future: - return Future( - legion.legion_issue_timing_op_seconds(self.runtime, self.context) - ) - - def measure_microseconds(self) -> Future: - return Future( - legion.legion_issue_timing_op_microseconds( - self.runtime, self.context - ) - ) - - def measure_nanoseconds(self) -> Future: - return Future( - legion.legion_issue_timing_op_nanoseconds( - self.runtime, self.context - ) - ) - - -class Time: - def __init__(self, future: Future, dtype: ty.Dtype) -> None: - self.future = future - self.dtype = dtype - self.value: Union[int, float, None] = None - - @property - def type(self) -> ty.Dtype: - return self.dtype - - @property - def kind(self) -> Type[Future]: - return Future - - @property - def storage(self) -> Union[int, float, None]: - return self.value - - @property - def stores(self) -> list[Any]: - return [None, self] - - @property - def region(self) -> None: - raise RuntimeError() - - def __int__(self) -> int: - return int(self.get_value()) - - def __str__(self) -> str: - return str(self.get_value()) - - def __float__(self) -> float: - return float(self.get_value()) - - def __add__(self, rhs: Union[int, float]) -> Union[int, float]: - return self.get_value() + rhs - - def __radd__(self, lhs: Union[int, float]) -> Union[int, float]: - return lhs + self.get_value() - - def __sub__(self, rhs: Union[int, float]) -> Union[int, float]: - return self.get_value() - rhs - - def __rsub__(self, lhs: Union[int, float]) -> Union[int, float]: - return lhs - self.get_value() - - def __mul__(self, rhs: Union[int, float]) -> Union[int, float]: - return self.get_value() * rhs - - def __rmul__(self, lhs: Union[int, float]) -> Union[int, float]: - return lhs * self.get_value() - - def __div__(self, rhs: Union[int, float]) -> float: - return self.get_value() / rhs - - def __rdiv__(self, lhs: Union[int, float]) -> float: - return lhs / self.get_value() - - def get_value(self) -> Union[int, float]: - if self.value is None: - if self.dtype == ty.int64: - self.value = struct.unpack_from( - "q", self.future.get_buffer(8) - )[0] - else: - assert self.dtype == ty.float64 - self.value = struct.unpack_from( - "d", self.future.get_buffer(8) - )[0] - return self.value - - -_timing = TimingRuntime() - - -def time(units: str = "us") -> Time: - # Issue a Legion execution fence and then perform a timing operation - # immediately after it - _timing.issue_execution_fence() - if units == "s": - return Time(_timing.measure_seconds(), ty.float64) - elif units == "us": - return Time(_timing.measure_microseconds(), ty.int64) - elif units == "ns": - return Time(_timing.measure_nanoseconds(), ty.int64) - else: - raise ValueError('time units must be one of "s", "us", or "ns"') diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index b737e3e443..00c3a981a9 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -256,6 +256,7 @@ list(APPEND legate_core_SOURCES src/core/utilities/machine.cc src/core/utilities/linearize.cc src/core/utilities/detail/buffer_builder.cc + src/timing/timing.cc ) if(Legion_NETWORKS) @@ -390,6 +391,7 @@ if (legate_core_BUILD_DOCS) src/core/runtime/tracker.h src/core/utilities/debug.h src/core/utilities/dispatch.h + src/timing/timing.h # main page src/legate.h ) diff --git a/legate_core_python.cmake b/legate_core_python.cmake index 222f246208..f050bebe90 100644 --- a/legate_core_python.cmake +++ b/legate_core_python.cmake @@ -58,6 +58,7 @@ include(rapids-cython) rapids_cython_init() add_subdirectory(legate/core/_lib) +add_subdirectory(legate/timing/_lib) set(cython_lib_dir "../../") diff --git a/src/timing/timing.cc b/src/timing/timing.cc new file mode 100644 index 0000000000..2232e12c8c --- /dev/null +++ b/src/timing/timing.cc @@ -0,0 +1,59 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include "timing/timing.h" + +#include + +#include "legion.h" + +namespace legate::timing { + +class Time::Impl { + public: + explicit Impl(const Legion::Future& future) : future_{future} {} + + public: + int64_t value() + { + if (!value_) { value_ = future_.get_result(); } + return *value_; + } + + private: + Legion::Future future_; + std::optional value_{std::nullopt}; +}; + +int64_t Time::value() const { return impl_->value(); } + +Time measure_microseconds() +{ + auto runtime = Legion::Runtime::get_runtime(); + auto context = Legion::Runtime::get_context(); + + auto future = runtime->get_current_time_in_microseconds(context); + + return Time{std::make_shared(future)}; +} + +Time measure_nanoseconds() +{ + auto runtime = Legion::Runtime::get_runtime(); + auto context = Legion::Runtime::get_context(); + + auto future = runtime->get_current_time_in_nanoseconds(context); + + return Time{std::make_shared(future)}; +} + +} // namespace legate::timing diff --git a/src/timing/timing.h b/src/timing/timing.h new file mode 100644 index 0000000000..09985c51e3 --- /dev/null +++ b/src/timing/timing.h @@ -0,0 +1,79 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#pragma once + +#include +#include + +#include + +/** + * @file + * @brief Class definition legate::timing::Time + */ + +namespace legate::timing { + +/** + * @ingroup util + * @brief Deferred timestamp class + */ +class Time { + public: + /** + * @brief Returns the timestamp value in this `Time` object + * + * Blocks on all Legate operations preceding the call that generated this `Time` object. + * + * @return A timestamp value + */ + int64_t value() const; + + public: + Time() = default; + + private: + class Impl; + explicit Time(std::shared_ptr impl) : impl_{std::move(impl)} {} + std::shared_ptr impl_{nullptr}; + + private: + friend Time measure_microseconds(); + friend Time measure_nanoseconds(); +}; + +/** + * @ingroup util + * @brief Returns a timestamp at the resolution of microseconds + * + * The returned timestamp indicates the time at which all preceding Legate operations finish. This + * timestamp generation is a non-blocking operation, and the blocking happens when the value wrapped + * within the returned `Time` object is retrieved. + * + * @return A `Time` object + */ +Time measure_microseconds(); + +/** + * @ingroup util + * @brief Returns a timestamp at the resolution of nanoseconds + * + * The returned timestamp indicates the time at which all preceding Legate operations finish. This + * timestamp generation is a non-blocking operation, and the blocking happens when the value wrapped + * within the returned `Time` object is retrieved. + * + * @return A `Time` object + */ +Time measure_nanoseconds(); + +} // namespace legate::timing diff --git a/typings/legate/timing/_lib/timing.pyi b/typings/legate/timing/_lib/timing.pyi new file mode 100644 index 0000000000..ddd5173ffe --- /dev/null +++ b/typings/legate/timing/_lib/timing.pyi @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NvidiaProprietary +# +# NVIDIA CORPORATION, its affiliates and licensors retain all intellectual +# property and proprietary rights in and to this material, related +# documentation and any modifications thereto. Any use, reproduction, +# disclosure or distribution of this material and related documentation +# without an express license agreement from NVIDIA CORPORATION or +# its affiliates is strictly prohibited. + +from typing import Union + +class PyTime: + @staticmethod + def measure_microseconds() -> PyTime: ... + @staticmethod + def measure_nanoseconds() -> PyTime: ... + def value(self) -> int: ... + def __int__(self) -> int: ... + def __str__(self) -> str: ... + def __float__(self) -> float: ... + def __add__(self, rhs: Union[int, PyTime]) -> int: ... + def __radd__(self, lhs: Union[int, PyTime]) -> int: ... + def __sub__(self, rhs: Union[int, PyTime]) -> int: ... + def __rsub__(self, lhs: Union[int, PyTime]) -> int: ... + def __mul__(self, rhs: Union[int, PyTime]) -> int: ... + def __rmul__(self, lhs: Union[int, PyTime]) -> int: ... + def __div__(self, rhs: Union[int, PyTime]) -> float: ... + def __rdiv__(self, lhs: Union[int, PyTime]) -> float: ... + +def time(units: str) -> PyTime: ... From 4895873d3a36d99f02b8fd5e02e1b85181df1288 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 15 Sep 2023 23:37:47 -0700 Subject: [PATCH 0220/1425] Exclude mapping tags from the coalescing analysis (#70) * Exclude mapping tags from the coalescing analysis * Materialize mapping tags when generating requirements --- src/core/data/detail/logical_store.cc | 2 +- src/core/operation/detail/copy_launcher.cc | 2 +- src/core/operation/detail/fill_launcher.cc | 4 +- src/core/operation/detail/launcher_arg.cc | 3 +- src/core/operation/detail/operation.cc | 2 +- src/core/operation/detail/projection.cc | 65 ++++++++++++++-------- src/core/operation/detail/projection.h | 36 ++++++++---- src/core/operation/detail/req_analyzer.cc | 17 ++++-- src/core/operation/detail/req_analyzer.h | 11 +++- 9 files changed, 94 insertions(+), 48 deletions(-) diff --git a/src/core/data/detail/logical_store.cc b/src/core/data/detail/logical_store.cc index 97581e09da..01c138ab15 100644 --- a/src/core/data/detail/logical_store.cc +++ b/src/core/data/detail/logical_store.cc @@ -683,7 +683,7 @@ std::unique_ptr LogicalStore::to_launcher_arg(const Variable* variab auto partition = strategy[variable]; auto store_partition = create_partition(partition); auto proj_info = store_partition->create_projection_info(launch_domain); - proj_info->tag = strategy.is_key_partition(variable) ? LEGATE_CORE_KEY_STORE_TAG : 0; + proj_info->is_key = strategy.is_key_partition(variable); proj_info->redop = redop; if (privilege == REDUCE && store_partition->is_disjoint_for(launch_domain)) { diff --git a/src/core/operation/detail/copy_launcher.cc b/src/core/operation/detail/copy_launcher.cc index 910697c5fb..ef62993838 100644 --- a/src/core/operation/detail/copy_launcher.cc +++ b/src/core/operation/detail/copy_launcher.cc @@ -97,7 +97,7 @@ void CopyLauncher::add_store(std::vector& args, uint32_t req_idx = args.size(); auto region_field = store->get_region_field(); auto field_id = region_field->field_id(); - if (LEGATE_CORE_KEY_STORE_TAG == proj_info->tag) key_proj_id_ = proj_info->proj_id; + if (proj_info->is_key) key_proj_id_ = proj_info->proj_id; args.push_back(new CopyArg(req_idx, store, field_id, privilege, std::move(proj_info))); } diff --git a/src/core/operation/detail/fill_launcher.cc b/src/core/operation/detail/fill_launcher.cc index 2ec2f3184b..ef760b8ab2 100644 --- a/src/core/operation/detail/fill_launcher.cc +++ b/src/core/operation/detail/fill_launcher.cc @@ -48,7 +48,7 @@ void FillLauncher::launch(const Legion::Domain& launch_domain, lhs_proj.proj_id, Legion::Predicate::TRUE_PRED, runtime->core_library()->get_mapper_id(), - lhs_proj.tag, + lhs_proj.is_key ? LEGATE_CORE_KEY_STORE_TAG : 0, mapper_arg.to_legion_buffer(), provenance.c_str()); @@ -74,7 +74,7 @@ void FillLauncher::launch_single(LogicalStore* lhs, future_value, Legion::Predicate::TRUE_PRED, runtime->core_library()->get_mapper_id(), - lhs_proj.tag, + lhs_proj.is_key ? LEGATE_CORE_KEY_STORE_TAG : 0, mapper_arg.to_legion_buffer(), provenance.c_str()); diff --git a/src/core/operation/detail/launcher_arg.cc b/src/core/operation/detail/launcher_arg.cc index c3106b9acd..29c3c4dfe9 100644 --- a/src/core/operation/detail/launcher_arg.cc +++ b/src/core/operation/detail/launcher_arg.cc @@ -50,8 +50,7 @@ void RegionFieldArg::analyze(StoreAnalyzer& analyzer) std::optional RegionFieldArg::get_key_proj_id() const { - return LEGATE_CORE_KEY_STORE_TAG == proj_info_->tag ? std::make_optional(proj_info_->proj_id) - : std::nullopt; + return proj_info_->is_key ? std::make_optional(proj_info_->proj_id) : std::nullopt; } void RegionFieldArg::perform_invalidations() const diff --git a/src/core/operation/detail/operation.cc b/src/core/operation/detail/operation.cc index 08936be041..a0121f3836 100644 --- a/src/core/operation/detail/operation.cc +++ b/src/core/operation/detail/operation.cc @@ -64,7 +64,7 @@ std::unique_ptr Operation::create_projection_info(const Strategy { auto store_partition = arg.store->create_partition(strategy[arg.variable]); auto proj_info = store_partition->create_projection_info(launch_domain); - proj_info->tag = strategy.is_key_partition(arg.variable) ? LEGATE_CORE_KEY_STORE_TAG : 0; + proj_info->is_key = strategy.is_key_partition(arg.variable); return proj_info; } diff --git a/src/core/operation/detail/projection.cc b/src/core/operation/detail/projection.cc index 57844a3d58..f2d85cd0b5 100644 --- a/src/core/operation/detail/projection.cc +++ b/src/core/operation/detail/projection.cc @@ -17,34 +17,36 @@ namespace legate::detail { template <> -void ProjectionInfo::populate_requirement(Legion::RegionRequirement& requirement, - const Legion::LogicalRegion& region, - const std::vector& fields, - Legion::PrivilegeMode privilege) const +void BaseProjectionInfo::populate_requirement(Legion::RegionRequirement& requirement, + const Legion::LogicalRegion& region, + const std::vector& fields, + Legion::PrivilegeMode privilege, + bool is_key) const { auto parent = Runtime::get_runtime()->find_parent_region(region); - + auto tag = is_key ? LEGATE_CORE_KEY_STORE_TAG : 0; if (LEGION_REDUCE == privilege) { new (&requirement) Legion::RegionRequirement(region, redop, LEGION_EXCLUSIVE, parent, tag); } else { new (&requirement) Legion::RegionRequirement(region, privilege, LEGION_EXCLUSIVE, parent, tag); } - requirement.add_fields(fields).add_flags(flags); + requirement.add_fields(fields); } template <> -void ProjectionInfo::populate_requirement(Legion::RegionRequirement& requirement, - const Legion::LogicalRegion& region, - const std::vector& fields, - Legion::PrivilegeMode privilege) const +void BaseProjectionInfo::populate_requirement(Legion::RegionRequirement& requirement, + const Legion::LogicalRegion& region, + const std::vector& fields, + Legion::PrivilegeMode privilege, + bool is_key) const { if (Legion::LogicalPartition::NO_PART == partition) { - populate_requirement(requirement, region, fields, privilege); + populate_requirement(requirement, region, fields, privilege, is_key); return; } auto parent = Runtime::get_runtime()->find_parent_region(region); - + auto tag = is_key ? LEGATE_CORE_KEY_STORE_TAG : 0; if (LEGION_REDUCE == privilege) { new (&requirement) Legion::RegionRequirement(partition, proj_id, redop, LEGION_EXCLUSIVE, parent, tag); @@ -52,15 +54,16 @@ void ProjectionInfo::populate_requirement(Legion::RegionRequirement& requ new (&requirement) Legion::RegionRequirement(partition, proj_id, privilege, LEGION_EXCLUSIVE, parent, tag); } - requirement.add_fields(fields).add_flags(flags); + requirement.add_fields(fields); } -ProjectionInfo::ProjectionInfo(Legion::LogicalPartition _partition, Legion::ProjectionID _proj_id) +BaseProjectionInfo::BaseProjectionInfo(Legion::LogicalPartition _partition, + Legion::ProjectionID _proj_id) : partition(_partition), proj_id(_proj_id) { } -bool ProjectionInfo::operator<(const ProjectionInfo& other) const +bool BaseProjectionInfo::operator<(const BaseProjectionInfo& other) const { if (partition < other.partition) return true; @@ -72,18 +75,36 @@ bool ProjectionInfo::operator<(const ProjectionInfo& other) const return false; if (redop < other.redop) return true; - else if (redop > other.redop) - return false; - if (tag < other.tag) - return true; else return false; } -bool ProjectionInfo::operator==(const ProjectionInfo& other) const +bool BaseProjectionInfo::operator==(const BaseProjectionInfo& other) const +{ + return partition == other.partition && proj_id == other.proj_id && redop == other.redop; +} + +ProjectionInfo::ProjectionInfo(Legion::LogicalPartition partition, Legion::ProjectionID proj_id) + : BaseProjectionInfo(partition, proj_id) +{ +} + +template <> +void ProjectionInfo::populate_requirement(Legion::RegionRequirement& requirement, + const Legion::LogicalRegion& region, + const std::vector& fields, + Legion::PrivilegeMode privilege) const +{ + populate_requirement(requirement, region, fields, privilege, is_key); +} + +template <> +void ProjectionInfo::populate_requirement(Legion::RegionRequirement& requirement, + const Legion::LogicalRegion& region, + const std::vector& fields, + Legion::PrivilegeMode privilege) const { - return partition == other.partition && proj_id == other.proj_id && redop == other.redop && - tag == other.tag && flags == other.flags; + populate_requirement(requirement, region, fields, privilege, is_key); } } // namespace legate::detail diff --git a/src/core/operation/detail/projection.h b/src/core/operation/detail/projection.h index 5b41146b41..38e27ff533 100644 --- a/src/core/operation/detail/projection.h +++ b/src/core/operation/detail/projection.h @@ -17,15 +17,15 @@ namespace legate::detail { -struct ProjectionInfo { - ProjectionInfo() {} - ProjectionInfo(Legion::LogicalPartition partition, Legion::ProjectionID proj_id); +struct BaseProjectionInfo { + BaseProjectionInfo() = default; + BaseProjectionInfo(Legion::LogicalPartition partition, Legion::ProjectionID proj_id); - ProjectionInfo(const ProjectionInfo&) = default; - ProjectionInfo& operator=(const ProjectionInfo&) = default; + BaseProjectionInfo(const BaseProjectionInfo&) = default; + BaseProjectionInfo& operator=(const BaseProjectionInfo&) = default; - bool operator<(const ProjectionInfo& other) const; - bool operator==(const ProjectionInfo& other) const; + bool operator<(const BaseProjectionInfo& other) const; + bool operator==(const BaseProjectionInfo& other) const; // TODO: Ideally we want this method to return a requirement, instead of taking an inout argument. // We go with an inout parameter for now, as RegionRequirement doesn't have a move @@ -34,15 +34,31 @@ struct ProjectionInfo { void populate_requirement(Legion::RegionRequirement& requirement, const Legion::LogicalRegion& region, const std::vector& fields, - Legion::PrivilegeMode privilege) const; + Legion::PrivilegeMode privilege, + bool is_key) const; void set_reduction_op(Legion::ReductionOpID _redop) { redop = _redop; } Legion::LogicalPartition partition{Legion::LogicalPartition::NO_PART}; Legion::ProjectionID proj_id{0}; Legion::ReductionOpID redop{-1}; - Legion::MappingTagID tag{0}; - Legion::RegionFlags flags{LEGION_NO_FLAG}; +}; + +struct ProjectionInfo : public BaseProjectionInfo { + ProjectionInfo() = default; + ProjectionInfo(Legion::LogicalPartition partition, Legion::ProjectionID proj_id); + + ProjectionInfo(const ProjectionInfo&) = default; + ProjectionInfo& operator=(const ProjectionInfo&) = default; + + using BaseProjectionInfo::populate_requirement; + template + void populate_requirement(Legion::RegionRequirement& requirement, + const Legion::LogicalRegion& region, + const std::vector& fields, + Legion::PrivilegeMode privilege) const; + + bool is_key{false}; }; } // namespace legate::detail diff --git a/src/core/operation/detail/req_analyzer.cc b/src/core/operation/detail/req_analyzer.cc index a4094ddd6a..c781c97c06 100644 --- a/src/core/operation/detail/req_analyzer.cc +++ b/src/core/operation/detail/req_analyzer.cc @@ -26,6 +26,7 @@ void ProjectionSet::insert(Legion::PrivilegeMode new_privilege, const Projection else if (!(privilege == new_privilege || privilege == NO_ACCESS || new_privilege == NO_ACCESS)) privilege = LEGION_READ_WRITE; proj_infos.emplace(proj_info); + is_key = is_key || proj_info.is_key; if (privilege != LEGION_READ_ONLY && privilege != NO_ACCESS && proj_infos.size() > 1) { log_legate.error("Interfering requirements are found"); @@ -49,8 +50,8 @@ uint32_t FieldSet::num_requirements() const { return static_cast(coale uint32_t FieldSet::get_requirement_index(Legion::PrivilegeMode privilege, const ProjectionInfo& proj_info) const { - auto finder = req_indices_.find(Key(privilege, proj_info)); - if (req_indices_.end() == finder) finder = req_indices_.find(Key(LEGION_READ_WRITE, proj_info)); + auto finder = req_indices_.find(Key{privilege, proj_info}); + if (req_indices_.end() == finder) finder = req_indices_.find(Key{LEGION_READ_WRITE, proj_info}); if (LegateDefined(LEGATE_USE_DEBUG)) { assert(finder != req_indices_.end()); } return finder->second; } @@ -58,8 +59,11 @@ uint32_t FieldSet::get_requirement_index(Legion::PrivilegeMode privilege, void FieldSet::coalesce() { for (const auto& [field_id, proj_set] : field_projs_) { - for (const auto& proj_info : proj_set.proj_infos) - coalesced_[Key(proj_set.privilege, proj_info)].push_back(field_id); + for (const auto& proj_info : proj_set.proj_infos) { + auto& [fields, is_key] = coalesced_[Key{proj_set.privilege, proj_info}]; + fields.push_back(field_id); + is_key = is_key || proj_set.is_key; + } } uint32_t idx = 0; for (const auto& [key, _] : coalesced_) req_indices_[key] = idx++; @@ -79,12 +83,13 @@ constexpr bool is_single = false; template void FieldSet::populate_launcher(Launcher& task, const Legion::LogicalRegion& region) const { - for (auto& [key, fields] : coalesced_) { + for (auto& [key, entry] : coalesced_) { + auto& [fields, is_key] = entry; auto& [privilege, proj_info] = key; task.region_requirements.push_back(Legion::RegionRequirement()); auto& requirement = task.region_requirements.back(); proj_info.template populate_requirement>( - requirement, region, fields, privilege); + requirement, region, fields, privilege, is_key); } } diff --git a/src/core/operation/detail/req_analyzer.h b/src/core/operation/detail/req_analyzer.h index 53a33a2903..4a66734f3b 100644 --- a/src/core/operation/detail/req_analyzer.h +++ b/src/core/operation/detail/req_analyzer.h @@ -24,12 +24,13 @@ class ProjectionSet { public: Legion::PrivilegeMode privilege; - std::set proj_infos; + std::set proj_infos; + bool is_key; }; class FieldSet { public: - using Key = std::pair; + using Key = std::pair; public: void insert(Legion::FieldID field_id, @@ -45,7 +46,11 @@ class FieldSet { void populate_launcher(Launcher& task, const Legion::LogicalRegion& region) const; private: - std::map> coalesced_; + struct Entry { + std::vector fields; + bool is_key; + }; + std::map coalesced_; std::map req_indices_; private: From 28ff3eadb59304ed7eaaa69ec5a2df39354f66d9 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Sat, 16 Sep 2023 00:25:47 -0700 Subject: [PATCH 0221/1425] Test case with aliased store arguments (#71) --- tests/cpp/integration/req_analyzer.cc | 60 +++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 tests/cpp/integration/req_analyzer.cc diff --git a/tests/cpp/integration/req_analyzer.cc b/tests/cpp/integration/req_analyzer.cc new file mode 100644 index 0000000000..a2606bba4b --- /dev/null +++ b/tests/cpp/integration/req_analyzer.cc @@ -0,0 +1,60 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include + +#include "legate.h" + +namespace req_analyzer { + +static const char* library_name = "test_req_analyzer"; + +enum TaskIDs { + TESTER = 0, +}; + +struct Tester : public legate::LegateTask { + static const int32_t TASK_ID = TESTER; + static void cpu_variant(legate::TaskContext context) {} +}; + +void prepare() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + Tester::register_variants(context); +} + +void test_isomorphic_transformed_stores() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + auto store = runtime->create_store({10}, legate::int64()); + runtime->issue_fill(store, legate::Scalar(int64_t{0})); + + // Create aliased stores that are semantically equivalent + auto promoted1 = store.promote(1, 5); + auto promoted2 = store.promote(1, 5); + auto task = runtime->create_task(context, TESTER); + task.add_input(promoted1); + task.add_output(promoted2); + runtime->submit(std::move(task)); +} + +TEST(ReqAnalyzer, IsomorphicTransformedStores) +{ + legate::Core::perform_registration(); + test_isomorphic_transformed_stores(); +} + +} // namespace req_analyzer From 9d8b1ac87be5e123e66894279f03cd99241ea354 Mon Sep 17 00:00:00 2001 From: Jacob Faibussowitsch Date: Mon, 18 Sep 2023 15:02:10 -0400 Subject: [PATCH 0222/1425] Fix 'warning: format string is not a string literal (potentially insecure) [-Wformat-security]' (#72) --- src/core/runtime/detail/runtime.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index 9793713416..bc5379ded2 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -1250,7 +1250,7 @@ void try_set_property(Runtime& runtime, { auto value = var.value(); if (value < 0) { - log_legate.error(error_msg.c_str()); + log_legate.error("%s", error_msg.c_str()); LEGATE_ABORT; } auto config = runtime.get_module_config(module_name); @@ -1258,12 +1258,12 @@ void try_set_property(Runtime& runtime, // If the variable doesn't have a value, we don't care if the module is nonexistent if (!var.has_value()) { return; } const std::string msg = error_msg + " (the " + module_name + " module is not available)"; - log_legate.error(msg.c_str()); + log_legate.error("%s", msg.c_str()); LEGATE_ABORT; } auto success = config->set_property(property_name, value); if (!success) { - log_legate.error(error_msg.c_str()); + log_legate.error("%s", error_msg.c_str()); LEGATE_ABORT; } } From 70b69ad7e072e450afae17b632d396227b49d2d6 Mon Sep 17 00:00:00 2001 From: Parag Kulkarni Date: Wed, 20 Sep 2023 20:09:23 +0530 Subject: [PATCH 0223/1425] Address syntax error. (#68) --- .github/workflows/merge-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/merge-ci.yml b/.github/workflows/merge-ci.yml index ea06eb5fd7..21024f7c89 100644 --- a/.github/workflows/merge-ci.yml +++ b/.github/workflows/merge-ci.yml @@ -7,7 +7,7 @@ on: jobs: merge: - if: github.repository_owner == 'nv-legate' + if: ${{ github.repository_owner == 'nv-legate' }} runs-on: ubuntu-latest steps: - name: Legate Core Internal repository From 0a986aaec82905c9ddfebae79fb3fc72aa86f015 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Thu, 21 Sep 2023 20:08:36 -0700 Subject: [PATCH 0224/1425] Fill and copy for scalar stores (#75) * Privilege checks for stores * Support fills and copies for scalar stores * Start deduplicating futures from scalar stores * Test cases for scalar fills and copies * Build flags for tests --- src/core/data/detail/store.cc | 13 +++++++- src/core/data/detail/store.h | 2 ++ src/core/data/store.cc | 4 +++ src/core/data/store.h | 2 ++ src/core/data/store.inl | 30 +++++++++++++---- src/core/operation/detail/copy.cc | 16 ++++++++-- src/core/operation/detail/fill.cc | 10 ++++-- src/core/operation/detail/req_analyzer.cc | 8 +++-- src/core/operation/detail/req_analyzer.h | 1 + tests/cpp/CMakeLists.txt | 9 ++++++ tests/cpp/integration/copy_normal.cc | 5 +-- tests/cpp/integration/fill.cc | 39 +++++++++++++---------- 12 files changed, 107 insertions(+), 32 deletions(-) diff --git a/src/core/data/detail/store.cc b/src/core/data/detail/store.cc index 118d5fb052..0d8678ea26 100644 --- a/src/core/data/detail/store.cc +++ b/src/core/data/detail/store.cc @@ -220,7 +220,8 @@ Store::Store(int32_t dim, redop_id_(redop_id), future_(future), transform_(std::move(transform)), - readable_(true) + readable_(future.valid()), + writable_(!future.is_read_only()) { } @@ -390,6 +391,16 @@ void Store::check_valid_binding(bool bind_buffer) const } } +void Store::check_write_access() const +{ + if (!writable_) { throw std::invalid_argument("Store isn't writable"); } +} + +void Store::check_reduction_access() const +{ + if (!(writable_ || reducible_)) { throw std::invalid_argument("Store isn't reducible"); } +} + Legion::DomainAffineTransform Store::get_inverse_transform() const { return transform_->inverse_transform(dim_); diff --git a/src/core/data/detail/store.h b/src/core/data/detail/store.h index 9ccd05cd15..ca1f68cf56 100644 --- a/src/core/data/detail/store.h +++ b/src/core/data/detail/store.h @@ -216,6 +216,8 @@ class Store { void check_buffer_dimension(const int32_t dim) const; void check_shape_dimension(const int32_t dim) const; void check_valid_binding(bool bind_buffer) const; + void check_write_access() const; + void check_reduction_access() const; Legion::DomainAffineTransform get_inverse_transform() const; private: diff --git a/src/core/data/store.cc b/src/core/data/store.cc index 4adcd6c95c..c4aabb2465 100644 --- a/src/core/data/store.cc +++ b/src/core/data/store.cc @@ -54,6 +54,10 @@ void Store::check_shape_dimension(const int32_t dim) const { impl_->check_shape_ void Store::check_valid_binding(bool bind_buffer) const { impl_->check_valid_binding(bind_buffer); } +void Store::check_write_access() const { impl_->check_write_access(); } + +void Store::check_reduction_access() const { impl_->check_reduction_access(); } + Legion::DomainAffineTransform Store::get_inverse_transform() const { return impl_->get_inverse_transform(); diff --git a/src/core/data/store.h b/src/core/data/store.h index 04debcb7fb..6390ec55f8 100644 --- a/src/core/data/store.h +++ b/src/core/data/store.h @@ -337,6 +337,8 @@ class Store { void check_buffer_dimension(const int32_t dim) const; void check_shape_dimension(const int32_t dim) const; void check_valid_binding(bool bind_buffer) const; + void check_write_access() const; + void check_reduction_access() const; template void check_accessor_type() const; Legion::DomainAffineTransform get_inverse_transform() const; diff --git a/src/core/data/store.inl b/src/core/data/store.inl index 53014a09b4..5cf0a4deab 100644 --- a/src/core/data/store.inl +++ b/src/core/data/store.inl @@ -71,7 +71,10 @@ AccessorWO Store::write_accessor() const check_accessor_type(); } - if (is_future()) { return AccessorWO(get_buffer(), shape()); } + if (is_future()) { + check_write_access(); + return AccessorWO(get_buffer(), shape()); + } return create_field_accessor, DIM>(shape()); } @@ -84,7 +87,10 @@ AccessorRW Store::read_write_accessor() const check_accessor_type(); } - if (is_future()) { return AccessorRW(get_buffer(), shape()); } + if (is_future()) { + check_write_access(); + return AccessorRW(get_buffer(), shape()); + } return create_field_accessor, DIM>(shape()); } @@ -98,7 +104,10 @@ AccessorRD Store::reduce_accessor() const check_accessor_type(); } - if (is_future()) { return AccessorRD(get_buffer(), shape()); } + if (is_future()) { + check_reduction_access(); + return AccessorRD(get_buffer(), shape()); + } return create_reduction_accessor, DIM>(shape()); } @@ -130,7 +139,10 @@ AccessorWO Store::write_accessor(const Rect& bounds) const check_accessor_type(); } - if (is_future()) { return AccessorWO(get_buffer(), bounds); } + if (is_future()) { + check_write_access(); + return AccessorWO(get_buffer(), bounds); + } return create_field_accessor, DIM>(bounds); } @@ -143,7 +155,10 @@ AccessorRW Store::read_write_accessor(const Rect& bounds) const check_accessor_type(); } - if (is_future()) { return AccessorRW(get_buffer(), bounds); } + if (is_future()) { + check_write_access(); + return AccessorRW(get_buffer(), bounds); + } return create_field_accessor, DIM>(bounds); } @@ -157,7 +172,10 @@ AccessorRD Store::reduce_accessor(const Rect& bounds) c check_accessor_type(); } - if (is_future()) { return AccessorRD(get_buffer(), bounds); } + if (is_future()) { + check_reduction_access(); + return AccessorRD(get_buffer(), bounds); + } return create_reduction_accessor, DIM>(bounds); } diff --git a/src/core/operation/detail/copy.cc b/src/core/operation/detail/copy.cc index e755703d3c..2b68e8a18c 100644 --- a/src/core/operation/detail/copy.cc +++ b/src/core/operation/detail/copy.cc @@ -42,17 +42,29 @@ void Copy::validate() throw std::invalid_argument("Source and target must have the same type"); } auto validate_store = [](auto* store) { - if (store->unbound() || store->has_scalar_storage() || store->transformed()) { - throw std::invalid_argument("Copy accepts only normal, untransformed, region-backed stores"); + if (store->unbound() || store->transformed()) { + throw std::invalid_argument("Copy accepts only normal and untransformed stores"); } }; validate_store(target_.store); validate_store(source_.store); constraint_->validate(); + + if (target_.store->has_scalar_storage() != source_.store->has_scalar_storage()) { + throw std::runtime_error("Copies are supported only between the same kind of stores"); + } + if (redop_ && target_.store->has_scalar_storage()) { + throw std::runtime_error("Reduction copies don't support future-backed target stores"); + } } void Copy::launch(Strategy* p_strategy) { + if (target_.store->has_scalar_storage()) { + assert(source_.store->has_scalar_storage()); + target_.store->set_future(source_.store->get_future()); + return; + } auto& strategy = *p_strategy; CopyLauncher launcher(machine_); auto launch_domain = strategy.launch_domain(this); diff --git a/src/core/operation/detail/fill.cc b/src/core/operation/detail/fill.cc index 093afb00bc..f23beb6f5a 100644 --- a/src/core/operation/detail/fill.cc +++ b/src/core/operation/detail/fill.cc @@ -30,11 +30,10 @@ Fill::Fill(std::shared_ptr&& lhs, value_(std::move(value)) { store_mappings_[*lhs_var_] = lhs_; - if (lhs_->unbound() || lhs_->has_scalar_storage()) - throw std::runtime_error("Fill lhs must be a normal, region-backed store"); + if (lhs_->unbound()) throw std::invalid_argument("Fill lhs must be a normal store"); if (!value_->has_scalar_storage()) - throw std::runtime_error("Fill value should be a Future-back store"); + throw std::invalid_argument("Fill value should be a Future-back store"); } void Fill::validate() @@ -46,6 +45,11 @@ void Fill::validate() void Fill::launch(Strategy* strategy) { + if (lhs_->has_scalar_storage()) { + lhs_->set_future(value_->get_future()); + return; + } + FillLauncher launcher(machine_); auto launch_domain = strategy->launch_domain(this); auto part = (*strategy)[lhs_var_]; diff --git a/src/core/operation/detail/req_analyzer.cc b/src/core/operation/detail/req_analyzer.cc index c781c97c06..b6b35ef252 100644 --- a/src/core/operation/detail/req_analyzer.cc +++ b/src/core/operation/detail/req_analyzer.cc @@ -196,13 +196,17 @@ int32_t FutureAnalyzer::get_future_index(const Legion::Future& future) const void FutureAnalyzer::analyze_futures() { int32_t index = 0; - for (auto& future : futures_) { future_indices_[future] = index++; } + for (auto& future : futures_) { + if (future_indices_.find(future) != future_indices_.end()) { continue; } + future_indices_[future] = index++; + coalesced_.push_back(future); + } } template void FutureAnalyzer::_populate_launcher(Launcher& task) const { - for (auto& future : futures_) { task.add_future(future); } + for (auto& future : coalesced_) { task.add_future(future); } } void FutureAnalyzer::populate_launcher(Legion::IndexTaskLauncher& task) const { diff --git a/src/core/operation/detail/req_analyzer.h b/src/core/operation/detail/req_analyzer.h index 4a66734f3b..7ce368cac4 100644 --- a/src/core/operation/detail/req_analyzer.h +++ b/src/core/operation/detail/req_analyzer.h @@ -115,6 +115,7 @@ class FutureAnalyzer { private: std::map future_indices_{}; + std::vector coalesced_{}; std::vector futures_{}; }; diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt index 305bef2298..5eb2fe8fdd 100644 --- a/tests/cpp/CMakeLists.txt +++ b/tests/cpp/CMakeLists.txt @@ -18,6 +18,15 @@ if (NOT CMAKE_CXX_STANDARD) set(CMAKE_CXX_STANDARD 17) endif() +set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g") +set(CMAKE_CUDA_FLAGS_DEBUG "-O0 -g") +set(CMAKE_CXX_FLAGS_RELEASE "-O2") +set(CMAKE_CUDA_FLAGS_RELEASE "-O2") +set(CMAKE_CXX_FLAGS_MINSIZEREL "-Os") +set(CMAKE_CUDA_FLAGS_MINSIZEREL "-Os") +set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g") +set(CMAKE_CUDA_FLAGS_RELWITHDEBINFO "-O2 -g") + include(FetchContent) FetchContent_Declare( googletest diff --git a/tests/cpp/integration/copy_normal.cc b/tests/cpp/integration/copy_normal.cc index ced41207bd..877d34d1a7 100644 --- a/tests/cpp/integration/copy_normal.cc +++ b/tests/cpp/integration/copy_normal.cc @@ -170,8 +170,8 @@ void test_normal_copy(const NormalCopySpec& spec) auto& [shape, type, seed] = spec; - auto input = runtime->create_store(shape, type); - auto output = runtime->create_store(shape, type); + auto input = runtime->create_store(shape, type, true /*optimize_scalar*/); + auto output = runtime->create_store(shape, type, true /*optimize_scalar*/); fill_input(library, input, seed); runtime->issue_copy(output, input); @@ -203,6 +203,7 @@ TEST(Copy, Single) legate::Core::perform_registration(); test_normal_copy({{4, 7}, legate::int64(), legate::Scalar(int64_t(12))}); test_normal_copy({{1000, 100}, legate::uint32(), legate::Scalar(uint32_t(3))}); + test_normal_copy({{1}, legate::int64(), legate::Scalar(int64_t(12))}); } TEST(Copy, SingleReduction) diff --git a/tests/cpp/integration/fill.cc b/tests/cpp/integration/fill.cc index a333d33d1d..cd7a2994f6 100644 --- a/tests/cpp/integration/fill.cc +++ b/tests/cpp/integration/fill.cc @@ -58,7 +58,7 @@ template if (shape.empty()) return; - auto acc = input.write_accessor(shape); + auto acc = input.read_accessor(shape); for (legate::PointInRectIterator it(shape); it.valid(); ++it) { EXPECT_EQ(acc[*it], value); } } @@ -113,14 +113,15 @@ void check_output_slice(legate::LogicalStore store, runtime->submit(std::move(task)); } -void test_fill_index(int32_t dim) +void test_fill_index(int32_t dim, size_t size) { auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(library_name); int64_t v = 10; - auto lhs = runtime->create_store(legate::Shape(dim, SIZE), legate::int64()); + auto lhs = + runtime->create_store(legate::Shape(dim, size), legate::int64(), true /*optimize_scalar*/); auto value = runtime->create_store(v); // fill input store with some values @@ -130,15 +131,15 @@ void test_fill_index(int32_t dim) check_output(lhs, v); } -void test_fill_single(int32_t dim) +void test_fill_single(int32_t dim, size_t size) { auto runtime = legate::Runtime::get_runtime(); auto machine = runtime->get_machine(); legate::MachineTracker tracker(machine.slice(0, 1, legate::mapping::TaskTarget::CPU)); - test_fill_index(dim); + test_fill_index(dim, size); } -void test_fill_slice(int32_t dim) +void test_fill_slice(int32_t dim, size_t size) { auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(library_name); @@ -147,7 +148,7 @@ void test_fill_slice(int32_t dim) int64_t v2 = 200; int64_t offset = 3; - auto lhs = runtime->create_store(legate::Shape(dim, SIZE), legate::int64()); + auto lhs = runtime->create_store(legate::Shape(dim, size), legate::int64()); auto value_in_slice = runtime->create_store(v1); auto value_outside_slice = runtime->create_store(v2); @@ -177,25 +178,31 @@ void test_invalid() TEST(Fill, Index) { legate::Core::perform_registration(); - test_fill_index(1); - test_fill_index(2); - test_fill_index(3); + test_fill_index(1, SIZE); + test_fill_index(2, SIZE); + test_fill_index(3, SIZE); + test_fill_index(1, 1); + test_fill_index(2, 1); + test_fill_index(3, 1); } TEST(Fill, Single) { legate::Core::perform_registration(); - test_fill_single(1); - test_fill_single(2); - test_fill_single(3); + test_fill_single(1, SIZE); + test_fill_single(2, SIZE); + test_fill_single(3, SIZE); + test_fill_single(1, 1); + test_fill_single(2, 1); + test_fill_single(3, 1); } TEST(Fill, Slice) { legate::Core::perform_registration(); - test_fill_slice(1); - test_fill_slice(2); - test_fill_slice(3); + test_fill_slice(1, SIZE); + test_fill_slice(2, SIZE); + test_fill_slice(3, SIZE); } TEST(Fill, Invalid) { test_invalid(); } From 83b99d27a30107010fc11e82180c836fc0540ca2 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Thu, 21 Sep 2023 22:26:57 -0700 Subject: [PATCH 0225/1425] Add field ids to keys to retrieve requirement indices (#77) --- src/core/operation/detail/launcher_arg.cc | 2 +- src/core/operation/detail/req_analyzer.cc | 19 ++++++++---- src/core/operation/detail/req_analyzer.h | 13 +++++---- tests/cpp/integration/req_analyzer.cc | 35 ++++++++++++++++++++++- 4 files changed, 56 insertions(+), 13 deletions(-) diff --git a/src/core/operation/detail/launcher_arg.cc b/src/core/operation/detail/launcher_arg.cc index 29c3c4dfe9..99d530c4f8 100644 --- a/src/core/operation/detail/launcher_arg.cc +++ b/src/core/operation/detail/launcher_arg.cc @@ -39,7 +39,7 @@ void RegionFieldArg::pack(BufferBuilder& buffer, const StoreAnalyzer& analyzer) buffer.pack(proj_info_->redop); buffer.pack(region.get_dim()); - buffer.pack(analyzer.get_index(region, privilege_, *proj_info_)); + buffer.pack(analyzer.get_index(region, privilege_, *proj_info_, field_id)); buffer.pack(field_id); } diff --git a/src/core/operation/detail/req_analyzer.cc b/src/core/operation/detail/req_analyzer.cc index b6b35ef252..076b626f22 100644 --- a/src/core/operation/detail/req_analyzer.cc +++ b/src/core/operation/detail/req_analyzer.cc @@ -48,10 +48,13 @@ void FieldSet::insert(Legion::FieldID field_id, uint32_t FieldSet::num_requirements() const { return static_cast(coalesced_.size()); } uint32_t FieldSet::get_requirement_index(Legion::PrivilegeMode privilege, - const ProjectionInfo& proj_info) const + const ProjectionInfo& proj_info, + Legion::FieldID field_id) const { - auto finder = req_indices_.find(Key{privilege, proj_info}); - if (req_indices_.end() == finder) finder = req_indices_.find(Key{LEGION_READ_WRITE, proj_info}); + auto finder = req_indices_.find(std::make_pair(Key{privilege, proj_info}, field_id)); + if (req_indices_.end() == finder) { + finder = req_indices_.find(std::make_pair(Key{LEGION_READ_WRITE, proj_info}, field_id)); + } if (LegateDefined(LEGATE_USE_DEBUG)) { assert(finder != req_indices_.end()); } return finder->second; } @@ -66,7 +69,10 @@ void FieldSet::coalesce() } } uint32_t idx = 0; - for (const auto& [key, _] : coalesced_) req_indices_[key] = idx++; + for (const auto& [key, entry] : coalesced_) { + for (const auto& field : entry.fields) { req_indices_[std::make_pair(key, field)] = idx; } + ++idx; + } } namespace { @@ -107,12 +113,13 @@ void RequirementAnalyzer::insert(const Legion::LogicalRegion& region, uint32_t RequirementAnalyzer::get_requirement_index(const Legion::LogicalRegion& region, Legion::PrivilegeMode privilege, - const ProjectionInfo& proj_info) const + const ProjectionInfo& proj_info, + Legion::FieldID field_id) const { auto finder = field_sets_.find(region); if (LegateDefined(LEGATE_USE_DEBUG)) { assert(finder != field_sets_.end()); } auto& [field_set, req_offset] = finder->second; - return req_offset + field_set.get_requirement_index(privilege, proj_info); + return req_offset + field_set.get_requirement_index(privilege, proj_info, field_id); } void RequirementAnalyzer::analyze_requirements() diff --git a/src/core/operation/detail/req_analyzer.h b/src/core/operation/detail/req_analyzer.h index 7ce368cac4..7413088c92 100644 --- a/src/core/operation/detail/req_analyzer.h +++ b/src/core/operation/detail/req_analyzer.h @@ -38,7 +38,8 @@ class FieldSet { const ProjectionInfo& proj_info); uint32_t num_requirements() const; uint32_t get_requirement_index(Legion::PrivilegeMode privilege, - const ProjectionInfo& proj_info) const; + const ProjectionInfo& proj_info, + Legion::FieldID field_id) const; public: void coalesce(); @@ -51,7 +52,7 @@ class FieldSet { bool is_key; }; std::map coalesced_; - std::map req_indices_; + std::map, uint32_t> req_indices_; private: std::map field_projs_; @@ -65,7 +66,8 @@ class RequirementAnalyzer { const ProjectionInfo& proj_info); uint32_t get_requirement_index(const Legion::LogicalRegion& region, Legion::PrivilegeMode privilege, - const ProjectionInfo& proj_info) const; + const ProjectionInfo& proj_info, + Legion::FieldID field_id) const; bool empty() const { return field_sets_.empty(); } public: @@ -144,9 +146,10 @@ struct StoreAnalyzer { public: uint32_t get_index(const Legion::LogicalRegion& region, Legion::PrivilegeMode privilege, - const ProjectionInfo& proj_info) const + const ProjectionInfo& proj_info, + Legion::FieldID field_id) const { - return req_analyzer_.get_requirement_index(region, privilege, proj_info); + return req_analyzer_.get_requirement_index(region, privilege, proj_info, field_id); } uint32_t get_index(const Legion::FieldSpace& field_space, Legion::FieldID field_id) const { diff --git a/tests/cpp/integration/req_analyzer.cc b/tests/cpp/integration/req_analyzer.cc index a2606bba4b..141ce806eb 100644 --- a/tests/cpp/integration/req_analyzer.cc +++ b/tests/cpp/integration/req_analyzer.cc @@ -24,7 +24,16 @@ enum TaskIDs { struct Tester : public legate::LegateTask { static const int32_t TASK_ID = TESTER; - static void cpu_variant(legate::TaskContext context) {} + static void cpu_variant(legate::TaskContext context) + { + auto inputs = context.inputs(); + auto outputs = context.outputs(); + for (auto& input : inputs) { input.data().read_accessor(); } + for (auto& output : outputs) { + output.data().read_accessor(); + output.data().write_accessor(); + } + } }; void prepare() @@ -34,6 +43,24 @@ void prepare() Tester::register_variants(context); } +void test_inout_store() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + auto store1 = runtime->create_store({10, 5}, legate::int64()); + auto store2 = runtime->create_store({10, 5}, legate::int64()); + runtime->issue_fill(store1, legate::Scalar(int64_t{0})); + runtime->issue_fill(store2, legate::Scalar(int64_t{0})); + + auto task = runtime->create_task(context, TESTER); + auto part1 = task.add_input(store1); + auto part2 = task.add_input(store2); + task.add_output(store1); + task.add_constraint(legate::align(part1, part2)); + runtime->submit(std::move(task)); +} + void test_isomorphic_transformed_stores() { auto runtime = legate::Runtime::get_runtime(); @@ -51,6 +78,12 @@ void test_isomorphic_transformed_stores() runtime->submit(std::move(task)); } +TEST(ReqAnalyzer, InoutStore) +{ + legate::Core::perform_registration(); + test_inout_store(); +} + TEST(ReqAnalyzer, IsomorphicTransformedStores) { legate::Core::perform_registration(); From 16d255ddc19f78833463203b165dbed2bce44507 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Fri, 22 Sep 2023 16:55:25 -0700 Subject: [PATCH 0226/1425] Fix symbolic point inversion for transpose (#80) --- src/core/data/detail/transform.cc | 7 +------ tests/cpp/integration/alignment_constraints.cc | 1 + 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/core/data/detail/transform.cc b/src/core/data/detail/transform.cc index 5d29428764..a2c377f92f 100644 --- a/src/core/data/detail/transform.cc +++ b/src/core/data/detail/transform.cc @@ -632,12 +632,7 @@ std::unique_ptr Transpose::invert(const Partition* partition) const proj::SymbolicPoint Transpose::invert(const proj::SymbolicPoint& point) const { std::vector exprs; - exprs.resize(point.size()); - std::transform( - point.data().begin(), point.data().end(), exprs.begin(), [&](const proj::SymbolicExpr& expr) { - auto dim = inverse_[expr.dim()]; - return proj::SymbolicExpr(dim, expr.weight(), expr.offset()); - }); + for (int32_t idx : inverse_) { exprs.push_back(point[idx]); } return proj::SymbolicPoint(std::move(exprs)); } diff --git a/tests/cpp/integration/alignment_constraints.cc b/tests/cpp/integration/alignment_constraints.cc index eeac81fe8e..9e9d5d435b 100644 --- a/tests/cpp/integration/alignment_constraints.cc +++ b/tests/cpp/integration/alignment_constraints.cc @@ -182,6 +182,7 @@ void test_alignment_transformed() launch_tester(store1.promote(1, 5), store2.slice(1, legate::Slice(3, 8))); launch_tester(store1.promote(0, 10).transpose({1, 0}), store2); launch_tester(store1.delinearize(0, {10, 10}), store4); + launch_tester(store4.transpose({1, 0}).promote(0, 1), store4.promote(0, 1)); } void test_redundant_alignment() From d658584a03d8f3e1eac2bb77045fc1d0cfa8a44f Mon Sep 17 00:00:00 2001 From: Jacob Faibussowitsch Date: Mon, 25 Sep 2023 15:59:06 -0400 Subject: [PATCH 0227/1425] Add .dir-locals (#79) * Add .dir-locals --- .dir-locals.el | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .dir-locals.el diff --git a/.dir-locals.el b/.dir-locals.el new file mode 100644 index 0000000000..dca92887e0 --- /dev/null +++ b/.dir-locals.el @@ -0,0 +1,7 @@ +;;; legate.core specific configuration for Emacs +((nil . ((indent-tabs-mode . nil) + (tab-width . 4) + (eval . (add-hook 'before-save-hook #'delete-trailing-whitespace)))) + (python-mode . ((python-interpreter . (seq-find (lambda (item) (executable-find item)) '("python3" "python"))) + (python-indent-offset . 4) + (fill-column . 79)))) From 629b6d533f00cd7306f82554d9bcdffc8b56a109 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Mon, 25 Sep 2023 21:22:33 -0700 Subject: [PATCH 0228/1425] We've been setting key partitions at a wrong place (#82) --- src/core/operation/detail/task.cc | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/core/operation/detail/task.cc b/src/core/operation/detail/task.cc index 5e62be8e42..6a5d4880b6 100644 --- a/src/core/operation/detail/task.cc +++ b/src/core/operation/detail/task.cc @@ -84,11 +84,6 @@ void Task::launch_task(Strategy* p_strategy) for (auto& [arr, mapping] : inputs_) { launcher.add_input(arr->to_launcher_arg(mapping, strategy, launch_domain, READ_ONLY, -1)); - for (auto& [store, symb] : mapping) { - // Key partitions for unbound stores will be populated by the launcher - if (store->unbound()) continue; - store->set_key_partition(machine_, strategy[symb].get()); - } } for (auto& [arr, mapping] : outputs_) { launcher.add_output(arr->to_launcher_arg(mapping, strategy, launch_domain, WRITE_ONLY, -1)); From 7c33210d82eb18d9ccb65bd2f13c013f2593fbe8 Mon Sep 17 00:00:00 2001 From: Mona Han <129724851+mona-nv@users.noreply.github.com> Date: Tue, 26 Sep 2023 14:26:40 +0800 Subject: [PATCH 0229/1425] Add unit test for DimOrdering and InstanceMappingPolicy (#43) * Add unit test for DimOrdering and InstanceMappingPolicy * Add test for DimOrdering with default constructor --- tests/cpp/unit/mapping.cc | 186 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100644 tests/cpp/unit/mapping.cc diff --git a/tests/cpp/unit/mapping.cc b/tests/cpp/unit/mapping.cc new file mode 100644 index 0000000000..321b322b0b --- /dev/null +++ b/tests/cpp/unit/mapping.cc @@ -0,0 +1,186 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include + +#include "core/mapping/detail/mapping.h" +#include "legate.h" + +namespace unit { + +TEST(Mapping, DimOrdering) +{ + // Default construct + { + auto order = legate::mapping::DimOrdering(); + EXPECT_EQ(order.kind(), legate::mapping::DimOrdering::Kind::C); + EXPECT_EQ(order.dimensions(), std::vector()); + } + + // C ordering + { + auto c_order = legate::mapping::DimOrdering::c_order(); + EXPECT_EQ(c_order.kind(), legate::mapping::DimOrdering::Kind::C); + EXPECT_EQ(c_order.dimensions(), std::vector()); + + auto order = legate::mapping::DimOrdering(); + order.set_c_order(); + EXPECT_EQ(order.kind(), legate::mapping::DimOrdering::Kind::C); + } + + // Fortran ordering + { + auto fortran_order = legate::mapping::DimOrdering::fortran_order(); + EXPECT_EQ(fortran_order.kind(), legate::mapping::DimOrdering::Kind::FORTRAN); + EXPECT_EQ(fortran_order.dimensions(), std::vector()); + + auto order = legate::mapping::DimOrdering(); + order.set_fortran_order(); + EXPECT_EQ(order.kind(), legate::mapping::DimOrdering::Kind::FORTRAN); + } + + // custom ordering + { + std::vector dim = {0,1,2}; + auto custom_order = legate::mapping::DimOrdering::custom_order(dim); + EXPECT_EQ(custom_order.kind(), legate::mapping::DimOrdering::Kind::CUSTOM); + EXPECT_EQ(custom_order.dimensions(), dim); + + auto order = legate::mapping::DimOrdering(); + order.set_custom_order(dim); + EXPECT_EQ(order.kind(), legate::mapping::DimOrdering::Kind::CUSTOM); + EXPECT_EQ(order.dimensions(), dim); + } + + // custom ordering to c order + { + std::vector dim = {0,1,2}; + auto order = legate::mapping::DimOrdering::custom_order(dim); + order.set_c_order(); + EXPECT_EQ(order.kind(), legate::mapping::DimOrdering::Kind::C); + EXPECT_EQ(order.dimensions(), std::vector()); + } +} + +TEST(Mapping, InstanceMappingPolicy) +{ + // Empty InstanceMappingPolicy + { + auto policy = legate::mapping::InstanceMappingPolicy(); + EXPECT_EQ(policy.target, legate::mapping::StoreTarget::SYSMEM); + EXPECT_EQ(policy.allocation, legate::mapping::AllocPolicy::MAY_ALLOC); + EXPECT_EQ(policy.layout, legate::mapping::InstLayout::SOA); + EXPECT_EQ(policy.ordering, legate::mapping::DimOrdering()); + EXPECT_EQ(policy.exact, false); + } + + // Test set methods + { + auto target = legate::mapping::StoreTarget::FBMEM; + auto allocation = legate::mapping::AllocPolicy::MUST_ALLOC; + auto layout = legate::mapping::InstLayout::AOS; + auto dim_order = legate::mapping::DimOrdering::fortran_order(); + + auto policy = legate::mapping::InstanceMappingPolicy(); + policy.set_target(target); + EXPECT_EQ(policy, legate::mapping::InstanceMappingPolicy{}.with_target(target)); + + policy.set_allocation_policy(allocation); + EXPECT_EQ(policy, legate::mapping::InstanceMappingPolicy{}.with_target(target). + with_allocation_policy(allocation)); + + policy.set_instance_layout(layout); + EXPECT_EQ(policy, legate::mapping::InstanceMappingPolicy{}.with_target(target). + with_allocation_policy(allocation). + with_instance_layout(layout)); + + policy.set_ordering(dim_order); + EXPECT_EQ(policy, legate::mapping::InstanceMappingPolicy{}.with_target(target). + with_allocation_policy(allocation). + with_instance_layout(layout). + with_ordering(dim_order)); + + policy.set_exact(true); + EXPECT_EQ(policy, legate::mapping::InstanceMappingPolicy{}.with_target(target). + with_allocation_policy(allocation). + with_instance_layout(layout). + with_ordering(dim_order). + with_exact(true)); + } + + // Test subsumes + { + auto judge_subsumes = [](auto policy_a, auto policy_b){ + auto expect_result = true; + if(!(policy_a.target == policy_b.target)) + expect_result = false; + else if(!(policy_a.layout == policy_b.layout)) + expect_result = false; + else if(!(policy_a.ordering == policy_b.ordering)) + expect_result = false; + else if(!(policy_a.exact == policy_b.exact) && !policy_a.exact) + expect_result = false; + + EXPECT_EQ(policy_a.subsumes(policy_b), expect_result); + }; + + auto target = legate::mapping::StoreTarget::ZCMEM; + auto allocation = legate::mapping::AllocPolicy::MUST_ALLOC; + auto layout = legate::mapping::InstLayout::AOS; + std::vector dim = {0,1,2}; + auto dim_order = legate::mapping::DimOrdering::custom_order(dim); + + auto policy_a = legate::mapping::InstanceMappingPolicy{}.with_target(target). + with_allocation_policy(allocation). + with_instance_layout(layout). + with_ordering(dim_order). + with_exact(true); + auto policy_b = policy_a; + judge_subsumes(policy_a, policy_b); + + for (int i=(int)legate::mapping::StoreTarget::SYSMEM; i<=(int)legate::mapping::StoreTarget::SOCKETMEM; ++i) + { + policy_b.set_target(static_cast(i)); + judge_subsumes(policy_a, policy_b); + } + + policy_b = policy_a; + for (int i=(int)legate::mapping::AllocPolicy::MAY_ALLOC; i<=(int)legate::mapping::AllocPolicy::MUST_ALLOC; ++i) + { + policy_b.set_allocation_policy(static_cast(i)); + judge_subsumes(policy_a, policy_b); + } + + policy_b = policy_a; + for (int i = (int)legate::mapping::InstLayout::SOA; i<=(int)legate::mapping::InstLayout::AOS; ++i) + { + policy_b.set_instance_layout(static_cast(i)); + judge_subsumes(policy_a, policy_b); + } + + policy_b = policy_a; + policy_b.set_ordering(legate::mapping::DimOrdering::c_order()); + judge_subsumes(policy_a, policy_b); + + policy_b = policy_a; + policy_b.set_exact(false); + judge_subsumes(policy_a, policy_b); + + policy_a.set_exact(false); + judge_subsumes(policy_a, policy_b); + + policy_b.set_exact(true); + judge_subsumes(policy_a, policy_b); + } +} + +} // namespace unit \ No newline at end of file From 9ce81d2fb5e44d7ae19baa453cd2552e99a23ccb Mon Sep 17 00:00:00 2001 From: Jacob Faibussowitsch Date: Tue, 26 Sep 2023 13:18:59 -0400 Subject: [PATCH 0230/1425] Put all the caches in one place (#83) --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 5cacf19dfe..c466da6c52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ exclude_lines = [ [tool.mypy] python_version = "3.10" mypy_path = "typings/" +cache_dir = "./.cache/mypy" pretty = true show_error_codes = true @@ -97,3 +98,6 @@ module = [ "legate.jupyter.*", ] ignore_errors = true + +[tool.pytest.ini_options] +cache_dir = "./.cache/pytest" From 78935f0271c656ab8e5ed9af19dd2d9da0eb1adb Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Tue, 26 Sep 2023 22:26:03 -0700 Subject: [PATCH 0231/1425] Add cython-lint to the pre-commit hooks (#86) --- .pre-commit-config.yaml | 5 ++ legate/core/_lib/types.pyx | 42 +++++++------- legate/lgpatch.py | 3 +- tests/cpp/unit/mapping.cc | 114 ++++++++++++++++++++----------------- 4 files changed, 90 insertions(+), 74 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8f18375713..cafd639653 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,6 +24,11 @@ repos: pass_filenames: false args: ['legate', 'tests'] additional_dependencies: [numpy,pytest,pytest_mock] + - repo: https://github.com/MarcoGorelli/cython-lint + rev: v0.15.0 + hooks: + - id: cython-lint + - id: double-quote-cython-strings - repo: local hooks: - id: legate-defined diff --git a/legate/core/_lib/types.pyx b/legate/core/_lib/types.pyx index 2eb2fd4a06..caab79258f 100644 --- a/legate/core/_lib/types.pyx +++ b/legate/core/_lib/types.pyx @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual @@ -14,7 +15,6 @@ from libcpp.string cimport string from libcpp.utility cimport move from libcpp.vector cimport vector -import cython import numpy as np @@ -51,25 +51,25 @@ cdef extern from "core/legate_c.h" nogil: _AND "AND_LT" _XOR "XOR_LT" -NIL = legate_core_type_code_t._NULL -BOOL = legate_core_type_code_t._BOOL -INT8 = legate_core_type_code_t._INT8 -INT16 = legate_core_type_code_t._INT16 -INT32 = legate_core_type_code_t._INT32 -INT64 = legate_core_type_code_t._INT64 -UINT8 = legate_core_type_code_t._UINT8 -UINT16 = legate_core_type_code_t._UINT16 -UINT32 = legate_core_type_code_t._UINT32 -UINT64 = legate_core_type_code_t._UINT64 -FLOAT16 = legate_core_type_code_t._FLOAT16 -FLOAT32 = legate_core_type_code_t._FLOAT32 -FLOAT64 = legate_core_type_code_t._FLOAT64 -COMPLEX64 = legate_core_type_code_t._COMPLEX64 -COMPLEX128 = legate_core_type_code_t._COMPLEX128 -BINARY = legate_core_type_code_t._BINARY +NIL = legate_core_type_code_t._NULL +BOOL = legate_core_type_code_t._BOOL +INT8 = legate_core_type_code_t._INT8 +INT16 = legate_core_type_code_t._INT16 +INT32 = legate_core_type_code_t._INT32 +INT64 = legate_core_type_code_t._INT64 +UINT8 = legate_core_type_code_t._UINT8 +UINT16 = legate_core_type_code_t._UINT16 +UINT32 = legate_core_type_code_t._UINT32 +UINT64 = legate_core_type_code_t._UINT64 +FLOAT16 = legate_core_type_code_t._FLOAT16 +FLOAT32 = legate_core_type_code_t._FLOAT32 +FLOAT64 = legate_core_type_code_t._FLOAT64 +COMPLEX64 = legate_core_type_code_t._COMPLEX64 +COMPLEX128 = legate_core_type_code_t._COMPLEX128 +BINARY = legate_core_type_code_t._BINARY FIXED_ARRAY = legate_core_type_code_t._FIXED_ARRAY -STRUCT = legate_core_type_code_t._STRUCT -STRING = legate_core_type_code_t._STRING +STRUCT = legate_core_type_code_t._STRUCT +STRING = legate_core_type_code_t._STRING ADD = legate_core_reduction_op_kind_t._ADD SUB = legate_core_reduction_op_kind_t._SUB @@ -77,7 +77,7 @@ MUL = legate_core_reduction_op_kind_t._MUL DIV = legate_core_reduction_op_kind_t._DIV MAX = legate_core_reduction_op_kind_t._MAX MIN = legate_core_reduction_op_kind_t._MIN -OR = legate_core_reduction_op_kind_t._OR +OR = legate_core_reduction_op_kind_t._OR AND = legate_core_reduction_op_kind_t._AND XOR = legate_core_reduction_op_kind_t._XOR diff --git a/legate/lgpatch.py b/legate/lgpatch.py index 95cedb8411..d147aaa651 100644 --- a/legate/lgpatch.py +++ b/legate/lgpatch.py @@ -1,5 +1,6 @@ #! /usr/bin/env legate -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. # SPDX-License-Identifier: LicenseRef-NvidiaProprietary # # NVIDIA CORPORATION, its affiliates and licensors retain all intellectual diff --git a/tests/cpp/unit/mapping.cc b/tests/cpp/unit/mapping.cc index 321b322b0b..c294bda724 100644 --- a/tests/cpp/unit/mapping.cc +++ b/tests/cpp/unit/mapping.cc @@ -36,7 +36,7 @@ TEST(Mapping, DimOrdering) order.set_c_order(); EXPECT_EQ(order.kind(), legate::mapping::DimOrdering::Kind::C); } - + // Fortran ordering { auto fortran_order = legate::mapping::DimOrdering::fortran_order(); @@ -45,13 +45,13 @@ TEST(Mapping, DimOrdering) auto order = legate::mapping::DimOrdering(); order.set_fortran_order(); - EXPECT_EQ(order.kind(), legate::mapping::DimOrdering::Kind::FORTRAN); + EXPECT_EQ(order.kind(), legate::mapping::DimOrdering::Kind::FORTRAN); } // custom ordering { - std::vector dim = {0,1,2}; - auto custom_order = legate::mapping::DimOrdering::custom_order(dim); + std::vector dim = {0, 1, 2}; + auto custom_order = legate::mapping::DimOrdering::custom_order(dim); EXPECT_EQ(custom_order.kind(), legate::mapping::DimOrdering::Kind::CUSTOM); EXPECT_EQ(custom_order.dimensions(), dim); @@ -63,8 +63,8 @@ TEST(Mapping, DimOrdering) // custom ordering to c order { - std::vector dim = {0,1,2}; - auto order = legate::mapping::DimOrdering::custom_order(dim); + std::vector dim = {0, 1, 2}; + auto order = legate::mapping::DimOrdering::custom_order(dim); order.set_c_order(); EXPECT_EQ(order.kind(), legate::mapping::DimOrdering::Kind::C); EXPECT_EQ(order.dimensions(), std::vector()); @@ -85,87 +85,97 @@ TEST(Mapping, InstanceMappingPolicy) // Test set methods { - auto target = legate::mapping::StoreTarget::FBMEM; + auto target = legate::mapping::StoreTarget::FBMEM; auto allocation = legate::mapping::AllocPolicy::MUST_ALLOC; - auto layout = legate::mapping::InstLayout::AOS; - auto dim_order = legate::mapping::DimOrdering::fortran_order(); + auto layout = legate::mapping::InstLayout::AOS; + auto dim_order = legate::mapping::DimOrdering::fortran_order(); auto policy = legate::mapping::InstanceMappingPolicy(); policy.set_target(target); EXPECT_EQ(policy, legate::mapping::InstanceMappingPolicy{}.with_target(target)); - + policy.set_allocation_policy(allocation); - EXPECT_EQ(policy, legate::mapping::InstanceMappingPolicy{}.with_target(target). - with_allocation_policy(allocation)); - + EXPECT_EQ(policy, + legate::mapping::InstanceMappingPolicy{}.with_target(target).with_allocation_policy( + allocation)); + policy.set_instance_layout(layout); - EXPECT_EQ(policy, legate::mapping::InstanceMappingPolicy{}.with_target(target). - with_allocation_policy(allocation). - with_instance_layout(layout)); - + EXPECT_EQ(policy, + legate::mapping::InstanceMappingPolicy{} + .with_target(target) + .with_allocation_policy(allocation) + .with_instance_layout(layout)); + policy.set_ordering(dim_order); - EXPECT_EQ(policy, legate::mapping::InstanceMappingPolicy{}.with_target(target). - with_allocation_policy(allocation). - with_instance_layout(layout). - with_ordering(dim_order)); - + EXPECT_EQ(policy, + legate::mapping::InstanceMappingPolicy{} + .with_target(target) + .with_allocation_policy(allocation) + .with_instance_layout(layout) + .with_ordering(dim_order)); + policy.set_exact(true); - EXPECT_EQ(policy, legate::mapping::InstanceMappingPolicy{}.with_target(target). - with_allocation_policy(allocation). - with_instance_layout(layout). - with_ordering(dim_order). - with_exact(true)); + EXPECT_EQ(policy, + legate::mapping::InstanceMappingPolicy{} + .with_target(target) + .with_allocation_policy(allocation) + .with_instance_layout(layout) + .with_ordering(dim_order) + .with_exact(true)); } // Test subsumes { - auto judge_subsumes = [](auto policy_a, auto policy_b){ + auto judge_subsumes = [](auto policy_a, auto policy_b) { auto expect_result = true; - if(!(policy_a.target == policy_b.target)) + if (!(policy_a.target == policy_b.target)) expect_result = false; - else if(!(policy_a.layout == policy_b.layout)) + else if (!(policy_a.layout == policy_b.layout)) expect_result = false; - else if(!(policy_a.ordering == policy_b.ordering)) + else if (!(policy_a.ordering == policy_b.ordering)) expect_result = false; - else if(!(policy_a.exact == policy_b.exact) && !policy_a.exact) + else if (!(policy_a.exact == policy_b.exact) && !policy_a.exact) expect_result = false; - + EXPECT_EQ(policy_a.subsumes(policy_b), expect_result); }; - auto target = legate::mapping::StoreTarget::ZCMEM; - auto allocation = legate::mapping::AllocPolicy::MUST_ALLOC; - auto layout = legate::mapping::InstLayout::AOS; - std::vector dim = {0,1,2}; - auto dim_order = legate::mapping::DimOrdering::custom_order(dim); - - auto policy_a = legate::mapping::InstanceMappingPolicy{}.with_target(target). - with_allocation_policy(allocation). - with_instance_layout(layout). - with_ordering(dim_order). - with_exact(true); + auto target = legate::mapping::StoreTarget::ZCMEM; + auto allocation = legate::mapping::AllocPolicy::MUST_ALLOC; + auto layout = legate::mapping::InstLayout::AOS; + std::vector dim = {0, 1, 2}; + auto dim_order = legate::mapping::DimOrdering::custom_order(dim); + + auto policy_a = legate::mapping::InstanceMappingPolicy{} + .with_target(target) + .with_allocation_policy(allocation) + .with_instance_layout(layout) + .with_ordering(dim_order) + .with_exact(true); auto policy_b = policy_a; judge_subsumes(policy_a, policy_b); - for (int i=(int)legate::mapping::StoreTarget::SYSMEM; i<=(int)legate::mapping::StoreTarget::SOCKETMEM; ++i) - { + for (int i = (int)legate::mapping::StoreTarget::SYSMEM; + i <= (int)legate::mapping::StoreTarget::SOCKETMEM; + ++i) { policy_b.set_target(static_cast(i)); judge_subsumes(policy_a, policy_b); } - + policy_b = policy_a; - for (int i=(int)legate::mapping::AllocPolicy::MAY_ALLOC; i<=(int)legate::mapping::AllocPolicy::MUST_ALLOC; ++i) - { + for (int i = (int)legate::mapping::AllocPolicy::MAY_ALLOC; + i <= (int)legate::mapping::AllocPolicy::MUST_ALLOC; + ++i) { policy_b.set_allocation_policy(static_cast(i)); judge_subsumes(policy_a, policy_b); } policy_b = policy_a; - for (int i = (int)legate::mapping::InstLayout::SOA; i<=(int)legate::mapping::InstLayout::AOS; ++i) - { + for (int i = (int)legate::mapping::InstLayout::SOA; i <= (int)legate::mapping::InstLayout::AOS; + ++i) { policy_b.set_instance_layout(static_cast(i)); judge_subsumes(policy_a, policy_b); - } + } policy_b = policy_a; policy_b.set_ordering(legate::mapping::DimOrdering::c_order()); From 255af7e07cf474b39341d6006e1149f1dad0a562 Mon Sep 17 00:00:00 2001 From: Jacob Faibussowitsch Date: Wed, 27 Sep 2023 14:13:54 -0400 Subject: [PATCH 0232/1425] Feature: CMake Presets (#39) * update gitignore * Add cmake presets * Properly set openmp compile and link options * Add cmake presets to CI * Add cmake presets to tests * Create .dSYM directories for debug info on macOS when doing debug builds * Bump rapids-cmake to 23.10 * Set Legion as system in its configure stage * Mark Python headers as internal for cython builds * Centralize setting legate_core_ROOT if not needed * Always download the correct rapids-cmake version * Fix warnings as a result of enabling -Werror --- .../workflows/ci-gh-cpu-build-and-test.yml | 7 + .../workflows/ci-gh-gpu-build-and-test.yml | 7 + .github/workflows/gh-build.yml | 11 +- .github/workflows/gh-test.yml | 9 +- .gitignore | 2 + CMakeLists.txt | 19 +- CMakePresets.json | 15 ++ cmake/Modules/legate_core_options.cmake | 9 + cmake/legate_helper_functions.cmake | 20 +- cmake/presets/base.json | 60 ++++++ cmake/presets/debug.json | 70 +++++++ cmake/presets/debug_sanitizer.json | 72 +++++++ cmake/thirdparty/get_legion.cmake | 9 + continuous_integration/Dockerfile | 2 + continuous_integration/build-docker-image | 1 + .../home/coder/.local/bin/build-cpp-test | 8 +- .../home/coder/.local/bin/build-legate-cpp | 10 +- .../home/coder/.local/bin/build-legate-wheel | 2 +- install.py | 49 ++++- legate/core/_lib/CMakeLists.txt | 5 + legate/timing/_lib/CMakeLists.txt | 5 + legate_core_cpp.cmake | 176 ++++++++++++------ legate_core_python.cmake | 30 +++ src/core/comm/comm_nccl.cu | 2 +- src/core/comm/local_comm.cc | 2 +- src/core/data/detail/logical_store.cc | 22 ++- src/core/data/detail/transform.cc | 6 +- src/core/operation/detail/reduce.cc | 2 +- src/core/partitioning/detail/constraint.cc | 12 +- src/core/runtime/detail/partition_manager.cc | 2 +- src/core/runtime/detail/projection.cc | 2 +- src/core/type/detail/type_info.cc | 11 +- tests/cpp/CMakeLists.txt | 14 +- tests/cpp/CMakePresets.json | 11 ++ tests/cpp/integration/bloat_constraints.cc | 2 - tests/cpp/integration/consensus_match.cc | 6 +- tests/cpp/integration/copy_gather_scatter.cc | 1 + tests/cpp/integration/copy_normal.cc | 8 + tests/cpp/integration/copy_scatter.cc | 6 +- tests/cpp/integration/field_reuse.cc | 2 + tests/cpp/integration/fill.cc | 2 + tests/cpp/integration/image_constraints.cc | 1 + tests/cpp/integration/manual_simple.cc | 1 + tests/cpp/integration/multi_scalar_out.cc | 3 +- tests/cpp/integration/scale_constraints.cc | 2 - tests/cpp/unit/scalar.cc | 10 +- tests/cpp/unit/scoped_allocator.cc | 9 +- tests/cpp/unit/span.cc | 4 +- tests/integration/collective/CMakeLists.txt | 6 +- .../integration/region_manager/CMakeLists.txt | 6 +- tests/integration/registry/CMakeLists.txt | 6 +- tests/integration/scoping/CMakeLists.txt | 6 +- tests/integration/tree_reduce/CMakeLists.txt | 6 +- 53 files changed, 611 insertions(+), 160 deletions(-) create mode 100644 CMakePresets.json create mode 100644 cmake/presets/base.json create mode 100644 cmake/presets/debug.json create mode 100644 cmake/presets/debug_sanitizer.json create mode 100644 tests/cpp/CMakePresets.json diff --git a/.github/workflows/ci-gh-cpu-build-and-test.yml b/.github/workflows/ci-gh-cpu-build-and-test.yml index 9ff30264df..3ac2088691 100644 --- a/.github/workflows/ci-gh-cpu-build-and-test.yml +++ b/.github/workflows/ci-gh-cpu-build-and-test.yml @@ -12,10 +12,15 @@ on: jobs: build-cpu: + strategy: + fail-fast: false + matrix: + cmake-preset: [debug-gcc, debug-sanitizer-gcc] uses: ./.github/workflows/gh-build.yml with: build-target: cpu + cmake-preset: ${{ matrix.cmake-preset }} # Ref: https://docs.rapids.ai/resources/github-actions/#cpu-labels for `linux-amd64-cpu4` runs-on: ${{ github.repository == 'nv-legate/legate.core.internal' && 'linux-amd64-cpu4' || 'ubuntu-latest' }} sha: ${{ github.sha }} @@ -26,6 +31,7 @@ jobs: strategy: fail-fast: false matrix: + cmake-preset: [debug-gcc, debug-sanitizer-gcc] include: - { name: Pytest Unit Tests, test-scope: unit } - { name: mypy, test-scope: mypy } @@ -36,6 +42,7 @@ jobs: build-target: cpu runs-on: ${{ github.repository == 'nv-legate/legate.core.internal' && 'linux-amd64-cpu4' || 'ubuntu-latest' }} sha: ${{ github.sha }} + cmake-preset: ${{ matrix.cmake-preset }} test-scope: ${{ matrix.test-scope }} cleanup: diff --git a/.github/workflows/ci-gh-gpu-build-and-test.yml b/.github/workflows/ci-gh-gpu-build-and-test.yml index bc65b09ad7..4c301c18ad 100644 --- a/.github/workflows/ci-gh-gpu-build-and-test.yml +++ b/.github/workflows/ci-gh-gpu-build-and-test.yml @@ -12,10 +12,15 @@ on: jobs: build-gpu: + strategy: + fail-fast: false + matrix: + cmake-preset: [debug-gcc, debug-sanitizer-gcc] uses: ./.github/workflows/gh-build.yml with: build-target: gpu + cmake-preset: ${{ matrix.cmake-preset }} # Ref: https://docs.rapids.ai/resources/github-actions/#cpu-labels for `linux-amd64-cpu4` runs-on: ${{ github.repository == 'nv-legate/legate.core.internal' && 'linux-amd64-cpu4' || 'ubuntu-latest' }} sha: ${{ github.sha }} @@ -26,6 +31,7 @@ jobs: strategy: fail-fast: false matrix: + cmake-preset: [debug-gcc, debug-sanitizer-gcc] include: - { name: Pytest Unit Tests, test-scope: unit, runner: linux-amd64-gpu-v100-latest-1 } - { name: Pytest Unit Tests, test-scope: unit, runner: linux-amd64-2gpu } @@ -37,6 +43,7 @@ jobs: build-target: gpu runs-on: ${{ matrix.runner }} sha: ${{ github.sha }} + cmake-preset: ${{ matrix.cmake-preset }} test-scope: ${{ matrix.test-scope }} cleanup: diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index 1c847f4753..828eb43e44 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -9,6 +9,9 @@ on: runs-on: required: true type: string + cmake-preset: + required: true + type: string sha: required: true type: string @@ -16,12 +19,13 @@ on: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} BASE_IMAGE: rapidsai/devcontainers:23.06-cpp-cuda11.8-mambaforge-ubuntu22.04 - IMAGE_NAME: legate.core.internal_pkg-${{ inputs.build-target }} + IMAGE_NAME: legate.core.internal_pkg-${{ inputs.build-target }}-${{ inputs.cmake-preset }} USE_CUDA: ${{ (inputs.build-target == 'cpu' && 'OFF') || 'ON' }} + LEGATE_CORE_CMAKE_PRESET: ${{ inputs.cmake-preset }} jobs: build: - name: build-${{ inputs.build-target }}-sub-workflow + name: build-${{ inputs.build-target }}-${{ inputs.cmake-preset }}-sub-workflow permissions: id-token: write # This is required for configure-aws-credentials @@ -53,6 +57,7 @@ jobs: run: | echo BUILD_TARGET: ${{ inputs.build-target }} echo USE_CUDA: ${{ env.USE_CUDA }} + echo LEGATE_CORE_CMAKE_PRESET: ${{ env.LEGATE_CORE_CMAKE_PRESET }} IMAGE_TAG=${{ env.IMAGE_NAME }}:${{ inputs.sha }} @@ -100,6 +105,6 @@ jobs: - name: Upload build artifacts uses: actions/upload-artifact@v3 with: - name: "legate.core.internal-${{ inputs.build-target }}-${{ inputs.sha }}" + name: "legate.core.internal-${{ inputs.build-target }}-${{ inputs.cmake-preset }}-${{ inputs.sha }}" path: | artifacts diff --git a/.github/workflows/gh-test.yml b/.github/workflows/gh-test.yml index a1ecbcdcb6..091cc1459d 100644 --- a/.github/workflows/gh-test.yml +++ b/.github/workflows/gh-test.yml @@ -15,15 +15,18 @@ on: test-scope: required: true type: string + cmake-preset: + required: true + type: string jobs: test: if: github.repository_owner == 'nv-legate' - name: test-${{ inputs.build-target }}-sub-workflow + name: test-${{ inputs.build-target }}-${{ inputs.cmake-preset }}-sub-workflow runs-on: ${{ inputs.runs-on }} container: options: -u root - image: ghcr.io/nv-legate/legate.core.internal_pkg-${{ inputs.build-target }}:${{ inputs.sha }} + image: ghcr.io/nv-legate/legate.core.internal_pkg-${{ inputs.build-target }}-${{ inputs.cmake-preset }}:${{ inputs.sha }} env: PYTHONDONTWRITEBYTECODE: 1 NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} @@ -43,7 +46,7 @@ jobs: - name: Download build artifacts uses: actions/download-artifact@v3 with: - name: "legate.core.internal-${{inputs.build-target}}-${{ inputs.sha }}" + name: "legate.core.internal-${{inputs.build-target}}-${{ inputs.cmake-preset }}-${{ inputs.sha }}" path: artifacts - name: Display structure of downloaded artifacts diff --git a/.gitignore b/.gitignore index 1af00dee8c..1d2f12a480 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,5 @@ tests/cpp/build .legate-test-last-failed out/ venv/ +!cmake/presets/*.json +!CMakePresets.json diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ff502afba..e6824a956d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -57,24 +57,17 @@ endif() ############################################################################## # - Download and initialize RAPIDS CMake helpers ----------------------------- -if(NOT EXISTS ${CMAKE_BINARY_DIR}/RAPIDS.cmake) - file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-23.08/RAPIDS.cmake - ${CMAKE_BINARY_DIR}/RAPIDS.cmake) -endif() -include(${CMAKE_BINARY_DIR}/RAPIDS.cmake) -include(rapids-cmake) -include(rapids-cpm) -include(rapids-cuda) -include(rapids-export) -include(rapids-find) - set(legate_core_version 23.09.00) +include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/legate_helper_functions.cmake) + +legate_include_rapids() + # For now we want the optimization flags to match on both normal make and cmake # builds so we override the cmake defaults here for release, this changes # -O3 to -O2 and removes -DNDEBUG -set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g") -set(CMAKE_CUDA_FLAGS_DEBUG "-O0 -g") +# set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g") +# set(CMAKE_CUDA_FLAGS_DEBUG "-O0 -g") set(CMAKE_CXX_FLAGS_RELEASE "-O2") set(CMAKE_CUDA_FLAGS_RELEASE "-O2") set(CMAKE_CXX_FLAGS_MINSIZEREL "-Os") diff --git a/CMakePresets.json b/CMakePresets.json new file mode 100644 index 0000000000..5df219c5c2 --- /dev/null +++ b/CMakePresets.json @@ -0,0 +1,15 @@ +{ + // version 4 since CMake 3.23 + "version": 4, + "cmakeMinimumRequired": { + "major": 3, + "minor": 23, + "patch": 0 + }, + // include since version 4 + "include": [ + "cmake/presets/base.json", + "cmake/presets/debug.json", + "cmake/presets/debug_sanitizer.json" + ] +} diff --git a/cmake/Modules/legate_core_options.cmake b/cmake/Modules/legate_core_options.cmake index aaa08eed15..58b1702fa6 100644 --- a/cmake/Modules/legate_core_options.cmake +++ b/cmake/Modules/legate_core_options.cmake @@ -120,3 +120,12 @@ elseif(NOT DEFINED Legion_CUDA_DYNAMIC_LOAD) set(Legion_CUDA_DYNAMIC_LOAD OFF) set(CUDA_USE_STATIC_CUDA_RUNTIME ON) endif() + +set(legate_core_CXX_FLAGS "" CACHE STRING "C++ flags for legate core") +set(legate_core_CUDA_FLAGS "" CACHE STRING "CUDA flags for legate core") +set(legate_core_LINKER_FLAGS "" CACHE STRING "CUDA flags for legate core") + +# there must be some way to automate creating these for all dependent packages... +set(Legion_CXX_FLAGS "" CACHE STRING "C++ flags for Legion") +set(Legion_CUDA_FLAGS "" CACHE STRING "CUDA flags for Legion") +set(Legion_LINKER_FLAGS "" CACHE STRING "CUDA flags for Legion") diff --git a/cmake/legate_helper_functions.cmake b/cmake/legate_helper_functions.cmake index 7ddd7d3002..6dff7e172b 100644 --- a/cmake/legate_helper_functions.cmake +++ b/cmake/legate_helper_functions.cmake @@ -11,9 +11,13 @@ #============================================================================= macro(legate_include_rapids) + if(NOT rapids-cmake-version) + # default + set(rapids-cmake-version 23.10) + endif() if (NOT _LEGATE_HAS_RAPIDS) if(NOT EXISTS ${CMAKE_BINARY_DIR}/LEGATE_RAPIDS.cmake) - file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-23.08/RAPIDS.cmake + file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-${rapids-cmake-version}/RAPIDS.cmake ${CMAKE_BINARY_DIR}/LEGATE_RAPIDS.cmake) endif() include(${CMAKE_BINARY_DIR}/LEGATE_RAPIDS.cmake) @@ -449,3 +453,17 @@ user_context = get_legate_runtime().register_library(user_lib) string(CONFIGURE "${file_template}" file_content @ONLY) file(WRITE "${fn_library}" "${file_content}") endfunction() + +macro(legate_ensure_legate) + if (legate_core_CMAKE_PRESET_NAME AND NOT legate_core_ROOT) + # If we are using a preset (and the user is not overriding the path anyways), then we + # know exactly where the root is + cmake_path(SET legate_core_ROOT NORMALIZE "${LEGATE_CORE_DIR}/build/${legate_core_CMAKE_PRESET_NAME}") + endif() + + if(NOT (CMAKE_PROJECT_NAME STREQUAL "legate_core")) + # If CMAKE_PROJECT_NAME is not legate_core, then we are not configuring from + # top-level. + find_package(legate_core REQUIRED) + endif() +endmacro() diff --git a/cmake/presets/base.json b/cmake/presets/base.json new file mode 100644 index 0000000000..66d0722b2b --- /dev/null +++ b/cmake/presets/base.json @@ -0,0 +1,60 @@ +{ + "version": 2, + "configurePresets": [ + { + "name": "base", + "hidden": true, + "binaryDir": "${sourceDir}/build/${presetName}", + "cacheVariables": { + "CMAKE_EXPORT_COMPILE_COMMANDS": { + "type": "BOOL", + "value": "ON" + }, + "CMAKE_CXX_STANDARD": { + "type": "STRING", + "value": "17" + }, + "CMAKE_CXX_STANDARD_REQUIRED": { + "type": "BOOL", + "value": "ON" + }, + "CMAKE_CUDA_STANDARD": { + "type": "STRING", + "value": "17" + }, + "CMAKE_CUDA_STANDARD_REQUIRED": { + "type": "BOOL", + "value": "ON" + }, + "Legion_USE_Python": { + "type": "BOOL", + "value": "ON" + }, + "legate_core_CMAKE_PRESET_NAME": { + "type": "STRING", + "value": "${presetName}" + } + } + }, + { + "inherits": "base", + "hidden": true, + "name": "base-ninja", + "generator": "Ninja" + }, + { + "inherits": "base", + "hidden": true, + "name": "base-make", + "generator": "Unix Makefiles" + } + ], + "buildPresets": [ + { + "name": "base", + "hidden": true, + "configurePreset": "base", + "jobs": 7 + } + ] +} diff --git a/cmake/presets/debug.json b/cmake/presets/debug.json new file mode 100644 index 0000000000..c13a5ddafb --- /dev/null +++ b/cmake/presets/debug.json @@ -0,0 +1,70 @@ +{ + "version": 4, + "include": [ + "base.json" + ], + "configurePresets": [ + { + "inherits": ["base", "base-ninja"], + "hidden": true, + "name": "base-debug", + "environment": { + "LEGATE_DEBUG_PRESET_CMAKE_CXX_FLAGS": "-O0 -g -g3", + "LEGATE_DEBUG_PRESET_CMAKE_CUDA_FLAGS": "-g -lineinfo --compiler-options='$env{LEGATE_DEBUG_PRESET_CMAKE_CXX_FLAGS}'", + "LEGATE_DEBUG_PRESET_LEGATE_CORE_CUDA_FLAGS": "-g -lineinfo --compiler-options='$env{LEGATE_DEBUG_PRESET_LEGATE_CORE_CXX_FLAGS}'" + }, + "cacheVariables": { + "CMAKE_BUILD_TYPE": { + "type": "STRING", + "value": "Debug" + }, + "CMAKE_CXX_FLAGS_DEBUG": { + "type": "STRING", + "value": "$env{LEGATE_DEBUG_PRESET_CMAKE_CXX_FLAGS}" + }, + "legate_core_CXX_FLAGS": { + "type": "STRING", + "value": "$env{LEGATE_DEBUG_PRESET_LEGATE_CORE_CXX_FLAGS}" + }, + "CMAKE_CUDA_FLAGS_DEBUG": { + "type": "STRING", + "value": "$env{LEGATE_DEBUG_PRESET_CMAKE_CUDA_FLAGS}" + }, + "legate_core_CUDA_FLAGS": { + "type": "STRING", + "value": "$env{LEGATE_DEBUG_PRESET_LEGATE_CORE_CUDA_FLAGS}" + } + } + }, + { + "inherits": "base-debug", + "name": "debug-clang", + "displayName": "Debug Clang Build", + "description": "Debug Clang Build", + "environment": { + "LEGATE_DEBUG_PRESET_LEGATE_CORE_CXX_FLAGS": "-Werror -Wall -fstack-protector -Walloca -Wdeprecated -Wimplicit-fallthrough -Wsign-compare -fdiagnostics-show-template-tree -Warray-bounds-pointer-arithmetic -Wassign-enum -Wformat-pedantic -D_LIBCPP_ENABLE_ASSERTIONS=1 -D_LIBCPP_ENABLE_NODISCARD=1" + } + }, + { + "inherits": "base-debug", + "name": "debug-gcc", + "displayName": "Debug GCC Build", + "description": "Debug GCC Build", + "environment": { + "LEGATE_DEBUG_PRESET_LEGATE_CORE_CXX_FLAGS": "-Werror -Wall -fstack-protector -Walloca -Wdeprecated -Wimplicit-fallthrough -fdiagnostics-show-template-tree" + } + } + ], + "buildPresets": [ + { + "inherits": "base", + "name": "debug-clang", + "configurePreset": "debug-clang" + }, + { + "inherits": "base", + "name": "debug-gcc", + "configurePreset": "debug-gcc" + } + ] +} diff --git a/cmake/presets/debug_sanitizer.json b/cmake/presets/debug_sanitizer.json new file mode 100644 index 0000000000..4c3912b34f --- /dev/null +++ b/cmake/presets/debug_sanitizer.json @@ -0,0 +1,72 @@ +{ + "version": 4, + "include": [ + "debug.json" + ], + "configurePresets": [ + { + "inherits": "base-debug", + "name": "base-debug-sanitizer", + "hidden": true, + "environment": { + // base flags + "LEGATE_DEBUG_SANITIZER_PRESET_SANITIZER_FLAGS": "-fsanitize=address,undefined,bounds", + "LEGATE_DEBUG_SANITIZER_PRESET_SANITIZER_COMPILER_FLAGS": "$env{LEGATE_DEBUG_SANITIZER_PRESET_SANITIZER_FLAGS} -fno-omit-frame-pointer", + // CXX flags + "LEGATE_DEBUG_SANITIZER_PRESET_CMAKE_CXX_FLAGS": "$env{LEGATE_DEBUG_PRESET_CMAKE_CXX_FLAGS} $env{LEGATE_DEBUG_SANITIZER_PRESET_SANITIZER_COMPILER_FLAGS}", + "LEGATE_DEBUG_SANITIZER_PRESET_LEGATE_CORE_CXX_FLAGS": "$env{LEGATE_DEBUG_PRESET_LEGATE_CORE_CXX_FLAGS} $env{LEGATE_DEBUG_SANITIZER_PRESET_SANITIZER_COMPILER_FLAGS}", + // CUDA flags, no sanitizers enabled because nvcc chokes on the arguments... + "LEGATE_DEBUG_SANITIZER_PRESET_CMAKE_CUDA_FLAGS": "$env{LEGATE_DEBUG_PRESET_CMAKE_CUDA_FLAGS} --compiler-options=-fno-omit-frame-pointer", + "LEGATE_DEBUG_SANITIZER_PRESET_LEGATE_CORE_CUDA_FLAGS": "$env{LEGATE_DEBUG_PRESET_LEGATE_CORE_CUDA_FLAGS} --compiler-options=-fno-omit-frame-pointer", + // Linker flags + "LEGATE_DEBUG_SANITIZER_PRESET_LEGATE_CORE_LINKER_FLAGS": "$env{LEGATE_DEBUG_SANITIZER_PRESET_SANITIZER_FLAGS}" + }, + "cacheVariables": { + "CMAKE_CXX_FLAGS_DEBUG": { + "type": "STRING", + "value": "$env{LEGATE_DEBUG_SANITIZER_PRESET_CMAKE_CXX_FLAGS}" + }, + "legate_core_CXX_FLAGS": { + "type": "STRING", + "value": "$env{LEGATE_DEBUG_SANITIZER_PRESET_LEGATE_CORE_CXX_FLAGS}" + }, + "CMAKE_CUDA_FLAGS_DEBUG": { + "type": "STRING", + "value": "$env{LEGATE_DEBUG_SANITIZER_PRESET_CMAKE_CUDA_FLAGS}" + }, + "legate_core_CUDA_FLAGS": { + "type": "STRING", + "value": "$env{LEGATE_DEBUG_SANITIZER_PRESET_LEGATE_CORE_CUDA_FLAGS}" + }, + "legate_core_LINKER_FLAGS": { + "type": "STRING", + "value": "$env{LEGATE_DEBUG_SANITIZER_PRESET_LEGATE_CORE_LINKER_FLAGS}" + } + } + }, + { + "inherits": ["base-debug-sanitizer", "debug-clang"], + "name": "debug-sanitizer-clang", + "displayName": "Debug Clang Build With Sanitizers", + "description": "Debug Clang Build With Sanitizers" + }, + { + "inherits": ["base-debug-sanitizer", "debug-gcc"], + "name": "debug-sanitizer-gcc", + "displayName": "Debug GCC Build With Sanitizers", + "description": "Debug GCC Build With Sanitizers" + } + ], + "buildPresets": [ + { + "inherits": "debug-clang", + "name": "debug-sanitizer-clang", + "configurePreset": "debug-sanitizer-clang" + }, + { + "inherits": "debug-gcc", + "name": "debug-sanitizer-gcc", + "configurePreset": "debug-sanitizer-gcc" + } + ] +} diff --git a/cmake/thirdparty/get_legion.cmake b/cmake/thirdparty/get_legion.cmake index 4dcac1755a..31dd219c86 100644 --- a/cmake/thirdparty/get_legion.cmake +++ b/cmake/thirdparty/get_legion.cmake @@ -117,10 +117,19 @@ function(find_or_configure_legion) set(Legion_BACKTRACE_USE_LIBDW OFF) endif() + string(APPEND CMAKE_CXX_FLAGS ${Legion_CXX_FLAGS}) + string(APPEND CMAKE_CUDA_FLAGS ${Legion_CUDA_FLAGS}) + string(APPEND CMAKE_LINKER_FLAGS ${Legion_LINKER_FLAGS}) + rapids_cpm_find(Legion ${version} ${FIND_PKG_ARGS} CPM_ARGS ${legion_cpm_git_args} + # HACK: Legion headers contain *many* warnings, but we would like to build with + # -Wall -Werror. But there is a work-around. Compilers treat system headers as + # special and do not emit any warnings about suspect code in them, so until + # legion cleans house, we mark their headers as "system" headers. FIND_PACKAGE_ARGUMENTS EXACT + SYSTEM TRUE EXCLUDE_FROM_ALL ${exclude_from_all} OPTIONS ${_legion_cuda_options} "CMAKE_CXX_STANDARD ${_cxx_std}" diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile index 03a3005d09..7b25deb306 100644 --- a/continuous_integration/Dockerfile +++ b/continuous_integration/Dockerfile @@ -11,6 +11,8 @@ ENV VAULT_HOST=https://vault.ops.k8s.rapids.ai ENV HISTFILE=/home/coder/.cache/._bash_history ENV DEFAULT_CONDA_ENV=legate +ARG LEGATE_CORE_CMAKE_PRESET +ENV LEGATE_CORE_CMAKE_PRESET=${LEGATE_CORE_CMAKE_PRESET} ARG USE_CUDA ENV USE_CUDA=${USE_CUDA} diff --git a/continuous_integration/build-docker-image b/continuous_integration/build-docker-image index e36cf79743..3d5d1cac1b 100755 --- a/continuous_integration/build-docker-image +++ b/continuous_integration/build-docker-image @@ -34,6 +34,7 @@ docker build \ --build-arg AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ --build-arg GITHUB_TOKEN="$GITHUB_TOKEN" \ --build-arg USE_CUDA="$USE_CUDA" \ + --build-arg LEGATE_CORE_CMAKE_PRESET="$LEGATE_CORE_CMAKE_PRESET" \ --progress=plain \ --tag="$IMAGE_TAG" \ -f "$SOURCE_DIR/continuous_integration/Dockerfile" "$SOURCE_DIR" diff --git a/continuous_integration/home/coder/.local/bin/build-cpp-test b/continuous_integration/home/coder/.local/bin/build-cpp-test index ba0048093c..043263582a 100755 --- a/continuous_integration/home/coder/.local/bin/build-cpp-test +++ b/continuous_integration/home/coder/.local/bin/build-cpp-test @@ -3,16 +3,16 @@ build_cpp_tests() { set -ex; - conda deactivate + conda deactivate cd ~/legate/tests/cpp rm -rf build - cmake -B build -S . -D legate_core_ROOT=../../build -D CMAKE_BUILD_TYPE=Debug - cmake --build build -v -j 4 + cmake --preset ${LEGATE_CORE_CMAKE_PRESET} + cmake --build --preset ${LEGATE_CORE_CMAKE_PRESET} ( - cd ~/legate/tests/cpp/build; + cd ./build/${LEGATE_CORE_CMAKE_PRESET}; cpack -G TGZ; cp cpp_tests*.tar.gz /tmp/out/; ); diff --git a/continuous_integration/home/coder/.local/bin/build-legate-cpp b/continuous_integration/home/coder/.local/bin/build-legate-cpp index 5e8cbdffc1..ba788ea3aa 100755 --- a/continuous_integration/home/coder/.local/bin/build-legate-cpp +++ b/continuous_integration/home/coder/.local/bin/build-legate-cpp @@ -7,7 +7,6 @@ build_legate_cpp() { local cmake_args=(${CMAKE_ARGS:-}); cmake_args+=(-DBUILD_SHARED_LIBS=ON); cmake_args+=(-DBUILD_MARCH=${BUILD_MARCH:-haswell}); - cmake_args+=(-DCMAKE_BUILD_TYPE=Release); cmake_args+=(-DLegion_CUDA_ARCH=all-major); cmake_args+=(-DLegion_NETWORKS=); cmake_args+=(-DLegion_USE_Python=ON); @@ -28,6 +27,7 @@ build_legate_cpp() { cmake_args+=(-DLegion_USE_OpenMP=${USE_OPENMP:-OFF}); cmake_args+=(-Dlegate_core_BUILD_DOCS=ON); cmake_args+=(-DCMAKE_BUILD_PARALLEL_LEVEL=${JOBS:-$(nproc --ignore=1)}); + cmake_args+=(--preset ${LEGATE_CORE_CMAKE_PRESET:-debug-gcc}) cmake_args+=(${@}); sccache --stop-server >/dev/null 2>&1 || true; @@ -39,19 +39,21 @@ build_legate_cpp() { rm -rf ~/legate/build; + cd ~/legate; + time CMAKE_BUILD_PARALLEL_LEVEL=${JOBS:-$(nproc --ignore=1)} \ - cmake -S ~/legate -B ~/legate/build "${cmake_args[@]}" -GNinja; + cmake -S ~/legate "${cmake_args[@]}" -GNinja; sccache --show-stats; time CMAKE_BUILD_PARALLEL_LEVEL=${JOBS:-$(nproc --ignore=1)} \ - cmake --build ~/legate/build --verbose --parallel ${JOBS:-$(nproc --ignore=1)}; + cmake --build --preset ${LEGATE_CORE_CMAKE_PRESET} --verbose --parallel ${JOBS:-$(nproc --ignore=1)}; sccache --show-stats; ( mkdir -p /tmp/out; - cd ~/legate/build; + cd ~/legate/build/${LEGATE_CORE_CMAKE_PRESET}; cpack -G TGZ; cp ./*-Linux.tar.gz /tmp/out/; ); diff --git a/continuous_integration/home/coder/.local/bin/build-legate-wheel b/continuous_integration/home/coder/.local/bin/build-legate-wheel index a51e65f4e7..5572eff856 100755 --- a/continuous_integration/home/coder/.local/bin/build-legate-wheel +++ b/continuous_integration/home/coder/.local/bin/build-legate-wheel @@ -21,7 +21,7 @@ build_legate_wheel() { local cmake_args=(${CMAKE_ARGS:-}); cmake_args+=("-DFIND_LEGATE_CORE_CPP=ON"); - cmake_args+=("-Dlegate_core_ROOT=$HOME/legate/build"); + cmake_args+=("-Dlegate_core_ROOT=$HOME/legate/build/${LEGATE_CORE_CMAKE_PRESET}"); # Build + package legate.core Python wheel time CMAKE_GENERATOR="Ninja" \ diff --git a/install.py b/install.py index f1aea8aac8..1f62863fb6 100755 --- a/install.py +++ b/install.py @@ -18,6 +18,12 @@ import shutil import subprocess import sys + +try: + import skbuild +except ModuleNotFoundError as ie: + raise ImportError("Must install scikit-build!") from ie + from distutils import sysconfig # Flush output on newlines @@ -106,12 +112,16 @@ def find_active_python_version_and_path(): def scikit_build_cmake_build_dir(skbuild_dir): - if os.path.exists(skbuild_dir): - for f in os.listdir(skbuild_dir): - if os.path.exists( - cmake_build := os.path.join(skbuild_dir, f, "cmake-build") - ): - return cmake_build + def yield_guesses(): + sk_cmake_build_dir = skbuild.constants.CMAKE_BUILD_DIR() + + yield os.path.join(skbuild_dir, sk_cmake_build_dir) + par_skbuild_dir = os.path.dirname(skbuild_dir) + yield os.path.join(par_skbuild_dir, sk_cmake_build_dir) + + for guess in yield_guesses(): + if os.path.exists(guess): + return os.path.abspath(guess) return None @@ -260,6 +270,7 @@ def install_legion_jupyter_notebook( def install( + cmake_preset, networks, cuda, arch, @@ -313,6 +324,7 @@ def install( print(f"Verbose build is {'on' if verbose else 'off'}") if verbose: + print(f"cmake_preset: {cmake_preset}") print(f"networks: {networks}") print(f"cuda: {cuda}") print(f"arch: {arch}") @@ -433,7 +445,10 @@ def validate_path(path): if clean_first: shutil.rmtree(skbuild_dir, ignore_errors=True) shutil.rmtree(join(legate_core_dir, "dist"), ignore_errors=True) - shutil.rmtree(join(legate_core_dir, "build"), ignore_errors=True) + build_dir = join(legate_core_dir, "build") + if cmake_preset: + build_dir = join(build_dir, cmake_preset) + shutil.rmtree(build_dir, ignore_errors=True) shutil.rmtree( join(legate_core_dir, "legate_core.egg-info"), ignore_errors=True, @@ -443,7 +458,12 @@ def validate_path(path): pip_install_cmd = [sys.executable, "-m", "pip", "install"] # Use preexisting CMAKE_ARGS from conda if set - cmake_flags = cmd_env.get("CMAKE_ARGS", "").split(" ") + cmake_flags = [] + + if cmake_preset: + cmake_flags.extend(["--preset", cmake_preset]) + + cmake_flags.extend(cmd_env.get("CMAKE_ARGS", "").split(" ")) if unknown is None: unknown = [] @@ -473,7 +493,7 @@ def validate_path(path): else: pip_install_cmd += ["--upgrade"] - pip_install_cmd += ["."] + pip_install_cmd += [os.getcwd()] if verbose: pip_install_cmd += ["-vv"] @@ -564,7 +584,10 @@ def validate_path(path): # execute python -m pip install . execute_command(pip_install_cmd, verbose, cwd=legate_core_dir, env=cmd_env) - + print("=" * 90, flush=True) + print("=" * 90, flush=True) + print("=" * 90, flush=True) + print("=" * 90, flush=True) install_legion_python_bindings( verbose, cmake_exe, @@ -587,6 +610,12 @@ def driver(): ) parser = argparse.ArgumentParser(description="Install Legate front end.") + parser.add_argument( + "--cmake-preset", + required=False, + default=os.environ.get("LEGATE_CORE_CMAKE_PRESET", ""), + help="Set a CMake Preset to use", + ) parser.add_argument( "--debug", dest="debug", diff --git a/legate/core/_lib/CMakeLists.txt b/legate/core/_lib/CMakeLists.txt index df69f7a2bf..1d1b1eb6fc 100644 --- a/legate/core/_lib/CMakeLists.txt +++ b/legate/core/_lib/CMakeLists.txt @@ -20,6 +20,11 @@ rapids_cython_create_modules( LINKED_LIBRARIES "${linked_libraries}" ) +find_package (Python3 REQUIRED COMPONENTS Development) + foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) target_compile_features(${target} PRIVATE cxx_std_17) + # Mark python headers as system (in case user has added warning flags, we don't want the + # python headers to trigger them) + target_include_directories(${target} SYSTEM PRIVATE ${Python3_INCLUDE_DIRS}) endforeach() diff --git a/legate/timing/_lib/CMakeLists.txt b/legate/timing/_lib/CMakeLists.txt index 1533af37bc..579c70f086 100644 --- a/legate/timing/_lib/CMakeLists.txt +++ b/legate/timing/_lib/CMakeLists.txt @@ -20,6 +20,11 @@ rapids_cython_create_modules( LINKED_LIBRARIES "${linked_libraries}" ) +find_package (Python3 REQUIRED COMPONENTS Development) + foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) target_compile_features(${target} PRIVATE cxx_std_17) + # Mark python headers as system (in case user has added warning flags, we don't want the + # python headers to trigger them) + target_include_directories(${target} SYSTEM PRIVATE ${Python3_INCLUDE_DIRS}) endforeach() diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index 00c3a981a9..82586ee805 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -120,59 +120,6 @@ include(cmake/thirdparty/get_thrust.cmake) # - legate.core -------------------------------------------------------------- set(legate_core_SOURCES "") -set(legate_core_CXX_DEFS "") -set(legate_core_CUDA_DEFS "") -set(legate_core_CXX_OPTIONS "") -set(legate_core_CUDA_OPTIONS "") - -include(cmake/Modules/set_cpu_arch_flags.cmake) -set_cpu_arch_flags(legate_core_CXX_OPTIONS) - -if (legate_core_COLLECTIVE) - list(APPEND legate_core_CXX_DEFS LEGATE_USE_COLLECTIVE) -endif() - -if(CMAKE_BUILD_TYPE STREQUAL "Debug") - list(APPEND legate_core_CXX_DEFS LEGATE_USE_DEBUG) - list(APPEND legate_core_CUDA_DEFS LEGATE_USE_DEBUG) -endif() - -if(Legion_USE_CUDA) - list(APPEND legate_core_CXX_DEFS LEGATE_USE_CUDA) - list(APPEND legate_core_CUDA_DEFS LEGATE_USE_CUDA) - - add_cuda_architecture_defines(legate_core_CUDA_DEFS ARCHS ${Legion_CUDA_ARCH}) - - list(APPEND legate_core_CUDA_OPTIONS -Xfatbin=-compress-all) - list(APPEND legate_core_CUDA_OPTIONS --expt-extended-lambda) - list(APPEND legate_core_CUDA_OPTIONS --expt-relaxed-constexpr) - list(APPEND legate_core_CUDA_OPTIONS -Wno-deprecated-gpu-targets) -endif() - -if(Legion_USE_OpenMP) - list(APPEND legate_core_CXX_DEFS LEGATE_USE_OPENMP) - list(APPEND legate_core_CUDA_DEFS LEGATE_USE_OPENMP) -endif() - -if(Legion_NETWORKS) - list(APPEND legate_core_CXX_DEFS LEGATE_USE_NETWORK) - list(APPEND legate_core_CUDA_DEFS LEGATE_USE_NETWORK) -endif() - -# Change THRUST_DEVICE_SYSTEM for `.cpp` files -# TODO: This is what we do in cuNumeric, should we do it here as well? -if(Legion_USE_OpenMP) - list(APPEND legate_core_CXX_OPTIONS -UTHRUST_DEVICE_SYSTEM) - list(APPEND legate_core_CXX_OPTIONS -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP) -elseif(NOT Legion_USE_CUDA) - list(APPEND legate_core_CXX_OPTIONS -UTHRUST_DEVICE_SYSTEM) - list(APPEND legate_core_CXX_OPTIONS -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_CPP) -endif() -# Or should we only do it if OpenMP and CUDA are both disabled? -# if(NOT Legion_USE_OpenMP AND (NOT Legion_USE_CUDA)) -# list(APPEND legate_core_CXX_OPTIONS -UTHRUST_DEVICE_SYSTEM) -# list(APPEND legate_core_CXX_OPTIONS -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_CPP) -# endif() list(APPEND legate_core_SOURCES src/core/legate_c.cc @@ -283,6 +230,63 @@ endif() add_library(legate_core ${legate_core_SOURCES}) add_library(legate::core ALIAS legate_core) +set(legate_core_CXX_DEFS "") +set(legate_core_CUDA_DEFS "") +set(legate_core_CXX_OPTIONS "") +set(legate_core_CUDA_OPTIONS "") +set(legate_core_LINKER_OPTIONS "") + +include(cmake/Modules/set_cpu_arch_flags.cmake) +set_cpu_arch_flags(legate_core_CXX_OPTIONS) + +if (legate_core_COLLECTIVE) + list(APPEND legate_core_CXX_DEFS LEGATE_USE_COLLECTIVE) +endif() + +if(CMAKE_BUILD_TYPE STREQUAL "Debug") + list(APPEND legate_core_CXX_DEFS LEGATE_USE_DEBUG) + list(APPEND legate_core_CUDA_DEFS LEGATE_USE_DEBUG) +endif() + +if(Legion_USE_CUDA) + list(APPEND legate_core_CXX_DEFS LEGATE_USE_CUDA) + list(APPEND legate_core_CUDA_DEFS LEGATE_USE_CUDA) + + add_cuda_architecture_defines(legate_core_CUDA_DEFS ARCHS ${Legion_CUDA_ARCH}) + + list(APPEND legate_core_CUDA_OPTIONS -Xfatbin=-compress-all) + list(APPEND legate_core_CUDA_OPTIONS --expt-extended-lambda) + list(APPEND legate_core_CUDA_OPTIONS --expt-relaxed-constexpr) + list(APPEND legate_core_CUDA_OPTIONS -Wno-deprecated-gpu-targets) +endif() + +if(Legion_NETWORKS) + list(APPEND legate_core_CXX_DEFS LEGATE_USE_NETWORK) + list(APPEND legate_core_CUDA_DEFS LEGATE_USE_NETWORK) +endif() + +# Change THRUST_DEVICE_SYSTEM for `.cpp` files +# TODO: This is what we do in cuNumeric, should we do it here as well? +if(Legion_USE_OpenMP) + find_package(OpenMP REQUIRED) + + target_link_libraries(legate_core PRIVATE OpenMP::OpenMP_CXX) + + list(APPEND legate_core_CXX_DEFS LEGATE_USE_OPENMP) + list(APPEND legate_core_CUDA_DEFS LEGATE_USE_OPENMP) + + list(APPEND legate_core_CXX_OPTIONS -UTHRUST_DEVICE_SYSTEM) + list(APPEND legate_core_CXX_OPTIONS -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP) +elseif(NOT Legion_USE_CUDA) + list(APPEND legate_core_CXX_OPTIONS -UTHRUST_DEVICE_SYSTEM) + list(APPEND legate_core_CXX_OPTIONS -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_CPP) +endif() +# Or should we only do it if OpenMP and CUDA are both disabled? +# if(NOT Legion_USE_OpenMP AND (NOT Legion_USE_CUDA)) +# list(APPEND legate_core_CXX_OPTIONS -UTHRUST_DEVICE_SYSTEM) +# list(APPEND legate_core_CXX_OPTIONS -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_CPP) +# endif() + if (CMAKE_SYSTEM_NAME STREQUAL "Linux") set(platform_rpath_origin "\$ORIGIN") elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin") @@ -330,14 +334,49 @@ target_link_libraries(legate_core $ PRIVATE $) -target_compile_options(legate_core - PRIVATE "$<$:${legate_core_CXX_OPTIONS}>" - "$<$:${legate_core_CUDA_OPTIONS}>") - target_compile_definitions(legate_core PUBLIC "$<$:${legate_core_CXX_DEFS}>" "$<$:${legate_core_CUDA_DEFS}>") +############################################################################## +# - Custom User Flags -------------------------------------------------------- + +macro(legate_core_add_target_compile_option OPTION_LANG VIS OPTION_NAME) + if (NOT ("${${OPTION_NAME}}" MATCHES ".*;.*")) + # Using this form of separate_arguments() makes sure that quotes are respected when + # the list is formed. Otherwise stuff like + # + # "--compiler-options='-foo -bar -baz'" + # + # becomes + # + # --compiler-options="'-foo";"-bar";"-baz'" + # + # which is obviously not what we wanted + separate_arguments(${OPTION_NAME} NATIVE_COMMAND "${${OPTION_NAME}}") + endif() + if(${OPTION_NAME}) + target_compile_options(legate_core ${VIS} "$<$:${${OPTION_NAME}}>") + endif() +endmacro() + +macro(legate_core_add_target_link_option VIS OPTION_NAME) + if (NOT ("${${OPTION_NAME}}" MATCHES ".*;.*")) + separate_arguments(${OPTION_NAME} NATIVE_COMMAND "${${OPTION_NAME}}") + endif() + if(${OPTION_NAME}) + target_link_options(legate_core ${VIS} "${${OPTION_NAME}}") + endif() +endmacro() + +legate_core_add_target_compile_option(CXX PRIVATE legate_core_CXX_OPTIONS) +legate_core_add_target_compile_option(CUDA PRIVATE legate_core_CUDA_OPTIONS) + +legate_core_add_target_compile_option(CXX PUBLIC legate_core_CXX_FLAGS) +legate_core_add_target_compile_option(CUDA PUBLIC legate_core_CUDA_FLAGS) + +legate_core_add_target_link_option(INTERFACE legate_core_LINKER_FLAGS) + target_include_directories(legate_core PUBLIC $ @@ -345,6 +384,25 @@ target_include_directories(legate_core $ ) +if (((CMAKE_BUILD_TYPE STREQUAL "Debug") OR (CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")) + AND (CMAKE_SYSTEM_NAME STREQUAL "Darwin")) + # Both clang and gcc will automatically generate a .dSYM directory (the debug + # symbols) when creating single executables, but refuse to do so when creating + # libraries. So we must do this ourselves... + find_program(DSYMUTIL dsymutil) + if (DSYMUTIL) + get_target_property(LEGATE_CORE_LIB_DIR legate_core LIBRARY_OUTPUT_DIRECTORY) + get_target_property(LEGATE_CORE_BIN_DIR legate_core BINARY_DIR) + + add_custom_command( + TARGET legate_core POST_BUILD + COMMAND "${DSYMUTIL}" "$" + WORKING_DIRECTORY "${LEGATE_CORE_BIN_DIR}/${LEGATE_CORE_LIB_DIR}" + DEPENDS legate_core + ) + endif() +endif() + ############################################################################## # - Doxygen target------------------------------------------------------------ @@ -419,6 +477,7 @@ endif() include(CPack) include(GNUInstallDirs) + rapids_cmake_install_lib_dir(lib_dir) install(TARGETS legate_core @@ -502,6 +561,7 @@ install( FILES src/core/type/type_info.h src/core/type/type_traits.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/legate/core/type) + install( FILES src/core/utilities/debug.h src/core/utilities/dispatch.h @@ -570,8 +630,6 @@ rapids_export( LANGUAGES ${ENABLED_LANGUAES} ) -include(cmake/legate_helper_functions.cmake) - set(legate_core_ROOT ${CMAKE_CURRENT_BINARY_DIR}) if(legate_core_BUILD_TESTS) diff --git a/legate_core_python.cmake b/legate_core_python.cmake index f050bebe90..130b59136c 100644 --- a/legate_core_python.cmake +++ b/legate_core_python.cmake @@ -133,3 +133,33 @@ rapids_export( NAMESPACE legate:: DOCUMENTATION doc_string FINAL_CODE_BLOCK code_string) + +# We want to extract where the _skbuild//cmake-build folder is when +# using scikit-build to build the project. scikit-build expects all build artifacts to go +# there after configuration, but this may not always be the case. The user may pass -B +# ./path/to/other/build, or this may be set automatically by a cmake preset. +# +# The workaround is therefore to locate this cmake-build folder, and -- after +# configuration is complete -- create a symlink from the *actual* build folder to +# cmake-build. +# +# But locating cmake-build is not easy: +# +# - CMAKE_CURRENT_BINARY_DIR is unreliable since it may be overridden (as detailed above). +# - CMAKE_CURRENT_[SOURCE|LIST]_DIR don't work, they are /path/to/legate.core.internal +# (not the directory from which cmake was invoked) +# +# So the trick is to exploit the fact that scikit-build sets CMAKE_INSTALL_PREFIX to +# _skbuild//cmake-install (and enforces that the user does not override +# this! see +# https://github.com/scikit-build/scikit-build/blob/main/skbuild/cmaker.py#L321). From +# this, we can reconstruct the cmake-build path. +if(SKBUILD) + cmake_path(GET CMAKE_INSTALL_PREFIX PARENT_PATH skbuild_root_dir) + cmake_path(APPEND skbuild_root_dir "cmake-build" OUTPUT_VARIABLE skbuild_cmake_build_dir) + if (NOT (${skbuild_cmake_build_dir} STREQUAL ${CMAKE_CURRENT_BINARY_DIR})) + # The binary dir has been overridden. + file(REMOVE_RECURSE ${skbuild_cmake_build_dir}) + file(CREATE_LINK ${CMAKE_CURRENT_BINARY_DIR} ${skbuild_cmake_build_dir} SYMBOLIC) + endif() +endif() diff --git a/src/core/comm/comm_nccl.cu b/src/core/comm/comm_nccl.cu index 1c225305ca..cf20037aa7 100644 --- a/src/core/comm/comm_nccl.cu +++ b/src/core/comm/comm_nccl.cu @@ -171,7 +171,7 @@ static ncclComm_t* init_nccl(const Legion::Task* task, CHECK_CUDA(cudaEventRecord(ev_start, stream)); CHECK_NCCL(ncclGroupStart()); - for (auto idx = 0; idx < num_ranks; ++idx) { + for (std::size_t idx = 0; idx < num_ranks; ++idx) { CHECK_NCCL(ncclSend(src_buffer.ptr(0), sizeof(_Payload), ncclInt8, idx, *comm, stream)); CHECK_NCCL(ncclRecv(tgt_buffer.ptr(0), sizeof(_Payload), ncclInt8, idx, *comm, stream)); } diff --git a/src/core/comm/local_comm.cc b/src/core/comm/local_comm.cc index 99af6092d9..dea305cce6 100644 --- a/src/core/comm/local_comm.cc +++ b/src/core/comm/local_comm.cc @@ -109,7 +109,7 @@ int LocalNetwork::init_comm() { int id = 0; collGetUniqueId(&id); - assert(thread_comms.size() == id); + assert(id >= 0 && thread_comms.size() == static_cast(id)); // create thread comm ThreadComm* thread_comm = (ThreadComm*)malloc(sizeof(ThreadComm)); thread_comm->ready_flag = false; diff --git a/src/core/data/detail/logical_store.cc b/src/core/data/detail/logical_store.cc index 01c138ab15..b4efb9d81b 100644 --- a/src/core/data/detail/logical_store.cc +++ b/src/core/data/detail/logical_store.cc @@ -414,7 +414,7 @@ std::shared_ptr LogicalStore::project(int32_t d, int64_t index) if (d < 0 || d >= dim()) { throw std::invalid_argument("Invalid projection on dimension " + std::to_string(d) + " for a " + std::to_string(dim()) + "-D store"); - } else if (index < 0 || index >= old_extents[d]) { + } else if (index < 0 || index >= static_cast(old_extents[d])) { throw std::invalid_argument("Projection index " + std::to_string(index) + " is out of bounds [0, " + std::to_string(old_extents[d]) + ")"); } @@ -483,23 +483,23 @@ std::shared_ptr LogicalStore::transpose(const std::vector std::shared_ptr LogicalStore::transpose(std::vector&& axes) { - if (axes.size() != dim()) { + if (axes.size() != static_cast(dim())) { throw std::invalid_argument("Dimension Mismatch: expected " + std::to_string(dim()) + " axes, but got " + std::to_string(axes.size())); } else if (axes.size() != (std::set(axes.begin(), axes.end())).size()) { throw std::invalid_argument("Duplicate axes found"); } - for (int i = 0; i < axes.size(); i++) { - if (axes[i] < 0 || axes[i] >= dim()) { - throw std::invalid_argument("Invalid axis " + std::to_string(axes[i]) + " for a " + + for (auto&& ax : axes) { + if (ax < 0 || ax >= dim()) { + throw std::invalid_argument("Invalid axis " + std::to_string(ax) + " for a " + std::to_string(dim()) + "-D store"); } } auto old_extents = extents(); auto new_extents = Shape(); - for (int i = 0; i < axes.size(); i++) { new_extents.append_inplace(old_extents[axes[i]]); } + for (auto&& ax : axes) { new_extents.append_inplace(old_extents[ax]); } auto transform = transform_->push(std::make_unique(std::move(axes))); return std::make_shared(std::move(new_extents), storage_, std::move(transform)); @@ -520,9 +520,9 @@ std::shared_ptr LogicalStore::delinearize(int32_t idx, std::vector auto old_shape = extents(); int64_t volume = 1; - for (int i = 0; i < sizes.size(); i++) { volume *= sizes[i]; } + for (auto&& size : sizes) volume *= size; - if (old_shape[idx] != volume) { + if (static_cast(old_shape[idx]) != volume) { throw std::invalid_argument("Dimension of size " + std::to_string(old_shape[idx]) + " cannot be delinearized into shape with volume " + std::to_string(volume)); @@ -531,8 +531,10 @@ std::shared_ptr LogicalStore::delinearize(int32_t idx, std::vector auto old_extents = extents(); auto new_extents = Shape(); for (int i = 0; i < idx; i++) { new_extents.append_inplace(old_extents[i]); } - for (int i = 0; i < sizes.size(); i++) { new_extents.append_inplace(sizes[i]); } - for (int i = idx + 1; i < old_extents.size(); i++) { new_extents.append_inplace(old_extents[i]); } + for (auto&& size : sizes) { new_extents.append_inplace(size); } + for (int i = idx + 1; static_cast(i) < old_extents.size(); i++) { + new_extents.append_inplace(old_extents[i]); + } auto transform = transform_->push(std::make_unique(idx, std::move(sizes))); return std::make_shared(std::move(new_extents), storage_, std::move(transform)); diff --git a/src/core/data/detail/transform.cc b/src/core/data/detail/transform.cc index a2c377f92f..4c23baf255 100644 --- a/src/core/data/detail/transform.cc +++ b/src/core/data/detail/transform.cc @@ -821,14 +821,14 @@ proj::SymbolicPoint Delinearize::invert(const proj::SymbolicPoint& point) const { std::vector exprs; for (int32_t dim = 0; dim < dim_ + 1; ++dim) exprs.push_back(point[dim]); - for (int32_t dim = dim_ + sizes_.size(); dim < point.size(); ++dim) exprs.push_back(point[dim]); + for (auto dim = dim_ + sizes_.size(); dim < point.size(); ++dim) exprs.push_back(point[dim]); return proj::SymbolicPoint(std::move(exprs)); } Restrictions Delinearize::convert(const Restrictions& restrictions) const { std::vector result; - for (uint32_t dim = 0; dim <= dim_; ++dim) result.push_back(restrictions[dim]); + for (int32_t dim = 0; dim <= dim_; ++dim) result.push_back(restrictions[dim]); for (uint32_t idx = 1; idx < sizes_.size(); ++idx) result.push_back(Restriction::FORBID); for (uint32_t dim = dim_ + 1; dim < restrictions.size(); ++dim) result.push_back(restrictions[dim]); @@ -838,7 +838,7 @@ Restrictions Delinearize::convert(const Restrictions& restrictions) const Restrictions Delinearize::invert(const Restrictions& restrictions) const { std::vector result; - for (uint32_t dim = 0; dim <= dim_; ++dim) result.push_back(restrictions[dim]); + for (int32_t dim = 0; dim <= dim_; ++dim) result.push_back(restrictions[dim]); for (uint32_t dim = dim_ + sizes_.size(); dim < restrictions.size(); ++dim) result.push_back(restrictions[dim]); return Restrictions(std::move(result)); diff --git a/src/core/operation/detail/reduce.cc b/src/core/operation/detail/reduce.cc index 097acb4cd7..910cd3cb0d 100644 --- a/src/core/operation/detail/reduce.cc +++ b/src/core/operation/detail/reduce.cc @@ -60,7 +60,7 @@ void Reduce::launch(Strategy* p_strategy) // generating projection functions to use in tree_reduction task std::vector proj_fns; if (n_tasks > 1) { - for (size_t i = 0; i < radix_; i++) proj_fns.push_back(proj::RadixProjectionFunctor(radix_, i)); + for (auto i = 0; i < radix_; i++) proj_fns.push_back(proj::RadixProjectionFunctor(radix_, i)); } auto to_array_arg = [](auto&& arg) { return std::make_unique(std::move(arg)); }; diff --git a/src/core/partitioning/detail/constraint.cc b/src/core/partitioning/detail/constraint.cc index ce4c8988c0..d24d48627e 100644 --- a/src/core/partitioning/detail/constraint.cc +++ b/src/core/partitioning/detail/constraint.cc @@ -172,7 +172,7 @@ void ScaleConstraint::validate() const "Scaling constraint requires the stores to have the same number of dimensions"); } - if (smaller->dim() != factors_.size()) { + if (static_cast(smaller->dim()) != factors_.size()) { throw std::invalid_argument( "Scaling constraint requires the number of factors to match the number of dimensions"); } @@ -211,15 +211,17 @@ void BloatConstraint::find_partition_symbols(std::vector& parti void BloatConstraint::validate() const { - auto source = var_source_->operation()->find_store(var_source_); - auto bloat = var_bloat_->operation()->find_store(var_bloat_); + auto source = var_source_->operation()->find_store(var_source_); + auto bloat = var_bloat_->operation()->find_store(var_bloat_); + const auto sdim = source->dim(); - if (source->dim() != bloat->dim()) { + if (sdim != bloat->dim()) { throw std::invalid_argument( "Bloating constraint requires the stores to have the same number of dimensions"); } - if (source->dim() != low_offsets_.size() || source->dim() != high_offsets_.size()) { + if (const auto sdim_s = static_cast(sdim); + sdim_s != low_offsets_.size() || sdim_s != high_offsets_.size()) { throw std::invalid_argument( "Bloating constraint requires the number of offsets to match the number of dimensions"); } diff --git a/src/core/runtime/detail/partition_manager.cc b/src/core/runtime/detail/partition_manager.cc index 94e74ac8fc..963ccbe6cc 100644 --- a/src/core/runtime/detail/partition_manager.cc +++ b/src/core/runtime/detail/partition_manager.cc @@ -141,7 +141,7 @@ Shape PartitionManager::compute_launch_shape(const mapping::Machine& machine, size_t factor_prod = 1; for (auto factor : get_factors(machine)) { // Avoid exceeding the maximum number of pieces - if (factor * factor_prod > max_pieces) break; + if (factor * factor_prod > static_cast(max_pieces)) break; factor_prod *= factor; diff --git a/src/core/runtime/detail/projection.cc b/src/core/runtime/detail/projection.cc index 7c3076f1d8..0f6ca7575a 100644 --- a/src/core/runtime/detail/projection.cc +++ b/src/core/runtime/detail/projection.cc @@ -93,7 +93,7 @@ SymbolicPoint RadixProjectionFunctor::operator()(const SymbolicPoint& in_p) cons auto ndim = in_p.size(); std::vector exprs; exprs.resize(ndim); - for (int32_t dim = 0; dim < ndim; ++dim) { exprs[dim] = in_p[dim] * radix_ + offset_; } + for (std::size_t dim = 0; dim < ndim; ++dim) { exprs[dim] = in_p[dim] * radix_ + offset_; } return SymbolicPoint(std::move(exprs)); } diff --git a/src/core/type/detail/type_info.cc b/src/core/type/detail/type_info.cc index 10715aee10..ef54a6be57 100644 --- a/src/core/type/detail/type_info.cc +++ b/src/core/type/detail/type_info.cc @@ -161,7 +161,10 @@ void BinaryType::pack(BufferBuilder& buffer) const buffer.pack(size_); } -bool BinaryType::equal(const Type& other) const { return uid_ == other.uid(); } +bool BinaryType::equal(const Type& other) const +{ + return static_cast(uid_) == other.uid(); +} FixedArrayType::FixedArrayType(int32_t uid, std::shared_ptr element_type, uint32_t N) : ExtensionType(uid, Type::Code::FIXED_ARRAY), @@ -531,11 +534,9 @@ bool is_point_type(const std::shared_ptr& type, int32_t ndim) return 1 == ndim; } case Type::Code::FIXED_ARRAY: { - return type->as_fixed_array_type().num_elements() == ndim; - } - default: { - return false; + return type->as_fixed_array_type().num_elements() == static_cast(ndim); } + default: break; } return false; } diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt index 5eb2fe8fdd..a5eb731e8e 100644 --- a/tests/cpp/CMakeLists.txt +++ b/tests/cpp/CMakeLists.txt @@ -14,6 +14,8 @@ cmake_minimum_required(VERSION 3.22.1 FATAL_ERROR) project(cpp_tests VERSION 0.1 LANGUAGES C CXX) +cmake_path(SET LEGATE_CORE_DIR NORMALIZE "${CMAKE_CURRENT_SOURCE_DIR}/../../") + if (NOT CMAKE_CXX_STANDARD) set(CMAKE_CXX_STANDARD 17) endif() @@ -36,7 +38,9 @@ FetchContent_Declare( set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) FetchContent_MakeAvailable(googletest) -find_package(legate_core REQUIRED) +include(${LEGATE_CORE_DIR}/cmake/legate_helper_functions.cmake) + +legate_ensure_legate() enable_testing() @@ -53,12 +57,8 @@ enable_language(CUDA) file(GLOB integration_GPU_SRC ${PROJECT_SOURCE_DIR}/integration/*.cu) list(APPEND integration_SRC ${integration_GPU_SRC}) -if(NOT EXISTS ${CMAKE_BINARY_DIR}/RAPIDS.cmake) - file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-23.04/RAPIDS.cmake - ${CMAKE_BINARY_DIR}/RAPIDS.cmake) -endif() -include(${CMAKE_BINARY_DIR}/RAPIDS.cmake) -include(rapids-find) +legate_include_rapids() + include(cmake/thirdparty/get_nccl.cmake) endif() diff --git a/tests/cpp/CMakePresets.json b/tests/cpp/CMakePresets.json new file mode 100644 index 0000000000..523dcf20f0 --- /dev/null +++ b/tests/cpp/CMakePresets.json @@ -0,0 +1,11 @@ +{ + "version": 4, + "cmakeMinimumRequired": { + "major": 3, + "minor": 23, + "patch": 0 + }, + "include": [ + "../../CMakePresets.json" + ] +} diff --git a/tests/cpp/integration/bloat_constraints.cc b/tests/cpp/integration/bloat_constraints.cc index 2a4a0f387f..287ea3bc8f 100644 --- a/tests/cpp/integration/bloat_constraints.cc +++ b/tests/cpp/integration/bloat_constraints.cc @@ -18,8 +18,6 @@ namespace bloat_constraints { static const char* library_name = "test_bloat_constraints"; -static const int32_t TEST_MAX_DIM = 3; - static legate::Logger logger(library_name); enum TaskIDs { diff --git a/tests/cpp/integration/consensus_match.cc b/tests/cpp/integration/consensus_match.cc index 41fd6f3257..bd39bf868a 100644 --- a/tests/cpp/integration/consensus_match.cc +++ b/tests/cpp/integration/consensus_match.cc @@ -27,6 +27,7 @@ void register_tasks() { auto runtime = legate::Runtime::get_runtime(); auto context = runtime->create_library(library_name); + static_cast(context); } struct Thing { @@ -40,6 +41,7 @@ TEST(Integration, ConsensusMatch) auto runtime = legate::Runtime::get_runtime(); legate::Core::perform_registration(); auto context = runtime->find_library(library_name); + static_cast(context); Legion::Runtime* legion_runtime = Legion::Runtime::get_runtime(); Legion::Context legion_context = legion_runtime->get_context(); @@ -85,8 +87,8 @@ TEST(Integration, ConsensusMatch) Thing ta{true, -1}; Thing tb{false, -2}; EXPECT_EQ(result.output().size(), 2); - EXPECT_TRUE(result.output()[0] == ta && result.output()[1] == tb || - result.output()[0] == tb && result.output()[1] == ta); + EXPECT_TRUE((result.output()[0] == ta && result.output()[1] == tb) || + (result.output()[0] == tb && result.output()[1] == ta)); } } diff --git a/tests/cpp/integration/copy_gather_scatter.cc b/tests/cpp/integration/copy_gather_scatter.cc index a5679d73aa..22ea0cbd4e 100644 --- a/tests/cpp/integration/copy_gather_scatter.cc +++ b/tests/cpp/integration/copy_gather_scatter.cc @@ -38,6 +38,7 @@ struct CheckGatherScatterTask auto init = context.scalar(0).value(); auto src_shape = src_store.shape(); + static_cast(src_shape); auto tgt_shape = tgt_store.shape(); auto ind_shape = src_ind_store.shape(); diff --git a/tests/cpp/integration/copy_normal.cc b/tests/cpp/integration/copy_normal.cc index 877d34d1a7..d52e44f0ba 100644 --- a/tests/cpp/integration/copy_normal.cc +++ b/tests/cpp/integration/copy_normal.cc @@ -17,6 +17,14 @@ #include "copy_util.inl" +// extern so that compilers don't also complain that function is unused! +extern void silence_unused_function_warnings() +{ + // defined in copy_util.inl + static_cast(::to_string); + static_cast(::fill_indirect); +} + namespace copy_normal { static const char* library_name = "test_copy_normal"; diff --git a/tests/cpp/integration/copy_scatter.cc b/tests/cpp/integration/copy_scatter.cc index 6ea7974bbb..a0b3939720 100644 --- a/tests/cpp/integration/copy_scatter.cc +++ b/tests/cpp/integration/copy_scatter.cc @@ -110,11 +110,7 @@ struct ScatterSpec { std::string to_string() const { std::stringstream ss; - auto print_shape = [&](auto& shape) { - ss << "("; - for (auto& ext : shape) ss << ext << ","; - ss << ")"; - }; + ss << "indirection/source shape: " << ::to_string(ind_shape) << ", target shape: " << ::to_string(data_shape); ss << ", data type: " << seed.type().to_string(); diff --git a/tests/cpp/integration/field_reuse.cc b/tests/cpp/integration/field_reuse.cc index 6df63aad9b..2695806597 100644 --- a/tests/cpp/integration/field_reuse.cc +++ b/tests/cpp/integration/field_reuse.cc @@ -28,6 +28,7 @@ void register_tasks() { auto runtime = legate::Runtime::get_runtime(); auto context = runtime->create_library(library_name); + static_cast(context); } void check_field_is_new(Legion::FieldID fid) @@ -45,6 +46,7 @@ TEST(Integration, FieldReuse) auto runtime = legate::Runtime::get_runtime(); legate::Core::perform_registration(); auto context = runtime->find_library(library_name); + static_cast(context); Legion::Runtime* legion_runtime = Legion::Runtime::get_runtime(); Legion::Context legion_context = legion_runtime->get_context(); diff --git a/tests/cpp/integration/fill.cc b/tests/cpp/integration/fill.cc index cd7a2994f6..317c6c5c97 100644 --- a/tests/cpp/integration/fill.cc +++ b/tests/cpp/integration/fill.cc @@ -117,6 +117,7 @@ void test_fill_index(int32_t dim, size_t size) { auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(library_name); + static_cast(context); int64_t v = 10; @@ -143,6 +144,7 @@ void test_fill_slice(int32_t dim, size_t size) { auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(library_name); + static_cast(context); int64_t v1 = 100; int64_t v2 = 200; diff --git a/tests/cpp/integration/image_constraints.cc b/tests/cpp/integration/image_constraints.cc index cb2e831de7..49f966eced 100644 --- a/tests/cpp/integration/image_constraints.cc +++ b/tests/cpp/integration/image_constraints.cc @@ -237,6 +237,7 @@ void test_image(const ImageTestSpec& spec) { auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(library_name); + static_cast(context); int32_t tgt_dim = spec.range_extents.size(); auto image_type = spec.is_rect ? legate::rect_type(tgt_dim) : legate::point_type(tgt_dim); diff --git a/tests/cpp/integration/manual_simple.cc b/tests/cpp/integration/manual_simple.cc index 44fdc8f11e..3a630803f5 100644 --- a/tests/cpp/integration/manual_simple.cc +++ b/tests/cpp/integration/manual_simple.cc @@ -38,6 +38,7 @@ void test_manual_task(legate::Library library, legate::LogicalStore store) void print_store(legate::LogicalStore store) { auto runtime = legate::Runtime::get_runtime(); + static_cast(runtime); auto p_store = store.get_physical_store(); auto acc = p_store.read_accessor(); auto shape = p_store.shape<2>(); diff --git a/tests/cpp/integration/multi_scalar_out.cc b/tests/cpp/integration/multi_scalar_out.cc index d99a39ae60..aa5d0d223b 100644 --- a/tests/cpp/integration/multi_scalar_out.cc +++ b/tests/cpp/integration/multi_scalar_out.cc @@ -59,7 +59,8 @@ void validate_stores(legate::LogicalStore scalar1, int32_t to_match1, int64_t to_match2) { - auto runtime = legate::Runtime::get_runtime(); + auto runtime = legate::Runtime::get_runtime(); + static_cast(runtime); auto p_scalar1 = scalar1.get_physical_store(); auto p_scalar2 = scalar2.get_physical_store(); auto acc1 = p_scalar1.read_accessor(); diff --git a/tests/cpp/integration/scale_constraints.cc b/tests/cpp/integration/scale_constraints.cc index a4dd449ad0..e383e346ec 100644 --- a/tests/cpp/integration/scale_constraints.cc +++ b/tests/cpp/integration/scale_constraints.cc @@ -19,8 +19,6 @@ namespace scale_constraints { static const char* library_name = "test_scale_constraints"; -static const int32_t TEST_MAX_DIM = 3; - static legate::Logger logger(library_name); enum TaskIDs { diff --git a/tests/cpp/unit/scalar.cc b/tests/cpp/unit/scalar.cc index 4f37d06e80..4e3839c698 100644 --- a/tests/cpp/unit/scalar.cc +++ b/tests/cpp/unit/scalar.cc @@ -28,7 +28,7 @@ constexpr uint16_t UINT16_VALUE = 65535; constexpr uint32_t UINT32_VALUE = 999; constexpr uint64_t UINT64_VALUE = 100; constexpr float FLOAT_VALUE = 1.23f; -constexpr double DOUBLE_VALUE = -4.567d; +constexpr double DOUBLE_VALUE = -4.567; const std::string STRING_VALUE = "123"; const __half FLOAT16_VALUE(0.1f); const complex COMPLEX_FLOAT_VALUE{0, 1}; @@ -217,7 +217,7 @@ void check_rect_bounds(const MultiDimRectStruct& to_compare, EXPECT_EQ(std::size(expect.lo), std::size(expect.hi)); EXPECT_EQ(std::size(to_compare.lo), std::size(expect.lo)); - for (int i = 0; i < std::size(to_compare.lo); i++) { + for (std::size_t i = 0; i < std::size(to_compare.lo); i++) { EXPECT_EQ(to_compare.lo[i], expect.lo[i]); EXPECT_EQ(to_compare.hi[i], expect.hi[i]); } @@ -349,7 +349,9 @@ TEST(ScalarUnit, CreateWithVector) const auto* data = data_vec.data(); legate::Span expectedValues = legate::Span(data, scalar_data.size()); legate::Span actualValues(scalar.values()); - for (int i = 0; i < scalar_data.size(); i++) { EXPECT_EQ(actualValues[i], expectedValues[i]); } + for (std::size_t i = 0; i < scalar_data.size(); i++) { + EXPECT_EQ(actualValues[i], expectedValues[i]); + } EXPECT_EQ(actualValues.size(), expectedValues.size()); // Invalid type @@ -417,7 +419,7 @@ TEST(ScalarUnit, CreateWithPoint) } // Invalid dim - EXPECT_THROW(legate::Scalar(legate::Point<10>::ONES()), std::out_of_range); + EXPECT_THROW(legate::Scalar{legate::Point<10>::ONES()}, std::out_of_range); } TEST(ScalarUnit, CreateWithRect) diff --git a/tests/cpp/unit/scoped_allocator.cc b/tests/cpp/unit/scoped_allocator.cc index 099ad9a366..0082120691 100644 --- a/tests/cpp/unit/scoped_allocator.cc +++ b/tests/cpp/unit/scoped_allocator.cc @@ -76,10 +76,11 @@ struct ScopedAllocatorTask : public legate::LegateTask { void test_allocator( BufferOpCode op_code, legate::Memory::Kind kind, bool scoped, size_t bytes, size_t alignment = 16) { - auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library(library_name); - auto task = runtime->create_task(context, ALLOCATOR_TASK_ID); - auto part = task.declare_partition(); + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + auto task = runtime->create_task(context, ALLOCATOR_TASK_ID); + auto part = task.declare_partition(); + static_cast(part); AllocatorParams struct_data = { static_cast(op_code), static_cast(kind), scoped, alignment, bytes}; task.add_scalar_arg(legate::Scalar(struct_data, diff --git a/tests/cpp/unit/span.cc b/tests/cpp/unit/span.cc index d4057a8faf..62d6fd488f 100644 --- a/tests/cpp/unit/span.cc +++ b/tests/cpp/unit/span.cc @@ -25,7 +25,7 @@ constexpr uint16_t UINT16_VALUE = 65535; constexpr uint32_t UINT32_VALUE = 999; constexpr uint64_t UINT64_VALUE = 100; constexpr float FLOAT_VALUE = 1.23f; -constexpr double DOUBLE_VALUE = -4.567d; +constexpr double DOUBLE_VALUE = -4.567; const std::string STRING_VALUE = "123"; const __half FLOAT16_VALUE1(0.1f); const __half FLOAT16_VALUE2(0.2f); @@ -70,7 +70,7 @@ TEST(SpanUnit, Create) create( UINT64_VALUE, static_cast(UINT64_VALUE + 1), static_cast(UINT64_VALUE + 2)); create(FLOAT_VALUE, FLOAT_VALUE + 1.0f, FLOAT_VALUE + 2.0f); - create(DOUBLE_VALUE, DOUBLE_VALUE + 1.0d, DOUBLE_VALUE + 2.0d); + create(DOUBLE_VALUE, DOUBLE_VALUE + 1.0, DOUBLE_VALUE + 2.0); create(STRING_VALUE, STRING_VALUE + "1", STRING_VALUE + "2"); create(FLOAT16_VALUE1, FLOAT16_VALUE2, FLOAT16_VALUE3); create(COMPLEX_FLOAT_VALUE1, COMPLEX_FLOAT_VALUE2, COMPLEX_FLOAT_VALUE3); diff --git a/tests/integration/collective/CMakeLists.txt b/tests/integration/collective/CMakeLists.txt index f495cd8bd6..f2c72f8612 100644 --- a/tests/integration/collective/CMakeLists.txt +++ b/tests/integration/collective/CMakeLists.txt @@ -21,7 +21,11 @@ project(collective VERSION 1.0 LANGUAGES C CXX) set(CMAKE_CXX_STANDARD 17) set(BUILD_SHARED_LIBS ON) -find_package(legate_core REQUIRED) +cmake_path(SET LEGATE_CORE_DIR NORMALIZE "${CMAKE_CURRENT_SOURCE_DIR}/../../../") + +include(${LEGATE_CORE_DIR}/cmake/legate_helper_functions.cmake) + +legate_ensure_legate() legate_add_cpp_subdirectory(src TARGET collective EXPORT collective-export) diff --git a/tests/integration/region_manager/CMakeLists.txt b/tests/integration/region_manager/CMakeLists.txt index 817a877f82..b85e1faf86 100644 --- a/tests/integration/region_manager/CMakeLists.txt +++ b/tests/integration/region_manager/CMakeLists.txt @@ -17,7 +17,11 @@ project(region_manager VERSION 0.1 LANGUAGES C CXX) set(CMAKE_CXX_STANDARD 17) set(BUILD_SHARED_LIBS ON) -find_package(legate_core REQUIRED) +cmake_path(SET LEGATE_CORE_DIR NORMALIZE "${CMAKE_CURRENT_SOURCE_DIR}/../../../") + +include(${LEGATE_CORE_DIR}/cmake/legate_helper_functions.cmake) + +legate_ensure_legate() legate_add_cpp_subdirectory(src TARGET region_manager EXPORT region_manager-export) diff --git a/tests/integration/registry/CMakeLists.txt b/tests/integration/registry/CMakeLists.txt index 9370184477..65dc512122 100644 --- a/tests/integration/registry/CMakeLists.txt +++ b/tests/integration/registry/CMakeLists.txt @@ -17,7 +17,11 @@ project(registry VERSION 0.1 LANGUAGES C CXX) set(CMAKE_CXX_STANDARD 17) set(BUILD_SHARED_LIBS ON) -find_package(legate_core REQUIRED) +cmake_path(SET LEGATE_CORE_DIR NORMALIZE "${CMAKE_CURRENT_SOURCE_DIR}/../../../") + +include(${LEGATE_CORE_DIR}/cmake/legate_helper_functions.cmake) + +legate_ensure_legate() legate_add_cpp_subdirectory(src TARGET registry EXPORT registry-export) diff --git a/tests/integration/scoping/CMakeLists.txt b/tests/integration/scoping/CMakeLists.txt index eec6224a13..6251b4dc18 100644 --- a/tests/integration/scoping/CMakeLists.txt +++ b/tests/integration/scoping/CMakeLists.txt @@ -17,7 +17,11 @@ project(scoping VERSION 0.1 LANGUAGES C CXX) set(CMAKE_CXX_STANDARD 17) set(BUILD_SHARED_LIBS ON) -find_package(legate_core REQUIRED) +cmake_path(SET LEGATE_CORE_DIR NORMALIZE "${CMAKE_CURRENT_SOURCE_DIR}/../../../") + +include(${LEGATE_CORE_DIR}/cmake/legate_helper_functions.cmake) + +legate_ensure_legate() legate_add_cpp_subdirectory(src TARGET scoping EXPORT scoping-export) diff --git a/tests/integration/tree_reduce/CMakeLists.txt b/tests/integration/tree_reduce/CMakeLists.txt index e9a400279a..b837492cb7 100644 --- a/tests/integration/tree_reduce/CMakeLists.txt +++ b/tests/integration/tree_reduce/CMakeLists.txt @@ -17,7 +17,11 @@ project(tree_reduce VERSION 0.1 LANGUAGES C CXX) set(CMAKE_CXX_STANDARD 17) set(BUILD_SHARED_LIBS ON) -find_package(legate_core REQUIRED) +cmake_path(SET LEGATE_CORE_DIR NORMALIZE "${CMAKE_CURRENT_SOURCE_DIR}/../../../") + +include(${LEGATE_CORE_DIR}/cmake/legate_helper_functions.cmake) + +legate_ensure_legate() legate_add_cpp_subdirectory(src TARGET tree_reduce EXPORT tree_reduce-export) From abe3f20974cca5f0b9f5ff6f70e5ebb7dba1a1f7 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 27 Sep 2023 22:04:15 -0700 Subject: [PATCH 0233/1425] Use invalid domains instead of null pointers for serial operations (#92) --- src/core/data/detail/logical_array.cc | 12 ++--- src/core/data/detail/logical_array.h | 16 +++---- src/core/data/detail/logical_store.cc | 14 +++--- src/core/data/detail/logical_store.h | 10 ++-- src/core/operation/detail/copy.cc | 4 +- src/core/operation/detail/fill.cc | 7 +-- src/core/operation/detail/gather.cc | 4 +- src/core/operation/detail/operation.cc | 2 +- src/core/operation/detail/operation.h | 2 +- src/core/operation/detail/reduce.cc | 8 ++-- src/core/operation/detail/scatter.cc | 4 +- src/core/operation/detail/scatter_gather.cc | 4 +- src/core/operation/detail/task.cc | 19 ++++---- src/core/partitioning/detail/partitioner.cc | 33 ++++++------- src/core/partitioning/detail/partitioner.h | 4 +- src/core/partitioning/partition.cc | 16 +++---- src/core/partitioning/partition.h | 10 ++-- tests/cpp/integration/tree_reduce.cc | 52 +++++++++++++++++++-- tests/cpp/integration/tree_reduce_unique.cc | 2 +- 19 files changed, 133 insertions(+), 90 deletions(-) diff --git a/src/core/data/detail/logical_array.cc b/src/core/data/detail/logical_array.cc index c84dd35d5d..305323570c 100644 --- a/src/core/data/detail/logical_array.cc +++ b/src/core/data/detail/logical_array.cc @@ -142,7 +142,7 @@ void BaseLogicalArray::generate_constraints( std::unique_ptr BaseLogicalArray::to_launcher_arg( const std::map, const Variable*>& mapping, const Strategy& strategy, - const Domain* launch_domain, + const Domain& launch_domain, Legion::PrivilegeMode privilege, int32_t redop) const { @@ -160,7 +160,7 @@ std::unique_ptr BaseLogicalArray::to_launcher_arg( } std::unique_ptr BaseLogicalArray::to_launcher_arg_for_fixup( - const Domain* launch_domain, Legion::PrivilegeMode privilege) const + const Domain& launch_domain, Legion::PrivilegeMode privilege) const { auto data_arg = data_->to_launcher_arg_for_fixup(launch_domain, privilege); return std::make_unique(std::move(data_arg)); @@ -270,7 +270,7 @@ void ListLogicalArray::generate_constraints( std::unique_ptr ListLogicalArray::to_launcher_arg( const std::map, const Variable*>& mapping, const Strategy& strategy, - const Domain* launch_domain, + const Domain& launch_domain, Legion::PrivilegeMode privilege, int32_t redop) const { @@ -283,7 +283,7 @@ std::unique_ptr ListLogicalArray::to_launcher_arg( } std::unique_ptr ListLogicalArray::to_launcher_arg_for_fixup( - const Domain* launch_domain, Legion::PrivilegeMode privilege) const + const Domain& launch_domain, Legion::PrivilegeMode privilege) const { auto descriptor_arg = descriptor_->to_launcher_arg_for_fixup(launch_domain, READ_WRITE); auto vardata_arg = vardata_->to_launcher_arg_for_fixup(launch_domain, privilege); @@ -417,7 +417,7 @@ void StructLogicalArray::generate_constraints( std::unique_ptr StructLogicalArray::to_launcher_arg( const std::map, const Variable*>& mapping, const Strategy& strategy, - const Domain* launch_domain, + const Domain& launch_domain, Legion::PrivilegeMode privilege, int32_t redop) const { @@ -439,7 +439,7 @@ std::unique_ptr StructLogicalArray::to_launcher_arg( } std::unique_ptr StructLogicalArray::to_launcher_arg_for_fixup( - const Domain* launch_domain, Legion::PrivilegeMode privilege) const + const Domain& launch_domain, Legion::PrivilegeMode privilege) const { std::vector> field_args; for (auto& field : fields_) { diff --git a/src/core/data/detail/logical_array.h b/src/core/data/detail/logical_array.h index 17fefe80e6..83006bf9c0 100644 --- a/src/core/data/detail/logical_array.h +++ b/src/core/data/detail/logical_array.h @@ -61,11 +61,11 @@ struct LogicalArray { virtual std::unique_ptr to_launcher_arg( const std::map, const Variable*>& mapping, const Strategy& strategy, - const Domain* launch_domain, + const Domain& launch_domain, Legion::PrivilegeMode privilege, int32_t redop) const = 0; virtual std::unique_ptr to_launcher_arg_for_fixup( - const Domain* launch_domain, Legion::PrivilegeMode privilege) const = 0; + const Domain& launch_domain, Legion::PrivilegeMode privilege) const = 0; }; class BaseLogicalArray : public LogicalArray { @@ -113,11 +113,11 @@ class BaseLogicalArray : public LogicalArray { std::unique_ptr to_launcher_arg( const std::map, const Variable*>& mapping, const Strategy& strategy, - const Domain* launch_domain, + const Domain& launch_domain, Legion::PrivilegeMode privilege, int32_t redop) const override; std::unique_ptr to_launcher_arg_for_fixup( - const Domain* launch_domain, Legion::PrivilegeMode privilege) const override; + const Domain& launch_domain, Legion::PrivilegeMode privilege) const override; private: std::shared_ptr data_; @@ -173,11 +173,11 @@ class ListLogicalArray : public LogicalArray { std::unique_ptr to_launcher_arg( const std::map, const Variable*>& mapping, const Strategy& strategy, - const Domain* launch_domain, + const Domain& launch_domain, Legion::PrivilegeMode privilege, int32_t redop) const override; std::unique_ptr to_launcher_arg_for_fixup( - const Domain* launch_domain, Legion::PrivilegeMode privilege) const override; + const Domain& launch_domain, Legion::PrivilegeMode privilege) const override; private: std::shared_ptr type_; @@ -229,11 +229,11 @@ class StructLogicalArray : public LogicalArray { std::unique_ptr to_launcher_arg( const std::map, const Variable*>& mapping, const Strategy& strategy, - const Domain* launch_domain, + const Domain& launch_domain, Legion::PrivilegeMode privilege, int32_t redop) const override; std::unique_ptr to_launcher_arg_for_fixup( - const Domain* launch_domain, Legion::PrivilegeMode privilege) const override; + const Domain& launch_domain, Legion::PrivilegeMode privilege) const override; private: std::shared_ptr type_; diff --git a/src/core/data/detail/logical_store.cc b/src/core/data/detail/logical_store.cc index b4efb9d81b..25e3e33d38 100644 --- a/src/core/data/detail/logical_store.cc +++ b/src/core/data/detail/logical_store.cc @@ -306,7 +306,7 @@ Legion::LogicalPartition StoragePartition::get_legion_partition() return parent_->get_region_field()->get_legion_partition(partition_.get(), complete_); } -bool StoragePartition::is_disjoint_for(const Domain* launch_domain) const +bool StoragePartition::is_disjoint_for(const Domain& launch_domain) const { return partition_->is_disjoint_for(launch_domain); } @@ -670,12 +670,12 @@ void LogicalStore::pack(BufferBuilder& buffer) const std::unique_ptr LogicalStore::to_launcher_arg(const Variable* variable, const Strategy& strategy, - const Domain* launch_domain, + const Domain& launch_domain, Legion::PrivilegeMode privilege, int32_t redop) { if (has_scalar_storage()) { - if (nullptr == launch_domain && REDUCE == privilege) { privilege = READ_WRITE; } + if (!launch_domain.is_valid() && REDUCE == privilege) { privilege = READ_WRITE; } auto read_only = privilege == READ_ONLY; auto has_storage = privilege == READ_ONLY || privilege == READ_WRITE; return std::make_unique(this, read_only, has_storage, redop); @@ -698,7 +698,7 @@ std::unique_ptr LogicalStore::to_launcher_arg(const Variable* variab } } -std::unique_ptr LogicalStore::to_launcher_arg_for_fixup(const Domain* launch_domain, +std::unique_ptr LogicalStore::to_launcher_arg_for_fixup(const Domain& launch_domain, Legion::PrivilegeMode privilege) { if (LegateDefined(LEGATE_USE_DEBUG)) { assert(key_partition_ != nullptr); } @@ -736,7 +736,7 @@ LogicalStorePartition::LogicalStorePartition(std::shared_ptr partitio // FIXME pass projection functor std::unique_ptr LogicalStorePartition::create_projection_info( - const Domain* launch_domain, std::optional proj_fn) + const Domain& launch_domain, std::optional proj_fn) { if (store_->has_scalar_storage()) return std::make_unique(); @@ -746,11 +746,11 @@ std::unique_ptr LogicalStorePartition::create_projection_info( // created. auto legion_partition = storage_partition_->get_legion_partition(); auto proj_id = - launch_domain != nullptr ? store_->compute_projection(launch_domain->dim, proj_fn) : 0; + launch_domain.is_valid() ? store_->compute_projection(launch_domain.dim, proj_fn) : 0; return std::make_unique(legion_partition, proj_id); } -bool LogicalStorePartition::is_disjoint_for(const Domain* launch_domain) const +bool LogicalStorePartition::is_disjoint_for(const Domain& launch_domain) const { return storage_partition_->is_disjoint_for(launch_domain); } diff --git a/src/core/data/detail/logical_store.h b/src/core/data/detail/logical_store.h index 2b192aac30..a24877aea8 100644 --- a/src/core/data/detail/logical_store.h +++ b/src/core/data/detail/logical_store.h @@ -144,7 +144,7 @@ class StoragePartition : public std::enable_shared_from_this { int32_t level() const { return level_; } public: - bool is_disjoint_for(const Domain* launch_domain) const; + bool is_disjoint_for(const Domain& launch_domain) const; private: bool complete_; @@ -242,10 +242,10 @@ class LogicalStore : public std::enable_shared_from_this { void pack(BufferBuilder& buffer) const; std::unique_ptr to_launcher_arg(const Variable* variable, const Strategy& strategy, - const Domain* launch_domain, + const Domain& launch_domain, Legion::PrivilegeMode privilege, int32_t redop = -1); - std::unique_ptr to_launcher_arg_for_fixup(const Domain* launch_domain, + std::unique_ptr to_launcher_arg_for_fixup(const Domain& launch_domain, Legion::PrivilegeMode privilege); public: @@ -274,8 +274,8 @@ class LogicalStorePartition : public std::enable_shared_from_this storage_partition() const { return storage_partition_; } std::shared_ptr store() const { return store_; } std::unique_ptr create_projection_info( - const Domain* launch_domain, std::optional proj_fn = nullptr); - bool is_disjoint_for(const Domain* launch_domain) const; + const Domain& launch_domain, std::optional proj_fn = nullptr); + bool is_disjoint_for(const Domain& launch_domain) const; private: std::shared_ptr partition_; diff --git a/src/core/operation/detail/copy.cc b/src/core/operation/detail/copy.cc index 2b68e8a18c..679bc6f8d9 100644 --- a/src/core/operation/detail/copy.cc +++ b/src/core/operation/detail/copy.cc @@ -80,8 +80,8 @@ void Copy::launch(Strategy* p_strategy) launcher.add_reduction(target_.store, std::move(proj)); } - if (launch_domain != nullptr) { - return launcher.execute(*launch_domain); + if (launch_domain.is_valid()) { + return launcher.execute(launch_domain); } else { return launcher.execute_single(); } diff --git a/src/core/operation/detail/fill.cc b/src/core/operation/detail/fill.cc index f23beb6f5a..62ed2d5d64 100644 --- a/src/core/operation/detail/fill.cc +++ b/src/core/operation/detail/fill.cc @@ -56,10 +56,11 @@ void Fill::launch(Strategy* strategy) auto lhs_proj = lhs_->create_partition(part)->create_projection_info(launch_domain); lhs_->set_key_partition(machine(), part.get()); - if (nullptr == launch_domain) + if (launch_domain.is_valid()) { + launcher.launch(launch_domain, lhs_.get(), *lhs_proj, value_.get()); + } else { launcher.launch_single(lhs_.get(), *lhs_proj, value_.get()); - else - launcher.launch(*launch_domain, lhs_.get(), *lhs_proj, value_.get()); + } } std::string Fill::to_string() const { return "Fill:" + std::to_string(unique_id_); } diff --git a/src/core/operation/detail/gather.cc b/src/core/operation/detail/gather.cc index 03b17b0d5c..8a53c716ba 100644 --- a/src/core/operation/detail/gather.cc +++ b/src/core/operation/detail/gather.cc @@ -82,8 +82,8 @@ void Gather::launch(Strategy* p_strategy) launcher.set_source_indirect_out_of_range(out_of_range_); - if (launch_domain != nullptr) { - return launcher.execute(*launch_domain); + if (launch_domain.is_valid()) { + return launcher.execute(launch_domain); } else { return launcher.execute_single(); } diff --git a/src/core/operation/detail/operation.cc b/src/core/operation/detail/operation.cc index a0121f3836..a44c5f6afb 100644 --- a/src/core/operation/detail/operation.cc +++ b/src/core/operation/detail/operation.cc @@ -59,7 +59,7 @@ void Operation::record_partition(const Variable* variable, std::shared_ptr Operation::create_projection_info(const Strategy& strategy, - const Domain* launch_domain, + const Domain& launch_domain, const StoreArg& arg) const { auto store_partition = arg.store->create_partition(strategy[arg.variable]); diff --git a/src/core/operation/detail/operation.h b/src/core/operation/detail/operation.h index 929c68a117..0bedb96395 100644 --- a/src/core/operation/detail/operation.h +++ b/src/core/operation/detail/operation.h @@ -54,7 +54,7 @@ class Operation { void record_partition(const Variable* variable, std::shared_ptr store); // Helper methods std::unique_ptr create_projection_info(const Strategy& strategy, - const Domain* launch_domain, + const Domain& launch_domain, const StoreArg& arg) const; protected: diff --git a/src/core/operation/detail/reduce.cc b/src/core/operation/detail/reduce.cc index 910cd3cb0d..f4a4b1be3e 100644 --- a/src/core/operation/detail/reduce.cc +++ b/src/core/operation/detail/reduce.cc @@ -51,8 +51,8 @@ Reduce::Reduce(const Library* library, void Reduce::launch(Strategy* p_strategy) { auto& strategy = *p_strategy; - auto launch_domain = *(strategy.launch_domain(this)); - auto n_tasks = launch_domain.get_volume(); + auto launch_domain = strategy.launch_domain(this); + auto n_tasks = launch_domain.is_valid() ? launch_domain.get_volume() : 1; auto input_part = strategy[input_part_]; auto input_partition = input_->create_partition(input_part); @@ -74,13 +74,13 @@ void Reduce::launch(Strategy* p_strategy) // if there are more than 1 sub-task, we add several slices of the input // for each sub-task for (auto& proj_fn : proj_fns) { - auto proj_info = input_partition->create_projection_info(&launch_domain, proj_fn); + auto proj_info = input_partition->create_projection_info(launch_domain, proj_fn); launcher.add_input(to_array_arg( std::make_unique(input_.get(), READ_ONLY, std::move(proj_info)))); } } else { // otherwise we just add an entire input region to the task - auto proj_info = input_partition->create_projection_info(&launch_domain); + auto proj_info = input_partition->create_projection_info(launch_domain); launcher.add_input(to_array_arg( std::make_unique(input_.get(), READ_ONLY, std::move(proj_info)))); } diff --git a/src/core/operation/detail/scatter.cc b/src/core/operation/detail/scatter.cc index 0a88be1543..13f020c587 100644 --- a/src/core/operation/detail/scatter.cc +++ b/src/core/operation/detail/scatter.cc @@ -82,8 +82,8 @@ void Scatter::launch(Strategy* p_strategy) create_projection_info(strategy, launch_domain, target_indirect_)); launcher.set_target_indirect_out_of_range(out_of_range_); - if (launch_domain != nullptr) { - return launcher.execute(*launch_domain); + if (launch_domain.is_valid()) { + return launcher.execute(launch_domain); } else { return launcher.execute_single(); } diff --git a/src/core/operation/detail/scatter_gather.cc b/src/core/operation/detail/scatter_gather.cc index e348adaf14..71860cb47d 100644 --- a/src/core/operation/detail/scatter_gather.cc +++ b/src/core/operation/detail/scatter_gather.cc @@ -103,8 +103,8 @@ void ScatterGather::launch(Strategy* p_strategy) launcher.set_target_indirect_out_of_range(target_indirect_out_of_range_); launcher.set_source_indirect_out_of_range(source_indirect_out_of_range_); - if (launch_domain != nullptr) { - return launcher.execute(*launch_domain); + if (launch_domain.is_valid()) { + return launcher.execute(launch_domain); } else { return launcher.execute_single(); } diff --git a/src/core/operation/detail/task.cc b/src/core/operation/detail/task.cc index 6a5d4880b6..e5f304534d 100644 --- a/src/core/operation/detail/task.cc +++ b/src/core/operation/detail/task.cc @@ -80,7 +80,7 @@ void Task::launch_task(Strategy* p_strategy) { auto& strategy = *p_strategy; detail::TaskLauncher launcher(library_, machine_, provenance_, task_id_); - const auto* launch_domain = strategy.launch_domain(this); + auto launch_domain = strategy.launch_domain(this); for (auto& [arr, mapping] : inputs_) { launcher.add_input(arr->to_launcher_arg(mapping, strategy, launch_domain, READ_ONLY, -1)); @@ -98,23 +98,24 @@ void Task::launch_task(Strategy* p_strategy) for (auto& scalar : scalars_) launcher.add_scalar(std::move(scalar)); // Add communicators - if (launch_domain != nullptr) + if (launch_domain.is_valid()) { for (auto* factory : communicator_factories_) { auto target = machine_.preferred_target; if (!factory->is_supported_target(target)) continue; auto& processor_range = machine_.processor_range(); - auto communicator = factory->find_or_create(target, processor_range, *launch_domain); + auto communicator = factory->find_or_create(target, processor_range, launch_domain); launcher.add_communicator(communicator); if (factory->needs_barrier()) launcher.set_insert_barrier(true); } + } launcher.set_side_effect(has_side_effect_); launcher.set_concurrent(concurrent_); launcher.throws_exception(can_throw_exception_); - if (launch_domain != nullptr) { - auto result = launcher.execute(*launch_domain); - demux_scalar_stores(result, *launch_domain); + if (launch_domain.is_valid()) { + auto result = launcher.execute(launch_domain); + demux_scalar_stores(result, launch_domain); } else { auto result = launcher.execute_single(); demux_scalar_stores(result); @@ -310,15 +311,15 @@ void AutoTask::launch(Strategy* p_strategy) void AutoTask::fixup_ranges(Strategy& strategy) { - const auto* launch_domain = strategy.launch_domain(this); - if (nullptr == launch_domain) return; + auto launch_domain = strategy.launch_domain(this); + if (!launch_domain.is_valid()) return; auto* core_lib = detail::Runtime::get_runtime()->core_library(); detail::TaskLauncher launcher(core_lib, machine_, provenance_, LEGATE_CORE_FIXUP_RANGES); for (auto* array : arrays_to_fixup_) { launcher.add_output(array->to_launcher_arg_for_fixup(launch_domain, NO_ACCESS)); } - launcher.execute(*launch_domain); + launcher.execute(launch_domain); } //////////////////////////////////////////////////// diff --git a/src/core/partitioning/detail/partitioner.cc b/src/core/partitioning/detail/partitioner.cc index c24ce893cf..5a94ec67cf 100644 --- a/src/core/partitioning/detail/partitioner.cc +++ b/src/core/partitioning/detail/partitioner.cc @@ -36,7 +36,7 @@ class LaunchDomainResolver { void set_must_be_sequential(bool must_be_sequential) { must_be_sequential_ = must_be_sequential; } public: - std::unique_ptr resolve_launch_domain() const; + Domain resolve_launch_domain() const; private: bool must_be_sequential_{false}; @@ -62,25 +62,25 @@ void LaunchDomainResolver::record_unbound_store(int32_t unbound_dim) unbound_dim_ = unbound_dim; } -std::unique_ptr LaunchDomainResolver::resolve_launch_domain() const +Domain LaunchDomainResolver::resolve_launch_domain() const { - if (must_be_sequential_ || launch_domains_.empty()) return nullptr; + if (must_be_sequential_ || launch_domains_.empty()) return Domain{}; if (must_be_1d_) { if (unbound_dim_ != UNSET && unbound_dim_ > 1) - return nullptr; + return Domain{}; else { if (LegateDefined(LEGATE_USE_DEBUG)) { assert(launch_volumes_.size() == 1); } int64_t volume = *launch_volumes_.begin(); - return std::make_unique(0, volume - 1); + return Domain{0, volume - 1}; } } else { if (LegateDefined(LEGATE_USE_DEBUG)) { assert(launch_domains_.size() == 1); } auto& launch_domain = *launch_domains_.begin(); if (unbound_dim_ != UNSET && launch_domain.dim != unbound_dim_) { int64_t volume = *launch_volumes_.begin(); - return std::make_unique(0, volume - 1); + return Domain{0, volume - 1}; } - return std::make_unique(launch_domain); + return launch_domain; } } @@ -90,16 +90,12 @@ std::unique_ptr LaunchDomainResolver::resolve_launch_domain() const Strategy::Strategy() {} -bool Strategy::parallel(const Operation* op) const -{ - auto finder = launch_domains_.find(op); - return finder != launch_domains_.end() && finder->second != nullptr; -} +bool Strategy::parallel(const Operation* op) const { return launch_domain(op).is_valid(); } -const Domain* Strategy::launch_domain(const Operation* op) const +Domain Strategy::launch_domain(const Operation* op) const { auto finder = launch_domains_.find(op); - return finder != launch_domains_.end() ? finder->second.get() : nullptr; + return finder != launch_domains_.end() ? finder->second : Domain{}; } void Strategy::set_launch_shape(const Operation* op, const Shape& shape) @@ -107,7 +103,7 @@ void Strategy::set_launch_shape(const Operation* op, const Shape& shape) if (LegateDefined(LEGATE_USE_DEBUG)) { assert(launch_domains_.find(op) == launch_domains_.end()); } - launch_domains_.insert({op, std::make_unique(to_domain(shape))}); + launch_domains_.insert({op, to_domain(shape)}); } void Strategy::insert(const Variable* partition_symbol, std::shared_ptr partition) @@ -161,10 +157,11 @@ void Strategy::dump() const for (const auto& [symbol, fspace] : field_spaces_) log_legate.debug() << symbol.to_string() << ": " << fspace; for (const auto& [op, domain] : launch_domains_) { - if (nullptr == domain) + if (!domain.is_valid()) { log_legate.debug() << op->to_string() << ": (sequential)"; - else - log_legate.debug() << op->to_string() << ": " << *domain; + } else { + log_legate.debug() << op->to_string() << ": " << domain; + } } log_legate.debug("===================="); } diff --git a/src/core/partitioning/detail/partitioner.h b/src/core/partitioning/detail/partitioner.h index cc884f1148..d1beb07186 100644 --- a/src/core/partitioning/detail/partitioner.h +++ b/src/core/partitioning/detail/partitioner.h @@ -38,7 +38,7 @@ class Strategy { public: bool parallel(const Operation* op) const; - const Domain* launch_domain(const Operation* op) const; + Domain launch_domain(const Operation* op) const; void set_launch_shape(const Operation* op, const Shape& shape); public: @@ -61,7 +61,7 @@ class Strategy { private: std::map> assignments_{}; std::map field_spaces_{}; - std::map> launch_domains_{}; + std::map launch_domains_{}; std::optional key_partition_{}; }; diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index 6859b2dcbb..897193d961 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -27,9 +27,9 @@ NoPartition::NoPartition() : Partition() {} bool NoPartition::is_complete_for(const detail::Storage* storage) const { return true; } -bool NoPartition::is_disjoint_for(const Domain* launch_domain) const +bool NoPartition::is_disjoint_for(const Domain& launch_domain) const { - return nullptr == launch_domain || launch_domain->get_volume() == 1; + return !launch_domain.is_valid() || launch_domain.get_volume() == 1; } bool NoPartition::satisfies_restrictions(const Restrictions& restrictions) const { return true; } @@ -139,12 +139,12 @@ bool Tiling::is_complete_for(const detail::Storage* storage) const return true; } -bool Tiling::is_disjoint_for(const Domain* launch_domain) const +bool Tiling::is_disjoint_for(const Domain& launch_domain) const { // TODO: The check really should be that every two points from the launch domain are mapped // to two different colors return !overlapped_ && - (nullptr == launch_domain || launch_domain->get_volume() <= color_shape_.volume()); + (!launch_domain.is_valid() || launch_domain.get_volume() <= color_shape_.volume()); } bool Tiling::satisfies_restrictions(const Restrictions& restrictions) const @@ -257,11 +257,11 @@ bool Weighted::is_complete_for(const detail::Storage*) const return true; } -bool Weighted::is_disjoint_for(const Domain* launch_domain) const +bool Weighted::is_disjoint_for(const Domain& launch_domain) const { // TODO: The check really should be that every two points from the launch domain are mapped // to two different colors - return nullptr == launch_domain || launch_domain->get_volume() <= color_domain_.get_volume(); + return !launch_domain.is_valid() || launch_domain.get_volume() <= color_domain_.get_volume(); } bool Weighted::satisfies_restrictions(const Restrictions& restrictions) const @@ -344,10 +344,10 @@ bool Image::is_complete_for(const detail::Storage* storage) const return false; } -bool Image::is_disjoint_for(const Domain* launch_domain) const +bool Image::is_disjoint_for(const Domain& launch_domain) const { // Disjointness check for image partitions is expensive, so we give a sound answer; - return false; + return !launch_domain.is_valid(); } bool Image::satisfies_restrictions(const Restrictions& restrictions) const diff --git a/src/core/partitioning/partition.h b/src/core/partitioning/partition.h index 6d49fd4113..e42adb22a3 100644 --- a/src/core/partitioning/partition.h +++ b/src/core/partitioning/partition.h @@ -49,7 +49,7 @@ struct Partition { public: virtual bool is_complete_for(const detail::Storage* storage) const = 0; - virtual bool is_disjoint_for(const Domain* launch_domain) const = 0; + virtual bool is_disjoint_for(const Domain& launch_domain) const = 0; virtual bool satisfies_restrictions(const Restrictions& restrictions) const = 0; virtual bool is_convertible() const = 0; @@ -85,7 +85,7 @@ class NoPartition : public Partition { public: bool is_complete_for(const detail::Storage* storage) const override; - bool is_disjoint_for(const Domain* launch_domain) const override; + bool is_disjoint_for(const Domain& launch_domain) const override; bool satisfies_restrictions(const Restrictions& restrictions) const override; bool is_convertible() const override { return true; } @@ -132,7 +132,7 @@ class Tiling : public Partition { public: bool is_complete_for(const detail::Storage* storage) const override; - bool is_disjoint_for(const Domain* launch_domain) const override; + bool is_disjoint_for(const Domain& launch_domain) const override; bool satisfies_restrictions(const Restrictions& restrictions) const override; bool is_convertible() const override { return true; } @@ -187,7 +187,7 @@ class Weighted : public Partition { public: bool is_complete_for(const detail::Storage* storage) const override; - bool is_disjoint_for(const Domain* launch_domain) const override; + bool is_disjoint_for(const Domain& launch_domain) const override; bool satisfies_restrictions(const Restrictions& restrictions) const override; bool is_convertible() const override { return false; } @@ -234,7 +234,7 @@ class Image : public Partition { public: bool is_complete_for(const detail::Storage* storage) const override; - bool is_disjoint_for(const Domain* launch_domain) const override; + bool is_disjoint_for(const Domain& launch_domain) const override; bool satisfies_restrictions(const Restrictions& restrictions) const override; bool is_convertible() const override { return false; } diff --git a/tests/cpp/integration/tree_reduce.cc b/tests/cpp/integration/tree_reduce.cc index d9e8301975..c10450a0ad 100644 --- a/tests/cpp/integration/tree_reduce.cc +++ b/tests/cpp/integration/tree_reduce.cc @@ -87,7 +87,7 @@ void register_tasks() ReduceUnboundTask::register_variants(context); } -TEST(Integration, TreeReduceNormal) +TEST(TreeReduce, AutoProducer) { legate::Core::perform_registration(); @@ -109,7 +109,29 @@ TEST(Integration, TreeReduceNormal) EXPECT_FALSE(result.unbound()); } -TEST(Integration, TreeReduceNormalTwoSteps) +TEST(TreeReduce, ManualProducer) +{ + legate::Core::perform_registration(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + size_t num_tasks = 3; + size_t tile_size = TILE_SIZE; + + auto store = runtime->create_store({num_tasks * tile_size}, legate::int64()); + + auto task = runtime->create_task(context, TASK_PRODUCE_NORMAL, {num_tasks}); + auto part = store.partition_by_tiling({tile_size}); + task.add_output(part); + runtime->submit(std::move(task)); + + auto result = runtime->tree_reduce(context, TASK_REDUCE_NORMAL, store, 4); + + EXPECT_FALSE(result.unbound()); +} + +TEST(TreeReduce, ManualProducerMultiLevel) { legate::Core::perform_registration(); @@ -131,7 +153,7 @@ TEST(Integration, TreeReduceNormalTwoSteps) EXPECT_FALSE(result.unbound()); } -TEST(Integration, TreeReduceUnboud) +TEST(TreeReduce, ManualProducerUnbound) { legate::Core::perform_registration(); @@ -150,7 +172,7 @@ TEST(Integration, TreeReduceUnboud) EXPECT_FALSE(result.unbound()); } -TEST(Integration, TreeReduceSingleProc) +TEST(TreeReduce, ManualProducerSingle) { legate::Core::perform_registration(); @@ -168,4 +190,26 @@ TEST(Integration, TreeReduceSingleProc) EXPECT_FALSE(result.unbound()); } +TEST(TreeReduce, AutoProducerSingle) +{ + legate::Core::perform_registration(); + + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + // unbound store + auto store = runtime->create_store(legate::int64()); + + { + auto machine = runtime->get_machine(); + legate::MachineTracker tracker(machine.slice(0, 1, legate::mapping::TaskTarget::CPU)); + auto task = runtime->create_task(context, TASK_PRODUCE_UNBOUND); + task.add_output(store); + runtime->submit(std::move(task)); + } + + auto result = runtime->tree_reduce(context, TASK_REDUCE_UNBOUND, store, 4); + EXPECT_FALSE(result.unbound()); +} + } // namespace tree_reduce diff --git a/tests/cpp/integration/tree_reduce_unique.cc b/tests/cpp/integration/tree_reduce_unique.cc index 48260e7781..c2f01dfe94 100644 --- a/tests/cpp/integration/tree_reduce_unique.cc +++ b/tests/cpp/integration/tree_reduce_unique.cc @@ -104,7 +104,7 @@ void register_tasks() CheckTask::register_variants(context); } -TEST(Integration, TreeReduceUnique) +TEST(TreeReduce, Unique) { legate::Core::perform_registration(); From c42cf4eeeb5107904d33dfde108fa89bbfad7db0 Mon Sep 17 00:00:00 2001 From: Parag Kulkarni Date: Fri, 29 Sep 2023 20:57:05 +0530 Subject: [PATCH 0234/1425] Sync internal repository with legate.core TOT. (#93) * Update as per legate.core but w/o image change. Check if test cpp us built. * Use the newer image and try activating conda env. * Modify docker file to include Cuda 12.0 version. * Add LEGATE_CORE_CMAKE_PRESET in Docker env. * Keep conda activated for gcc path in test cpp build * retain build-legate-all to original. * Remove the conda deactivation in the script. --- .../workflows/ci-gh-cpu-build-and-test.yml | 11 ++-- .../workflows/ci-gh-gpu-build-and-test.yml | 10 ++-- .github/workflows/gh-build.yml | 11 ++-- .github/workflows/gh-test.yml | 51 +++---------------- continuous_integration/Dockerfile | 5 +- .../home/coder/.local/bin/build-cpp-test | 2 - 6 files changed, 31 insertions(+), 59 deletions(-) diff --git a/.github/workflows/ci-gh-cpu-build-and-test.yml b/.github/workflows/ci-gh-cpu-build-and-test.yml index 3ac2088691..07a00d6107 100644 --- a/.github/workflows/ci-gh-cpu-build-and-test.yml +++ b/.github/workflows/ci-gh-cpu-build-and-test.yml @@ -1,4 +1,4 @@ -name: Build and test CPU legate.core.internal on GH +name: Build and test CPU legate.core on GH concurrency: group: ci-cpu-on-${{ github.event_name }}-from-${{ github.ref_name }} @@ -8,7 +8,7 @@ on: push: branches: - "pull-request/[0-9]+" - - "cpp-branch-*" + - "*branch-*" jobs: build-cpu: @@ -16,13 +16,15 @@ jobs: fail-fast: false matrix: cmake-preset: [debug-gcc, debug-sanitizer-gcc] + if: ${{ github.repository_owner == 'nv-legate' }} uses: ./.github/workflows/gh-build.yml with: build-target: cpu cmake-preset: ${{ matrix.cmake-preset }} # Ref: https://docs.rapids.ai/resources/github-actions/#cpu-labels for `linux-amd64-cpu4` - runs-on: ${{ github.repository == 'nv-legate/legate.core.internal' && 'linux-amd64-cpu4' || 'ubuntu-latest' }} + repos-name: ${{ github.event.repository.name }} + runs-on: ${{ contains(github.repository, 'nv-legate/legate.core') && 'linux-amd64-cpu4' || 'ubuntu-latest' }} sha: ${{ github.sha }} test-cpu: @@ -40,7 +42,8 @@ jobs: ./.github/workflows/gh-test.yml with: build-target: cpu - runs-on: ${{ github.repository == 'nv-legate/legate.core.internal' && 'linux-amd64-cpu4' || 'ubuntu-latest' }} + repos-name: ${{ github.event.repository.name }} + runs-on: ${{ contains(github.repository, 'nv-legate/legate.core') && 'linux-amd64-cpu4' || 'ubuntu-latest' }} sha: ${{ github.sha }} cmake-preset: ${{ matrix.cmake-preset }} test-scope: ${{ matrix.test-scope }} diff --git a/.github/workflows/ci-gh-gpu-build-and-test.yml b/.github/workflows/ci-gh-gpu-build-and-test.yml index 4c301c18ad..2fac53d919 100644 --- a/.github/workflows/ci-gh-gpu-build-and-test.yml +++ b/.github/workflows/ci-gh-gpu-build-and-test.yml @@ -1,4 +1,4 @@ -name: Build and test GPU legate.core.internal on GH +name: Build and test GPU legate.core on GH concurrency: group: ci-gpu-on-${{ github.event_name }}-from-${{ github.ref_name }} @@ -8,7 +8,7 @@ on: push: branches: - "pull-request/[0-9]+" - - "cpp-branch-*" + - "*branch-*" jobs: build-gpu: @@ -16,13 +16,15 @@ jobs: fail-fast: false matrix: cmake-preset: [debug-gcc, debug-sanitizer-gcc] + if: ${{ github.repository_owner == 'nv-legate' }} uses: ./.github/workflows/gh-build.yml with: build-target: gpu cmake-preset: ${{ matrix.cmake-preset }} # Ref: https://docs.rapids.ai/resources/github-actions/#cpu-labels for `linux-amd64-cpu4` - runs-on: ${{ github.repository == 'nv-legate/legate.core.internal' && 'linux-amd64-cpu4' || 'ubuntu-latest' }} + repos-name: ${{ github.event.repository.name }} + runs-on: ${{ contains(github.repository, 'nv-legate/legate.core') && 'linux-amd64-cpu4' || 'ubuntu-latest' }} sha: ${{ github.sha }} test-gpu: @@ -37,10 +39,12 @@ jobs: - { name: Pytest Unit Tests, test-scope: unit, runner: linux-amd64-2gpu } - { name: mypy, test-scope: mypy, runner: 'linux-amd64-gpu-v100-latest-1' } name: ${{ matrix.name }} + if: ${{ github.repository_owner == 'nv-legate' }} uses: ./.github/workflows/gh-test.yml with: build-target: gpu + repos-name: ${{ github.event.repository.name }} runs-on: ${{ matrix.runner }} sha: ${{ github.sha }} cmake-preset: ${{ matrix.cmake-preset }} diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index 828eb43e44..747db7f0de 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -6,6 +6,9 @@ on: build-target: required: true type: string + repos-name: + required: true + type: string runs-on: required: true type: string @@ -18,8 +21,8 @@ on: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BASE_IMAGE: rapidsai/devcontainers:23.06-cpp-cuda11.8-mambaforge-ubuntu22.04 - IMAGE_NAME: legate.core.internal_pkg-${{ inputs.build-target }}-${{ inputs.cmake-preset }} + BASE_IMAGE: rapidsai/devcontainers:23.06-cpp-mambaforge-ubuntu22.04 + IMAGE_NAME: ${{ inputs.repos-name }}-${{ inputs.build-target }}-${{ inputs.cmake-preset }} USE_CUDA: ${{ (inputs.build-target == 'cpu' && 'OFF') || 'ON' }} LEGATE_CORE_CMAKE_PRESET: ${{ inputs.cmake-preset }} @@ -35,7 +38,7 @@ jobs: runs-on: ${{ inputs.runs-on }} steps: - - name: Checkout legate.core.internal (= this repo) + - name: Checkout ${{ inputs.repos-name }} (= this repo) uses: actions/checkout@v3 with: fetch-depth: 0 @@ -105,6 +108,6 @@ jobs: - name: Upload build artifacts uses: actions/upload-artifact@v3 with: - name: "legate.core.internal-${{ inputs.build-target }}-${{ inputs.cmake-preset }}-${{ inputs.sha }}" + name: "${{ inputs.repos-name }}-${{ inputs.build-target }}-${{ inputs.cmake-preset }}-${{ inputs.sha }}" path: | artifacts diff --git a/.github/workflows/gh-test.yml b/.github/workflows/gh-test.yml index 091cc1459d..201fb3f7b2 100644 --- a/.github/workflows/gh-test.yml +++ b/.github/workflows/gh-test.yml @@ -1,4 +1,4 @@ -name: Test legate.core.internal on GH +name: Test legate.core on GH on: workflow_call: @@ -6,6 +6,9 @@ on: build-target: required: true type: string + repos-name: + required: true + type: string runs-on: required: true type: string @@ -26,7 +29,7 @@ jobs: runs-on: ${{ inputs.runs-on }} container: options: -u root - image: ghcr.io/nv-legate/legate.core.internal_pkg-${{ inputs.build-target }}-${{ inputs.cmake-preset }}:${{ inputs.sha }} + image: ghcr.io/nv-legate/${{ inputs.repos-name }}-${{ inputs.build-target }}-${{ inputs.cmake-preset }}:${{ inputs.sha }} env: PYTHONDONTWRITEBYTECODE: 1 NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} @@ -36,48 +39,6 @@ jobs: name: Run nvidia-smi to make sure GPU is working run: nvidia-smi - - name: Checkout legate.core.internal - uses: actions/checkout@v3 - with: - repository: nv-legate/legate.core.internal - fetch-depth: 0 - path: legate - - - name: Download build artifacts - uses: actions/download-artifact@v3 - with: - name: "legate.core.internal-${{inputs.build-target}}-${{ inputs.cmake-preset }}-${{ inputs.sha }}" - path: artifacts - - - name: Display structure of downloaded artifacts - run: ls -R - working-directory: artifacts - - - name: Setup - shell: bash -eo pipefail {0} - run: | - git -C legate checkout ${{ inputs.sha }} - - cp -ar legate/continuous_integration/home/coder/.gitconfig /home/coder/ - cp -ar legate/continuous_integration/home/coder/.local /home/coder/ - - chmod a+x /home/coder/.local/bin/* - chown -R coder:coder /home/coder/ - mkdir -p /tmp/out - chown -R coder:coder /tmp/out - - - name: Create the legate conda environment - shell: bash -eo pipefail {0} - run: | - mamba create -n "${DEFAULT_CONDA_ENV:-legate}" - - - name: Install legate-core and cunumeric in the legate conda environment - shell: bash -eo pipefail {0} - run: | - mamba install -y -n "${DEFAULT_CONDA_ENV:-legate}" -c nvidia -c conda-forge -c $(pwd)/artifacts/conda-build/legate_core legate-core - mamba install --name "${DEFAULT_CONDA_ENV:-legate}" -c conda-forge pytest pytest-mock ipython jupyter_client - - - if: inputs.test-scope == 'unit' - name: Run legate.core unit tests + - name: Run legate.core test / analysis shell: su coder {0} run: run-test-or-analysis ${{ inputs.test-scope }} diff --git a/continuous_integration/Dockerfile b/continuous_integration/Dockerfile index 7b25deb306..f259ef6f53 100644 --- a/continuous_integration/Dockerfile +++ b/continuous_integration/Dockerfile @@ -10,6 +10,9 @@ ENV SCCACHE_S3_KEY_PREFIX=legate-cunumeric-dev ENV VAULT_HOST=https://vault.ops.k8s.rapids.ai ENV HISTFILE=/home/coder/.cache/._bash_history ENV DEFAULT_CONDA_ENV=legate +ENV CUDA_VERSION=12.0 +ENV CUDA_VERSION_MAJOR=12 +ENV CUDA_VERSION_MINOR=0 ARG LEGATE_CORE_CMAKE_PRESET ENV LEGATE_CORE_CMAKE_PRESET=${LEGATE_CORE_CMAKE_PRESET} @@ -69,4 +72,4 @@ WORKDIR /home/coder COPY --from=build --chown=coder:coder /tmp/out /tmp/out COPY --from=build --chown=coder:coder /tmp/conda-build /tmp/conda-build -COPY --from=build --chown=coder:coder /tmp/env_yaml /tmp/env_yaml +COPY --from=build --chown=coder:coder /tmp/env_yaml /tmp/env_yaml \ No newline at end of file diff --git a/continuous_integration/home/coder/.local/bin/build-cpp-test b/continuous_integration/home/coder/.local/bin/build-cpp-test index 043263582a..61093ae40f 100755 --- a/continuous_integration/home/coder/.local/bin/build-cpp-test +++ b/continuous_integration/home/coder/.local/bin/build-cpp-test @@ -3,8 +3,6 @@ build_cpp_tests() { set -ex; - conda deactivate - cd ~/legate/tests/cpp rm -rf build From 909d0f445c028982046573f35889f69bfa244d52 Mon Sep 17 00:00:00 2001 From: Parag Kulkarni Date: Sun, 1 Oct 2023 19:54:19 +0530 Subject: [PATCH 0235/1425] Address merge conflict. (#97) --- .github/workflows/gh-build.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index 41b3e375af..a59d3a5e9b 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -108,11 +108,5 @@ jobs: - name: Upload build artifacts uses: actions/upload-artifact@v3 with: -<<<<<<< HEAD name: "${{ inputs.repos-name }}-${{ inputs.build-target }}-${{ inputs.cmake-preset }}-${{ inputs.sha }}" - path: | - artifacts -======= - name: "${{ inputs.repos-name }}-${{ inputs.build-target }}-${{ inputs.sha }}" path: artifacts ->>>>>>> branch-23.11 From b472c7de20de5b6de5ee54563c12c4709a9c4ea5 Mon Sep 17 00:00:00 2001 From: Parag Kulkarni Date: Tue, 3 Oct 2023 21:57:38 +0530 Subject: [PATCH 0236/1425] WAR: Enforce builds on ubuntu-latest till the issue is addressed. (#102) --- .github/workflows/gh-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index a59d3a5e9b..6cf5cced6b 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -35,7 +35,7 @@ jobs: contents: read # This is required for actions/checkout packages: write # This is required to push docker image to ghcr.io - runs-on: ${{ inputs.runs-on }} + runs-on: ubuntu-latest steps: - name: Checkout ${{ inputs.repos-name }} (= this repo) From 96d03c7752fe8a320eb0e5f82e9f472e9c2afe88 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Tue, 3 Oct 2023 16:14:13 -0700 Subject: [PATCH 0237/1425] Drop registration callbacks (#101) * Remove Core::perform_registration * Turn legate::Core into implementation details --- cmake/legate_helper_functions.cmake | 5 +- legate/core/corelib.py | 2 - legate_core_cpp.cmake | 3 - src/core/comm/comm_cpu.cc | 12 +- src/core/comm/comm_nccl.cu | 14 +- src/core/cuda/stream_pool.cu | 3 +- src/core/legate_c.cc | 17 --- src/core/legate_c.h | 3 - src/core/runtime/detail/library.cc | 40 ++++-- src/core/runtime/detail/library.h | 18 +-- src/core/runtime/detail/runtime.cc | 134 ++++++++++++------ src/core/runtime/detail/runtime.h | 25 ++-- src/core/runtime/library.cc | 2 +- src/core/runtime/runtime.cc | 108 +------------- src/core/runtime/runtime.h | 43 ------ src/core/runtime/runtime.inl | 38 ----- src/core/task/task.cc | 41 +++++- src/core/utilities/machine.cc | 3 +- .../cpp/integration/alignment_constraints.cc | 10 +- tests/cpp/integration/bloat_constraints.cc | 8 +- .../cpp/integration/broadcast_constraints.cc | 6 +- tests/cpp/integration/consensus_match.cc | 2 +- tests/cpp/integration/copy_gather.cc | 10 +- tests/cpp/integration/copy_gather_scatter.cc | 10 +- tests/cpp/integration/copy_normal.cc | 4 +- tests/cpp/integration/copy_scatter.cc | 8 +- tests/cpp/integration/cpu_communicator.cc | 2 +- tests/cpp/integration/exception.cc | 6 +- tests/cpp/integration/field_reuse.cc | 2 +- tests/cpp/integration/fill.cc | 6 +- tests/cpp/integration/image_constraints.cc | 14 +- tests/cpp/integration/inline_map.cc | 2 +- tests/cpp/integration/inout.cc | 2 +- tests/cpp/integration/machine_scope.cc | 2 +- tests/cpp/integration/manual_simple.cc | 2 +- tests/cpp/integration/multi_scalar_out.cc | 2 +- tests/cpp/integration/nccl.cu | 2 +- tests/cpp/integration/provenance.cc | 2 +- tests/cpp/integration/region_manager.cc | 2 +- tests/cpp/integration/req_analyzer.cc | 4 +- tests/cpp/integration/scale_constraints.cc | 8 +- tests/cpp/integration/tree_reduce.cc | 12 +- tests/cpp/integration/tree_reduce_unique.cc | 2 +- tests/cpp/integration/weighted.cc | 4 +- tests/cpp/unit/buffer.cc | 4 +- tests/cpp/unit/constraint.cc | 10 +- tests/cpp/unit/registration.cc | 7 +- tests/cpp/unit/scoped_allocator.cc | 6 +- tests/integration/collective/src/library.cc | 8 +- .../integration/region_manager/src/library.cc | 5 +- tests/integration/registry/src/library.cc | 2 +- .../scoping/examples/test_scoping.py | 3 +- tests/integration/scoping/src/library.cc | 21 ++- tests/integration/tree_reduce/src/library.cc | 5 +- 54 files changed, 306 insertions(+), 410 deletions(-) delete mode 100644 src/core/legate_c.cc delete mode 100644 src/core/runtime/runtime.inl diff --git a/cmake/legate_helper_functions.cmake b/cmake/legate_helper_functions.cmake index 6dff7e172b..5f6d37123a 100644 --- a/cmake/legate_helper_functions.cmake +++ b/cmake/legate_helper_functions.cmake @@ -354,10 +354,7 @@ extern "C" { void @target@_perform_registration(void) { - // Tell the runtime about our registration callback so we hook it - // in before the runtime starts and make it global so that we know - // that this call back is invoked everywhere across all nodes - legate::Core::perform_registration<@target@::registration_callback>(); + @target@::registration_callback(); } } diff --git a/legate/core/corelib.py b/legate/core/corelib.py index 813dcc4823..1987eeba58 100644 --- a/legate/core/corelib.py +++ b/legate/core/corelib.py @@ -40,7 +40,6 @@ def get_c_header(self) -> str: def initialize(self, shared_lib: Any) -> None: self._lib = shared_lib - shared_lib.legate_parse_config() def get_registration_callback(self) -> str: return "legate_core_perform_registration" @@ -48,7 +47,6 @@ def get_registration_callback(self) -> str: def destroy(self) -> None: if not self._lib: raise RuntimeError("CoreLib was never initialized") - self._lib.legate_shutdown() core_library = CoreLib() diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index 82586ee805..e0487edd32 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -122,7 +122,6 @@ include(cmake/thirdparty/get_thrust.cmake) set(legate_core_SOURCES "") list(APPEND legate_core_SOURCES - src/core/legate_c.cc src/core/comm/comm.cc src/core/comm/comm_cpu.cc src/core/comm/coll.cc @@ -433,7 +432,6 @@ if (legate_core_BUILD_DOCS) # runtime src/core/runtime/library.h src/core/runtime/runtime.h - src/core/runtime/runtime.inl # operation src/core/operation/task.h # partitioning @@ -542,7 +540,6 @@ install( src/core/runtime/library.inl src/core/runtime/resource.h src/core/runtime/runtime.h - src/core/runtime/runtime.inl src/core/runtime/tracker.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/legate/core/runtime) diff --git a/src/core/comm/comm_cpu.cc b/src/core/comm/comm_cpu.cc index 60fb60d394..fc1d782894 100644 --- a/src/core/comm/comm_cpu.cc +++ b/src/core/comm/comm_cpu.cc @@ -19,6 +19,12 @@ #include "core/comm/coll.h" +namespace legate::detail { + +void show_progress(const Legion::Task* task, Legion::Context ctx, Legion::Runtime* runtime); + +} // namespace legate::detail + namespace legate::comm::cpu { class Factory : public detail::CommunicatorFactory { @@ -92,7 +98,7 @@ static int init_cpucoll_mapping(const Legion::Task* task, Legion::Context context, Legion::Runtime* runtime) { - Core::show_progress(task, context, runtime); + legate::detail::show_progress(task, context, runtime); int mpi_rank = 0; #if LegateDefined(LEGATE_USE_NETWORK) if (coll::backend_network->comm_type == coll::CollCommType::CollMPI) { @@ -108,7 +114,7 @@ static coll::CollComm init_cpucoll(const Legion::Task* task, Legion::Context context, Legion::Runtime* runtime) { - Core::show_progress(task, context, runtime); + legate::detail::show_progress(task, context, runtime); const int point = task->index_point[0]; int num_ranks = task->index_domain.get_volume(); @@ -140,7 +146,7 @@ static void finalize_cpucoll(const Legion::Task* task, Legion::Context context, Legion::Runtime* runtime) { - Core::show_progress(task, context, runtime); + legate::detail::show_progress(task, context, runtime); assert(task->futures.size() == 1); coll::CollComm comm = task->futures[0].get_result(); diff --git a/src/core/comm/comm_nccl.cu b/src/core/comm/comm_nccl.cu index cf20037aa7..c9a54b4824 100644 --- a/src/core/comm/comm_nccl.cu +++ b/src/core/comm/comm_nccl.cu @@ -26,6 +26,12 @@ #include #include +namespace legate::detail { + +void show_progress(const Legion::Task* task, Legion::Context ctx, Legion::Runtime* runtime); + +} // namespace legate::detail + namespace legate::comm::nccl { struct _Payload { @@ -117,7 +123,7 @@ static ncclUniqueId init_nccl_id(const Legion::Task* task, { legate::nvtx::Range auto_range("core::comm::nccl::init_id"); - Core::show_progress(task, context, runtime); + legate::detail::show_progress(task, context, runtime); ncclUniqueId id; CHECK_NCCL(ncclGetUniqueId(&id)); @@ -132,7 +138,7 @@ static ncclComm_t* init_nccl(const Legion::Task* task, { legate::nvtx::Range auto_range("core::comm::nccl::init"); - Core::show_progress(task, context, runtime); + legate::detail::show_progress(task, context, runtime); assert(task->futures.size() == 1); @@ -154,7 +160,7 @@ static ncclComm_t* init_nccl(const Legion::Task* task, if (num_ranks == 1) return comm; - if (!Core::warmup_nccl) return comm; + if (!detail::Config::warmup_nccl) return comm; auto stream = cuda::StreamPool::get_stream_pool().get_stream(); @@ -207,7 +213,7 @@ static void finalize_nccl(const Legion::Task* task, { legate::nvtx::Range auto_range("core::comm::nccl::finalize"); - Core::show_progress(task, context, runtime); + legate::detail::show_progress(task, context, runtime); assert(task->futures.size() == 1); auto comm = task->futures[0].get_result(); diff --git a/src/core/cuda/stream_pool.cu b/src/core/cuda/stream_pool.cu index 5bde41d062..44386a83d0 100644 --- a/src/core/cuda/stream_pool.cu +++ b/src/core/cuda/stream_pool.cu @@ -13,13 +13,14 @@ #include "core/cuda/cuda_help.h" #include "core/cuda/stream_pool.h" #include "core/mapping/machine.h" +#include "core/runtime/detail/runtime.h" #include "core/runtime/runtime.h" namespace legate::cuda { StreamView::~StreamView() { - if (valid_ && Core::synchronize_stream_view) { + if (valid_ && detail::Config::synchronize_stream_view) { if (LegateDefined(LEGATE_USE_DEBUG)) { CHECK_CUDA_STREAM(stream_); } else { diff --git a/src/core/legate_c.cc b/src/core/legate_c.cc deleted file mode 100644 index 5c84235b7f..0000000000 --- a/src/core/legate_c.cc +++ /dev/null @@ -1,17 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: LicenseRef-NvidiaProprietary - * - * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual - * property and proprietary rights in and to this material, related - * documentation and any modifications thereto. Any use, reproduction, - * disclosure or distribution of this material and related documentation - * without an express license agreement from NVIDIA CORPORATION or - * its affiliates is strictly prohibited. - */ - -#include "core/runtime/runtime.h" - -void legate_parse_config(void) { legate::Core::parse_config(); } - -void legate_shutdown(void) { legate::Core::shutdown(); } diff --git a/src/core/legate_c.h b/src/core/legate_c.h index b73e5909ad..6cd68ca546 100644 --- a/src/core/legate_c.h +++ b/src/core/legate_c.h @@ -137,9 +137,6 @@ typedef enum legate_core_reduction_op_id_t { extern "C" { #endif -void legate_parse_config(void); -void legate_shutdown(void); - void legate_core_perform_registration(void); void legate_register_affine_projection_functor( diff --git a/src/core/runtime/detail/library.cc b/src/core/runtime/detail/library.cc index 6f74dc63b0..496dc71479 100644 --- a/src/core/runtime/detail/library.cc +++ b/src/core/runtime/detail/library.cc @@ -15,16 +15,15 @@ #include "core/mapping/detail/base_mapper.h" #include "core/mapping/machine.h" #include "core/mapping/mapping.h" +#include "core/runtime/detail/runtime.h" #include "core/runtime/runtime.h" #include "mappers/logging_wrapper.h" namespace legate::detail { -Library::Library(const std::string& library_name, - const ResourceConfig& config, - std::unique_ptr mapper) - : runtime_(Legion::Runtime::get_runtime()), library_name_(library_name) +Library::Library(const std::string& library_name, const ResourceConfig& config) + : runtime_(Legion::Runtime::get_runtime()), library_name_(library_name), legion_mapper_{nullptr} { task_scope_ = ResourceIdScope( runtime_->generate_library_task_ids(library_name.c_str(), config.max_tasks), config.max_tasks); @@ -38,8 +37,6 @@ Library::Library(const std::string& library_name, runtime_->generate_library_sharding_ids(library_name.c_str(), config.max_shardings), config.max_shardings); mapper_id_ = runtime_->generate_library_mapper_ids(library_name.c_str(), 1); - - register_mapper(std::move(mapper)); } const std::string& Library::get_library_name() const { return library_name_; } @@ -122,17 +119,38 @@ const std::string& Library::get_task_name(int64_t local_task_id) const return find_task(local_task_id)->name(); } -void Library::register_mapper(std::unique_ptr mapper) +namespace { + +void register_mapper_callback(const Legion::RegistrationCallbackArgs& args) +{ + const std::string library_name(static_cast(args.buffer.get_ptr())); + + auto* library = Runtime::get_runtime()->find_library(library_name, false /*can_fail*/); + auto* legion_mapper = library->get_legion_mapper(); + if (LegateDefined(LEGATE_USE_DEBUG)) { assert(legion_mapper != nullptr); } + Legion::Runtime::get_runtime()->add_mapper(library->get_mapper_id(), legion_mapper); +} + +} // namespace + +void Library::register_mapper(std::unique_ptr mapper, bool in_callback) { // Hold the pointer to the mapper to keep it alive mapper_ = std::move(mapper); auto base_mapper = new mapping::detail::BaseMapper(mapper_.get(), runtime_->get_mapper_runtime(), this); - Legion::Mapping::Mapper* legion_mapper = base_mapper; - if (Core::log_mapping_decisions) - legion_mapper = new Legion::Mapping::LoggingWrapper(base_mapper, &base_mapper->logger); - runtime_->add_mapper(mapper_id_, legion_mapper); + legion_mapper_ = base_mapper; + if (Config::log_mapping_decisions) + legion_mapper_ = new Legion::Mapping::LoggingWrapper(base_mapper, &base_mapper->logger); + + if (in_callback) { + Legion::Runtime::get_runtime()->add_mapper(get_mapper_id(), legion_mapper_); + } else { + Legion::UntypedBuffer args{library_name_.c_str(), library_name_.size() + 1}; + Legion::Runtime::perform_registration_callback( + register_mapper_callback, args, true /*global*/, false /*duplicate*/); + } } void Library::register_task(int64_t local_task_id, std::unique_ptr task_info) diff --git a/src/core/runtime/detail/library.h b/src/core/runtime/detail/library.h index 908b5cf78c..4a5dd97e11 100644 --- a/src/core/runtime/detail/library.h +++ b/src/core/runtime/detail/library.h @@ -60,9 +60,7 @@ class Library { private: friend class Runtime; - Library(const std::string& library_name, - const ResourceConfig& config, - std::unique_ptr mapper); + Library(const std::string& library_name, const ResourceConfig& config); public: Library(const Library&) = delete; @@ -94,23 +92,14 @@ class Library { int64_t get_new_task_id() { return task_scope_.generate_id(); } public: - /** - * @brief Returns the name of a task - * - * @param local_task_id Task id - * @return Name of the task - */ const std::string& get_task_name(int64_t local_task_id) const; - void register_mapper(std::unique_ptr mapper); + void register_mapper(std::unique_ptr mapper, bool in_callback); + Legion::Mapping::Mapper* get_legion_mapper() const { return legion_mapper_; } public: void register_task(int64_t local_task_id, std::unique_ptr task_info); const TaskInfo* find_task(int64_t local_task_id) const; - private: - void perform_callback(Legion::RegistrationWithArgsCallbackFnptr callback, - Legion::UntypedBuffer buffer); - private: Legion::Runtime* runtime_; const std::string library_name_; @@ -122,6 +111,7 @@ class Library { private: Legion::MapperID mapper_id_; std::unique_ptr mapper_; + Legion::Mapping::Mapper* legion_mapper_; std::unordered_map> tasks_; }; diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index ce49066bd8..cb5213d2d4 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -44,17 +44,30 @@ namespace legate { Logger log_legate("legate"); -// This is the unique string name for our library which can be used from both C++ and Python to -// generate IDs -const char* const core_library_name = "legate.core"; - } // namespace legate namespace legate::detail { +void show_progress(const Legion::Task* task, Legion::Context ctx, Legion::Runtime* runtime); + +/*static*/ bool Config::show_progress_requested = false; + +/*static*/ bool Config::use_empty_task = false; + +/*static*/ bool Config::synchronize_stream_view = false; + +/*static*/ bool Config::log_mapping_decisions = false; + +/*static*/ bool Config::has_socket_mem = false; + +/*static*/ bool Config::warmup_nccl = false; + namespace { -const char* TOPLEVEL_NAME = "Legate Core Toplevel Task"; +// This is the unique string name for our library which can be used from both C++ and Python to +// generate IDs +const char* const CORE_LIBRARY_NAME = "legate.core"; +const char* const TOPLEVEL_NAME = "Legate Core Toplevel Task"; } // namespace @@ -72,15 +85,17 @@ Runtime::~Runtime() {} Library* Runtime::create_library(const std::string& library_name, const ResourceConfig& config, - std::unique_ptr mapper) + std::unique_ptr mapper, + bool in_callback) { if (libraries_.find(library_name) != libraries_.end()) throw std::invalid_argument("Library " + library_name + " already exists"); log_legate.debug("Library %s is created", library_name.c_str()); if (nullptr == mapper) mapper = std::make_unique(); - auto context = new Library(library_name, config, std::move(mapper)); + auto context = new Library(library_name, config); libraries_[library_name] = context; + context->register_mapper(std::move(mapper), in_callback); return context; } @@ -97,14 +112,15 @@ Library* Runtime::find_library(const std::string& library_name, bool can_fail /* Library* Runtime::find_or_create_library(const std::string& library_name, const ResourceConfig& config, std::unique_ptr mapper, - bool* created) + bool* created, + bool in_callback) { Library* result = find_library(library_name, true /*can_fail*/); if (result != nullptr) { if (created != nullptr) *created = false; return result; } - result = create_library(library_name, config, std::move(mapper)); + result = create_library(library_name, config, std::move(mapper), in_callback); if (created != nullptr) *created = true; return result; } @@ -151,12 +167,12 @@ void Runtime::initialize(Legion::Context legion_context) if (initialized_) throw std::runtime_error("Legate runtime has already been initialized"); initialized_ = true; legion_context_ = legion_context; - core_library_ = find_library(core_library_name, false /*can_fail*/); + core_library_ = find_library(CORE_LIBRARY_NAME, false /*can_fail*/); communicator_manager_ = new CommunicatorManager(); partition_manager_ = new PartitionManager(this); machine_manager_ = new MachineManager(); provenance_manager_ = new ProvenanceManager(); - Core::has_socket_mem = + Config::has_socket_mem = get_tunable(core_library_->get_mapper_id(), LEGATE_CORE_TUNABLE_HAS_SOCKET_MEM); initialize_toplevel_machine(); comm::register_builtin_communicator_factories(core_library_); @@ -1000,7 +1016,8 @@ MachineManager* Runtime::machine_manager() const { return machine_manager_; } if (!Legion::Runtime::has_runtime()) { Legion::Runtime::initialize(&argc, &argv, true /*filter legion and realm args*/); - Legion::Runtime::add_registration_callback(registration_callback); + Legion::Runtime::perform_registration_callback(initialize_core_library_callback, + true /*global*/); handle_legate_args(argc, argv); @@ -1009,8 +1026,10 @@ MachineManager* Runtime::machine_manager() const { return machine_manager_; } log_legate.error("Legion Runtime failed to start."); return result; } - } else - Legion::Runtime::perform_registration_callback(registration_callback, true /*global*/); + } else { + Legion::Runtime::perform_registration_callback(initialize_core_library_callback, + true /*global*/); + } // Get the runtime now that we've started it auto legion_runtime = Legion::Runtime::get_runtime(); @@ -1018,9 +1037,9 @@ MachineManager* Runtime::machine_manager() const { return machine_manager_; } Legion::Context legion_context; // If the context already exists, that means that some other driver started the top-level task, // so here we just grab it to initialize the Legate runtime - if (Legion::Runtime::has_context()) + if (Legion::Runtime::has_context()) { legion_context = Legion::Runtime::get_context(); - else { + } else { // Otherwise we make this thread into an implicit top-level task legion_context = legion_runtime->begin_implicit_task(LEGATE_CORE_TOPLEVEL_TASK_ID, 0 /*mapper id*/, @@ -1104,7 +1123,7 @@ static void extract_scalar_task( Legion::Runtime* runtime; Legion::Runtime::legion_task_preamble(args, arglen, p, task, regions, legion_context, runtime); - Core::show_progress(task, legion_context, runtime); + legate::detail::show_progress(task, legion_context, runtime); detail::TaskContext context(task, *regions); auto idx = context.scalars()[0].value(); @@ -1114,9 +1133,7 @@ static void extract_scalar_task( value_and_size.finalize(legion_context); } -void register_legate_core_tasks(Legion::Machine machine, - Legion::Runtime* runtime, - Library* core_lib) +void register_legate_core_tasks(Legion::Runtime* runtime, Library* core_lib) { auto task_info = std::make_unique("core::extract_scalar"); auto register_extract_scalar_variant = [&](auto variant_id) { @@ -1183,10 +1200,56 @@ void register_builtin_reduction_ops() extern void register_exception_reduction_op(Legion::Runtime* runtime, const Library* context); -void core_library_registration(Legion::Machine machine, - Legion::Runtime* legion_runtime, - const std::set& local_procs) +namespace { + +void parse_config() { + if (!LegateDefined(LEGATE_USE_CUDA)) { + const char* need_cuda = getenv("LEGATE_NEED_CUDA"); + if (need_cuda != nullptr) { + fprintf(stderr, + "Legate was run with GPUs but was not built with GPU support. " + "Please install Legate again with the \"--cuda\" flag.\n"); + exit(1); + } + } + if (!LegateDefined(LEGATE_USE_OPENMP)) { + const char* need_openmp = getenv("LEGATE_NEED_OPENMP"); + if (need_openmp != nullptr) { + fprintf(stderr, + "Legate was run on multiple nodes but was not built with networking " + "support. Please install Legate again with \"--network\".\n"); + exit(1); + } + } + if (!LegateDefined(LEGATE_USE_NETWORK)) { + const char* need_network = getenv("LEGATE_NEED_NETWORK"); + if (need_network != nullptr) { + fprintf(stderr, + "Legate was run on multiple nodes but was not built with networking " + "support. Please install Legate again with \"--network\".\n"); + exit(1); + } + } + + auto parse_variable = [](const char* variable, bool& result) { + const char* value = getenv(variable); + if (value != nullptr && atoi(value) > 0) result = true; + }; + + parse_variable("LEGATE_SHOW_PROGRESS", Config::show_progress_requested); + parse_variable("LEGATE_EMPTY_TASK", Config::use_empty_task); + parse_variable("LEGATE_SYNC_STREAM_VIEW", Config::synchronize_stream_view); + parse_variable("LEGATE_LOG_MAPPING", Config::log_mapping_decisions); + parse_variable("LEGATE_WARMUP_NCCL", Config::warmup_nccl); +} + +} // namespace + +void initialize_core_library() +{ + parse_config(); + ResourceConfig config; config.max_tasks = LEGATE_CORE_NUM_TASK_IDS; config.max_projections = LEGATE_CORE_MAX_FUNCTOR_ID; @@ -1195,9 +1258,10 @@ void core_library_registration(Legion::Machine machine, config.max_reduction_ops = LEGATE_CORE_MAX_REDUCTION_OP_ID; auto core_lib = Runtime::get_runtime()->create_library( - core_library_name, config, mapping::detail::create_core_mapper()); + CORE_LIBRARY_NAME, config, mapping::detail::create_core_mapper(), true /*in_callback*/); - register_legate_core_tasks(machine, legion_runtime, core_lib); + auto legion_runtime = Legion::Runtime::get_runtime(); + register_legate_core_tasks(legion_runtime, core_lib); register_builtin_reduction_ops(); @@ -1208,22 +1272,9 @@ void core_library_registration(Legion::Machine machine, register_legate_core_sharding_functors(legion_runtime, core_lib); } -void registration_callback(Legion::Machine machine, - Legion::Runtime* legion_runtime, - const std::set& local_procs) -{ - core_library_registration(machine, legion_runtime, local_procs); - - Core::parse_config(); -} - -void registration_callback_for_python(Legion::Machine machine, - Legion::Runtime* legion_runtime, - const std::set& local_procs) +void initialize_core_library_callback(Legion::Machine, Legion::Runtime*, const std::set&) { - core_library_registration(machine, legion_runtime, local_procs); - - Runtime::get_runtime()->initialize(Legion::Runtime::get_context()); + initialize_core_library(); } // Simple wrapper for variables with default values @@ -1337,11 +1388,14 @@ void handle_legate_args(int32_t argc, char** argv) try_set_property(rt, "cuda", "fbmem", fbmem, "unable to set --fbmem"); try_set_property(rt, "cuda", "zcmem", zcmem, "unable to set --zcmem"); + if (gpus.value() > 0) { setenv("LEGATE_NEED_CUDA", "1", true); } + // Set OpenMP configuration properties if (omps.value() > 0 && ompthreads.value() == 0) { log_legate.error("--omps configured with zero threads"); LEGATE_ABORT; } + if (omps.value() > 0) { setenv("LEGATE_NEED_OPENMP", "1", true); } try_set_property(rt, "openmp", "ocpu", omps, "unable to set --omps"); try_set_property(rt, "openmp", "othr", ompthreads, "unable to set --ompthreads"); diff --git a/src/core/runtime/detail/runtime.h b/src/core/runtime/detail/runtime.h index e2bf15afed..dc0be9c07b 100644 --- a/src/core/runtime/detail/runtime.h +++ b/src/core/runtime/detail/runtime.h @@ -47,6 +47,15 @@ class ManualTask; class Operation; class StructLogicalArray; +struct Config { + static bool show_progress_requested; + static bool use_empty_task; + static bool synchronize_stream_view; + static bool log_mapping_decisions; + static bool has_socket_mem; + static bool warmup_nccl; +}; + class Runtime { public: Runtime(); @@ -55,12 +64,14 @@ class Runtime { public: Library* create_library(const std::string& library_name, const ResourceConfig& config, - std::unique_ptr mapper); + std::unique_ptr mapper, + bool in_callback); Library* find_library(const std::string& library_name, bool can_fail) const; Library* find_or_create_library(const std::string& library_name, const ResourceConfig& config, std::unique_ptr mapper, - bool* created); + bool* created, + bool in_callback); public: void record_reduction_operator(int32_t type_uid, int32_t op_kind, int32_t legion_op_id); @@ -325,13 +336,11 @@ class Runtime { std::deque outstanding_exceptions_{}; }; -void registration_callback(Legion::Machine machine, - Legion::Runtime* legion_runtime, - const std::set& local_procs); +void initialize_core_library(); -void registration_callback_for_python(Legion::Machine machine, - Legion::Runtime* legion_runtime, - const std::set& local_procs); +void initialize_core_library_callback(Legion::Machine, + Legion::Runtime*, + const std::set&); void handle_legate_args(int32_t argc, char** argv); diff --git a/src/core/runtime/library.cc b/src/core/runtime/library.cc index 398da76196..9cfbe1b55b 100644 --- a/src/core/runtime/library.cc +++ b/src/core/runtime/library.cc @@ -87,7 +87,7 @@ const std::string& Library::get_task_name(int64_t local_task_id) const void Library::register_mapper(std::unique_ptr mapper) { - impl_->register_mapper(std::move(mapper)); + impl_->register_mapper(std::move(mapper), false /*in_callback*/); } void Library::register_task(int64_t local_task_id, std::unique_ptr task_info) diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 1f2b4559ae..7ec611c25b 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -22,105 +22,6 @@ namespace legate { extern Logger log_legate; -/*static*/ bool Core::show_progress_requested = false; - -/*static*/ bool Core::use_empty_task = false; - -/*static*/ bool Core::synchronize_stream_view = false; - -/*static*/ bool Core::log_mapping_decisions = false; - -/*static*/ bool Core::has_socket_mem = false; - -/*static*/ bool Core::warmup_nccl = false; - -/*static*/ void Core::parse_config(void) -{ - if (!LegateDefined(LEGATE_USE_CUDA)) { - const char* need_cuda = getenv("LEGATE_NEED_CUDA"); - if (need_cuda != nullptr) { - fprintf(stderr, - "Legate was run with GPUs but was not built with GPU support. " - "Please install Legate again with the \"--cuda\" flag.\n"); - exit(1); - } - } - if (!LegateDefined(LEGATE_USE_OPENMP)) { - const char* need_openmp = getenv("LEGATE_NEED_OPENMP"); - if (need_openmp != nullptr) { - fprintf(stderr, - "Legate was run on multiple nodes but was not built with networking " - "support. Please install Legate again with \"--network\".\n"); - exit(1); - } - } - if (!LegateDefined(LEGATE_USE_NETWORK)) { - const char* need_network = getenv("LEGATE_NEED_NETWORK"); - if (need_network != nullptr) { - fprintf(stderr, - "Legate was run on multiple nodes but was not built with networking " - "support. Please install Legate again with \"--network\".\n"); - exit(1); - } - } - - auto parse_variable = [](const char* variable, bool& result) { - const char* value = getenv(variable); - if (value != nullptr && atoi(value) > 0) result = true; - }; - - parse_variable("LEGATE_SHOW_PROGRESS", show_progress_requested); - parse_variable("LEGATE_EMPTY_TASK", use_empty_task); - parse_variable("LEGATE_SYNC_STREAM_VIEW", synchronize_stream_view); - parse_variable("LEGATE_LOG_MAPPING", log_mapping_decisions); - parse_variable("LEGATE_WARMUP_NCCL", warmup_nccl); -} - -/*static*/ void Core::shutdown(void) -{ - // Nothing to do here yet... -} - -/*static*/ void Core::show_progress(const Legion::Task* task, - Legion::Context ctx, - Legion::Runtime* runtime) -{ - if (!Core::show_progress_requested) return; - const auto exec_proc = runtime->get_executing_processor(ctx); - const auto proc_kind_str = (exec_proc.kind() == Processor::LOC_PROC) ? "CPU" - : (exec_proc.kind() == Processor::TOC_PROC) ? "GPU" - : "OpenMP"; - - std::stringstream point_str; - const auto& point = task->index_point; - point_str << point[0]; - for (int32_t dim = 1; dim < point.dim; ++dim) point_str << "," << point[dim]; - - log_legate.print("%s %s task [%s], pt = (%s), proc = " IDFMT, - task->get_task_name(), - proc_kind_str, - task->get_provenance_string().c_str(), - point_str.str().c_str(), - exec_proc.id); -} - -/*static*/ void Core::report_unexpected_exception(const Legion::Task* task, - const legate::TaskException& e) -{ - log_legate.error( - "Task %s threw an exception \"%s\", but the task did not declare any exception. " - "Please specify a Python exception that you want this exception to be re-thrown with " - "using 'throws_exception'.", - task->get_task_name(), - e.error_message().c_str()); - LEGATE_ABORT; -} - -/*static*/ void Core::perform_callback(Legion::RegistrationCallbackFnptr callback) -{ - Legion::Runtime::perform_registration_callback(callback, true /*global*/); -} - Library Runtime::find_library(const std::string& library_name) const { return Library(impl_->find_library(library_name, false)); @@ -136,7 +37,8 @@ Library Runtime::create_library(const std::string& library_name, const ResourceConfig& config, std::unique_ptr mapper) { - return Library(impl_->create_library(library_name, config, std::move(mapper))); + return Library( + impl_->create_library(library_name, config, std::move(mapper), false /*in_callback*/)); } Library Runtime::find_or_create_library(const std::string& library_name, @@ -144,7 +46,8 @@ Library Runtime::find_or_create_library(const std::string& library_name, std::unique_ptr mapper, bool* created) { - return Library(impl_->find_or_create_library(library_name, config, std::move(mapper), created)); + return Library(impl_->find_or_create_library( + library_name, config, std::move(mapper), created, false /*in_callback*/)); } AutoTask Runtime::create_task(Library library, int64_t task_id) @@ -335,7 +238,8 @@ void legate_core_perform_registration() { // Tell the runtime about our registration callback so we can register ourselves // Make sure it is global so this shared object always gets loaded on all nodes - Legion::Runtime::perform_registration_callback(legate::detail::registration_callback_for_python, + Legion::Runtime::perform_registration_callback(legate::detail::initialize_core_library_callback, true /*global*/); + legate::detail::Runtime::get_runtime()->initialize(Legion::Runtime::get_context()); } } diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index e79d014666..409975f9db 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -43,47 +43,6 @@ extern uint32_t extract_env(const char* env_name, const uint32_t default_value, const uint32_t test_value); -/** - * @ingroup runtime - * @brief A utility class that collects static members shared by all Legate libraries - */ -struct Core { - public: - static void parse_config(void); - static void shutdown(void); - static void show_progress(const Legion::Task* task, - Legion::Context ctx, - Legion::Runtime* runtime); - static void report_unexpected_exception(const Legion::Task* task, const TaskException& e); - - public: - /** - * @brief Type signature for registration callbacks - */ - using RegistrationCallback = void (*)(); - - /** - * @brief Performs a registration callback. Libraries must perform - * registration of tasks and other components through this function. - * - * @tparam CALLBACK Registration callback to perform - */ - template - static void perform_registration(); - - private: - static void perform_callback(Legion::RegistrationCallbackFnptr callback); - - public: - // Configuration settings - static bool show_progress_requested; - static bool use_empty_task; - static bool synchronize_stream_view; - static bool log_mapping_decisions; - static bool has_socket_mem; - static bool warmup_nccl; -}; - /** * @ingroup runtime * @brief Class that implements the Legate runtime @@ -509,5 +468,3 @@ int32_t finish(); mapping::Machine get_machine(); } // namespace legate - -#include "core/runtime/runtime.inl" diff --git a/src/core/runtime/runtime.inl b/src/core/runtime/runtime.inl deleted file mode 100644 index 3d895c48f7..0000000000 --- a/src/core/runtime/runtime.inl +++ /dev/null @@ -1,38 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: LicenseRef-NvidiaProprietary - * - * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual - * property and proprietary rights in and to this material, related - * documentation and any modifications thereto. Any use, reproduction, - * disclosure or distribution of this material and related documentation - * without an express license agreement from NVIDIA CORPORATION or - * its affiliates is strictly prohibited. - */ - -#pragma once - -// Useful for IDEs -#include "core/runtime/runtime.h" - -namespace legate { - -namespace detail { - -template -void invoke_legate_registration_callback(Legion::Machine, - Legion::Runtime*, - const std::set&) -{ - CALLBACK(); -}; - -} // namespace detail - -template -/*static*/ void Core::perform_registration() -{ - perform_callback(detail::invoke_legate_registration_callback); -} - -} // namespace legate diff --git a/src/core/task/task.cc b/src/core/task/task.cc index d67eade3e9..9029a90471 100644 --- a/src/core/task/task.cc +++ b/src/core/task/task.cc @@ -16,6 +16,7 @@ #include "realm/faults.h" +#include "core/runtime/detail/runtime.h" #include "core/runtime/runtime.h" #include "core/task/detail/return.h" #include "core/task/detail/task_context.h" @@ -26,8 +27,35 @@ #include "core/utilities/nvtx_help.h" #include "core/utilities/typedefs.h" +namespace legate { + +extern Logger log_legate; + +} // namespace legate + namespace legate::detail { +void show_progress(const Legion::Task* task, Legion::Context ctx, Legion::Runtime* runtime) +{ + if (!Config::show_progress_requested) return; + const auto exec_proc = runtime->get_executing_processor(ctx); + const auto proc_kind_str = (exec_proc.kind() == Processor::LOC_PROC) ? "CPU" + : (exec_proc.kind() == Processor::TOC_PROC) ? "GPU" + : "OpenMP"; + + std::stringstream point_str; + const auto& point = task->index_point; + point_str << point[0]; + for (int32_t dim = 1; dim < point.dim; ++dim) point_str << "," << point[dim]; + + log_legate.print("%s %s task [%s], pt = (%s), proc = " IDFMT, + task->get_task_name(), + proc_kind_str, + task->get_provenance_string().c_str(), + point_str.str().c_str(), + exec_proc.id); +} + std::string generate_task_name(const std::type_info& ti) { std::string result; @@ -62,23 +90,28 @@ void task_wrapper(VariantImpl variant_impl, nvtx::Range auto_range(msg.c_str()); } - Core::show_progress(task, legion_context, runtime); + show_progress(task, legion_context, runtime); detail::TaskContext context(task, *regions); ReturnValues return_values{}; try { legate::TaskContext ctx(&context); - if (!Core::use_empty_task) (*variant_impl)(ctx); + if (!Config::use_empty_task) (*variant_impl)(ctx); return_values = context.pack_return_values(); } catch (legate::TaskException& e) { if (context.can_raise_exception()) { context.make_all_unbound_stores_empty(); return_values = context.pack_return_values_with_exception(e.index(), e.error_message()); - } else + } else { // If a Legate exception is thrown by a task that does not declare any exception, // this is a bug in the library that needs to be reported to the developer - Core::report_unexpected_exception(task, e); + log_legate.error( + "Task %s threw an exception \"%s\", but the task did not declare any exception.", + task->get_task_name(), + e.error_message().c_str()); + LEGATE_ABORT; + } } // Legion postamble diff --git a/src/core/utilities/machine.cc b/src/core/utilities/machine.cc index 42e803a1d9..195e54fb02 100644 --- a/src/core/utilities/machine.cc +++ b/src/core/utilities/machine.cc @@ -12,6 +12,7 @@ #include "core/utilities/machine.h" +#include "core/runtime/detail/runtime.h" #include "core/runtime/runtime.h" #include "legate_defines.h" @@ -28,7 +29,7 @@ Memory::Kind find_memory_kind_for_executing_processor(bool host_accessible) return host_accessible ? Memory::Kind::Z_COPY_MEM : Memory::Kind::GPU_FB_MEM; } case Processor::Kind::OMP_PROC: { - return Core::has_socket_mem ? Memory::Kind::SOCKET_MEM : Memory::Kind::SYSTEM_MEM; + return detail::Config::has_socket_mem ? Memory::Kind::SOCKET_MEM : Memory::Kind::SYSTEM_MEM; } default: break; } diff --git a/tests/cpp/integration/alignment_constraints.cc b/tests/cpp/integration/alignment_constraints.cc index 9e9d5d435b..6b51ef90cd 100644 --- a/tests/cpp/integration/alignment_constraints.cc +++ b/tests/cpp/integration/alignment_constraints.cc @@ -228,31 +228,31 @@ void test_invalid_alignment() TEST(Alignment, Basic) { - legate::Core::perform_registration(); + prepare(); test_alignment(); } TEST(Alignment, WithBroadcast) { - legate::Core::perform_registration(); + prepare(); test_alignment_and_broadcast(); } TEST(Alignment, WithTransform) { - legate::Core::perform_registration(); + prepare(); test_alignment_transformed(); } TEST(Alignment, Redundant) { - legate::Core::perform_registration(); + prepare(); test_redundant_alignment(); } TEST(Alignment, Invalid) { - legate::Core::perform_registration(); + prepare(); test_invalid_alignment(); } diff --git a/tests/cpp/integration/bloat_constraints.cc b/tests/cpp/integration/bloat_constraints.cc index 287ea3bc8f..4b16351ea8 100644 --- a/tests/cpp/integration/bloat_constraints.cc +++ b/tests/cpp/integration/bloat_constraints.cc @@ -133,25 +133,25 @@ void test_invalid() TEST(BloatConstraint, 1D) { - legate::Core::perform_registration(); + prepare(); test_bloat({{10}, {2}, {4}}); } TEST(BloatConstraint, 2D) { - legate::Core::perform_registration(); + prepare(); test_bloat({{9, 9}, {2, 3}, {3, 4}}); } TEST(BloatConstraint, 3D) { - legate::Core::perform_registration(); + prepare(); test_bloat({{10, 10, 10}, {2, 3, 4}, {4, 3, 2}}); } TEST(BloatConstraint, Invalid) { - legate::Core::perform_registration(); + prepare(); test_invalid(); } diff --git a/tests/cpp/integration/broadcast_constraints.cc b/tests/cpp/integration/broadcast_constraints.cc index b2f97987a4..40b365406c 100644 --- a/tests/cpp/integration/broadcast_constraints.cc +++ b/tests/cpp/integration/broadcast_constraints.cc @@ -122,19 +122,19 @@ void test_invalid_broadcast() TEST(Broadcast, Basic) { - legate::Core::perform_registration(); + prepare(); test_normal_store(); } TEST(Broadcast, WithPromotion) { - legate::Core::perform_registration(); + prepare(); test_promoted_store(); } TEST(Broadcast, Invalid) { - legate::Core::perform_registration(); + prepare(); test_invalid_broadcast(); } diff --git a/tests/cpp/integration/consensus_match.cc b/tests/cpp/integration/consensus_match.cc index bd39bf868a..87cb4d65b6 100644 --- a/tests/cpp/integration/consensus_match.cc +++ b/tests/cpp/integration/consensus_match.cc @@ -39,7 +39,7 @@ struct Thing { TEST(Integration, ConsensusMatch) { auto runtime = legate::Runtime::get_runtime(); - legate::Core::perform_registration(); + register_tasks(); auto context = runtime->find_library(library_name); static_cast(context); diff --git a/tests/cpp/integration/copy_gather.cc b/tests/cpp/integration/copy_gather.cc index 8ae40f5d22..6cee18d8c6 100644 --- a/tests/cpp/integration/copy_gather.cc +++ b/tests/cpp/integration/copy_gather.cc @@ -147,33 +147,33 @@ void test_gather(const GatherSpec& spec) TEST(Copy, Gather2Dto1D) { - legate::Core::perform_registration(); + register_tasks(); std::vector shape1d{5}; test_gather(GatherSpec{shape1d, {7, 11}, legate::Scalar(int64_t(123))}); } TEST(Copy, Gather3Dto2D) { - legate::Core::perform_registration(); + register_tasks(); test_gather(GatherSpec{{3, 7}, {3, 2, 5}, legate::Scalar(uint32_t(456))}); } TEST(Copy, Gather1Dto3D) { - legate::Core::perform_registration(); + register_tasks(); std::vector shape1d{5}; test_gather(GatherSpec{{2, 5, 4}, shape1d, legate::Scalar(789.0)}); } TEST(Copy, Gather2Dto2D) { - legate::Core::perform_registration(); + register_tasks(); test_gather(GatherSpec{{4, 5}, {10, 11}, legate::Scalar(int64_t(12))}); } TEST(Copy, Gather2Dto3D) { - legate::Core::perform_registration(); + register_tasks(); test_gather(GatherSpec{{100, 100, 100}, {10, 10}, legate::Scalar(7.0)}); } diff --git a/tests/cpp/integration/copy_gather_scatter.cc b/tests/cpp/integration/copy_gather_scatter.cc index 22ea0cbd4e..7814a0f1b5 100644 --- a/tests/cpp/integration/copy_gather_scatter.cc +++ b/tests/cpp/integration/copy_gather_scatter.cc @@ -177,7 +177,7 @@ void test_gather_scatter(const GatherScatterSpec& spec) TEST(Copy, GatherScatter1Dto3Dvia2D) { - legate::Core::perform_registration(); + register_tasks(); std::vector shape1d{5}; test_gather_scatter(GatherScatterSpec{ shape1d, {7, 11}, {10, 10, 10}, legate::Scalar(int64_t(123)), legate::Scalar(int64_t(42))}); @@ -185,7 +185,7 @@ TEST(Copy, GatherScatter1Dto3Dvia2D) TEST(Copy, GatherScatter2Dto1Dvia3D) { - legate::Core::perform_registration(); + register_tasks(); std::vector shape1d{1000}; test_gather_scatter(GatherScatterSpec{ {3, 7}, {3, 6, 5}, shape1d, legate::Scalar(uint32_t(456)), legate::Scalar(uint32_t(42))}); @@ -193,7 +193,7 @@ TEST(Copy, GatherScatter2Dto1Dvia3D) TEST(Copy, GatherScatter3Dto2Dvia1D) { - legate::Core::perform_registration(); + register_tasks(); std::vector shape1d{100}; test_gather_scatter(GatherScatterSpec{ {4, 5, 2}, shape1d, {50, 50}, legate::Scalar(int64_t(12)), legate::Scalar(int64_t(42))}); @@ -201,7 +201,7 @@ TEST(Copy, GatherScatter3Dto2Dvia1D) TEST(Copy, GatherScatter3Dto3Dvia3D) { - legate::Core::perform_registration(); + register_tasks(); test_gather_scatter(GatherScatterSpec{{10, 10, 10}, {5, 4, 2}, {10, 10, 10}, @@ -211,7 +211,7 @@ TEST(Copy, GatherScatter3Dto3Dvia3D) TEST(Copy, GatherScatter2Dto3Dvia2D) { - legate::Core::perform_registration(); + register_tasks(); test_gather_scatter(GatherScatterSpec{ {27, 33}, {11, 7}, {132, 121, 3}, legate::Scalar(int64_t(2)), legate::Scalar(int64_t(84))}); } diff --git a/tests/cpp/integration/copy_normal.cc b/tests/cpp/integration/copy_normal.cc index d52e44f0ba..89c8244730 100644 --- a/tests/cpp/integration/copy_normal.cc +++ b/tests/cpp/integration/copy_normal.cc @@ -208,7 +208,7 @@ void test_normal_copy_reduction(const NormalCopyReductionSpec& spec) TEST(Copy, Single) { - legate::Core::perform_registration(); + register_tasks(); test_normal_copy({{4, 7}, legate::int64(), legate::Scalar(int64_t(12))}); test_normal_copy({{1000, 100}, legate::uint32(), legate::Scalar(uint32_t(3))}); test_normal_copy({{1}, legate::int64(), legate::Scalar(int64_t(12))}); @@ -216,7 +216,7 @@ TEST(Copy, Single) TEST(Copy, SingleReduction) { - legate::Core::perform_registration(); + register_tasks(); test_normal_copy_reduction( {{4, 7}, legate::int64(), legate::Scalar(int64_t(12)), legate::ReductionOpKind::ADD}); test_normal_copy_reduction( diff --git a/tests/cpp/integration/copy_scatter.cc b/tests/cpp/integration/copy_scatter.cc index a0b3939720..d25a1400e2 100644 --- a/tests/cpp/integration/copy_scatter.cc +++ b/tests/cpp/integration/copy_scatter.cc @@ -171,7 +171,7 @@ void test_scatter(const ScatterSpec& spec) // duplicate updates on the same element, whose semantics is undefined. TEST(Copy, Scatter1Dto2D) { - legate::Core::perform_registration(); + register_tasks(); std::vector shape1d{5}; test_scatter( ScatterSpec{shape1d, {7, 11}, legate::Scalar(int64_t(123)), legate::Scalar(int64_t(42))}); @@ -179,21 +179,21 @@ TEST(Copy, Scatter1Dto2D) TEST(Copy, Scatter2Dto3D) { - legate::Core::perform_registration(); + register_tasks(); test_scatter( ScatterSpec{{3, 7}, {3, 6, 5}, legate::Scalar(uint32_t(456)), legate::Scalar(uint32_t(42))}); } TEST(Copy, Scatter2Dto2D) { - legate::Core::perform_registration(); + register_tasks(); test_scatter( ScatterSpec{{4, 5}, {10, 11}, legate::Scalar(int64_t(12)), legate::Scalar(int64_t(42))}); } TEST(Copy, Scatter3Dto2D) { - legate::Core::perform_registration(); + register_tasks(); test_scatter( ScatterSpec{{10, 10, 10}, {200, 200}, legate::Scalar(int64_t(1)), legate::Scalar(int64_t(42))}); } diff --git a/tests/cpp/integration/cpu_communicator.cc b/tests/cpp/integration/cpu_communicator.cc index bfe9d84ce2..c26dbfa193 100644 --- a/tests/cpp/integration/cpu_communicator.cc +++ b/tests/cpp/integration/cpu_communicator.cc @@ -87,7 +87,7 @@ void test_cpu_communicator_manual(int32_t ndim) // Test case with single unbound store TEST(Integration, CPUCommunicator) { - legate::Core::perform_registration(); + prepare(); for (int32_t ndim : {1, 3}) { test_cpu_communicator_auto(ndim); diff --git a/tests/cpp/integration/exception.cc b/tests/cpp/integration/exception.cc index b556177f4b..989fb91f4b 100644 --- a/tests/cpp/integration/exception.cc +++ b/tests/cpp/integration/exception.cc @@ -176,14 +176,14 @@ void test_pending() TEST(Exception, Single) { - legate::Core::perform_registration(); + prepare(); test_single(); } TEST(Exception, Multi) { - legate::Core::perform_registration(); + prepare(); test_multi(true /* use_auto_task */); test_multi(false /* use_auto_task */); @@ -191,7 +191,7 @@ TEST(Exception, Multi) TEST(Exception, Pending) { - legate::Core::perform_registration(); + prepare(); test_pending(); } diff --git a/tests/cpp/integration/field_reuse.cc b/tests/cpp/integration/field_reuse.cc index 2695806597..150c453289 100644 --- a/tests/cpp/integration/field_reuse.cc +++ b/tests/cpp/integration/field_reuse.cc @@ -44,7 +44,7 @@ TEST(Integration, FieldReuse) // TODO: Also test the reuse of a field originally returned by an unbounded-output task. auto runtime = legate::Runtime::get_runtime(); - legate::Core::perform_registration(); + register_tasks(); auto context = runtime->find_library(library_name); static_cast(context); diff --git a/tests/cpp/integration/fill.cc b/tests/cpp/integration/fill.cc index 317c6c5c97..a4d8c1b36e 100644 --- a/tests/cpp/integration/fill.cc +++ b/tests/cpp/integration/fill.cc @@ -179,7 +179,7 @@ void test_invalid() TEST(Fill, Index) { - legate::Core::perform_registration(); + register_tasks(); test_fill_index(1, SIZE); test_fill_index(2, SIZE); test_fill_index(3, SIZE); @@ -190,7 +190,7 @@ TEST(Fill, Index) TEST(Fill, Single) { - legate::Core::perform_registration(); + register_tasks(); test_fill_single(1, SIZE); test_fill_single(2, SIZE); test_fill_single(3, SIZE); @@ -201,7 +201,7 @@ TEST(Fill, Single) TEST(Fill, Slice) { - legate::Core::perform_registration(); + register_tasks(); test_fill_slice(1, SIZE); test_fill_slice(2, SIZE); test_fill_slice(3, SIZE); diff --git a/tests/cpp/integration/image_constraints.cc b/tests/cpp/integration/image_constraints.cc index 49f966eced..98359faa5d 100644 --- a/tests/cpp/integration/image_constraints.cc +++ b/tests/cpp/integration/image_constraints.cc @@ -276,43 +276,43 @@ void test_invalid() TEST(ImageConstraint, Point1D) { - legate::Core::perform_registration(); + prepare(); test_image({{9}, {100}, false}); } TEST(ImageConstraint, Point2D) { - legate::Core::perform_registration(); + prepare(); test_image({{4, 4}, {10, 10}, false}); } TEST(ImageConstraint, Point3D) { - legate::Core::perform_registration(); + prepare(); test_image({{2, 3, 4}, {5, 5, 5}, false}); } TEST(ImageConstraint, Rect1D) { - legate::Core::perform_registration(); + prepare(); test_image({{9}, {100}, true}); } TEST(ImageConstraint, Rect2D) { - legate::Core::perform_registration(); + prepare(); test_image({{4, 4}, {10, 10}, true}); } TEST(ImageConstraint, Rect3D) { - legate::Core::perform_registration(); + prepare(); test_image({{2, 3, 4}, {5, 5, 5}, true}); } TEST(ImageConstraint, Invalid) { - legate::Core::perform_registration(); + prepare(); test_invalid(); } diff --git a/tests/cpp/integration/inline_map.cc b/tests/cpp/integration/inline_map.cc index d6bac7555a..8a1f4c1ff0 100644 --- a/tests/cpp/integration/inline_map.cc +++ b/tests/cpp/integration/inline_map.cc @@ -111,7 +111,7 @@ TEST(InlineMap, RegionAndSlice) { test_inline_map_region_and_slice(); } TEST(InlineMap, WithTask) { - legate::Core::perform_registration(); + register_tasks(); test_inline_map_and_task(); } diff --git a/tests/cpp/integration/inout.cc b/tests/cpp/integration/inout.cc index 991f411171..e80627fc23 100644 --- a/tests/cpp/integration/inout.cc +++ b/tests/cpp/integration/inout.cc @@ -33,7 +33,7 @@ void test_inout() TEST(Integration, InOut) { - legate::Core::perform_registration(); + task::simple::register_tasks(); test_inout(); } diff --git a/tests/cpp/integration/machine_scope.cc b/tests/cpp/integration/machine_scope.cc index 3257cb1d36..07619c9ab5 100644 --- a/tests/cpp/integration/machine_scope.cc +++ b/tests/cpp/integration/machine_scope.cc @@ -133,7 +133,7 @@ void test_cpu_only(legate::Library library) TEST(Integration, MachineScope) { - legate::Core::perform_registration(); + register_tasks(); auto runtime = legate::Runtime::get_runtime(); auto library = runtime->find_library(library_name); diff --git a/tests/cpp/integration/manual_simple.cc b/tests/cpp/integration/manual_simple.cc index 3a630803f5..32b9f6419d 100644 --- a/tests/cpp/integration/manual_simple.cc +++ b/tests/cpp/integration/manual_simple.cc @@ -49,7 +49,7 @@ void print_store(legate::LogicalStore store) TEST(Integration, ManualSimple) { - legate::Core::perform_registration(); + task::simple::register_tasks(); auto runtime = legate::Runtime::get_runtime(); auto library = runtime->find_library(task::simple::library_name); diff --git a/tests/cpp/integration/multi_scalar_out.cc b/tests/cpp/integration/multi_scalar_out.cc index aa5d0d223b..3eb7bf0f1e 100644 --- a/tests/cpp/integration/multi_scalar_out.cc +++ b/tests/cpp/integration/multi_scalar_out.cc @@ -73,7 +73,7 @@ void validate_stores(legate::LogicalStore scalar1, TEST(Integration, MultiScalarOut) { - legate::Core::perform_registration(); + task::simple::register_tasks(); auto runtime = legate::Runtime::get_runtime(); auto library = runtime->find_library(task::simple::library_name); diff --git a/tests/cpp/integration/nccl.cu b/tests/cpp/integration/nccl.cu index 8a7dfd813f..9d93ef1093 100644 --- a/tests/cpp/integration/nccl.cu +++ b/tests/cpp/integration/nccl.cu @@ -101,7 +101,7 @@ void test_nccl_manual(int32_t ndim) // Test case with single unbound store TEST(Integration, NCCL) { - legate::Core::perform_registration(); + prepare(); auto runtime = legate::Runtime::get_runtime(); auto machine = runtime->get_machine(); diff --git a/tests/cpp/integration/provenance.cc b/tests/cpp/integration/provenance.cc index 4fcd2ab84c..09addbccec 100644 --- a/tests/cpp/integration/provenance.cc +++ b/tests/cpp/integration/provenance.cc @@ -134,7 +134,7 @@ void test_manual_tracker(legate::Library library) TEST(Integration, Provenance) { - legate::Core::perform_registration(); + register_tasks(); auto runtime = legate::Runtime::get_runtime(); auto library = runtime->find_library(library_name); diff --git a/tests/cpp/integration/region_manager.cc b/tests/cpp/integration/region_manager.cc index 3f7f75076f..356632aa80 100644 --- a/tests/cpp/integration/region_manager.cc +++ b/tests/cpp/integration/region_manager.cc @@ -19,7 +19,7 @@ namespace region_manager { TEST(Integration, RegionManager) { - legate::Core::perform_registration(); + task::region_manager::register_tasks(); auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(task::region_manager::library_name); diff --git a/tests/cpp/integration/req_analyzer.cc b/tests/cpp/integration/req_analyzer.cc index 141ce806eb..96148a9ae9 100644 --- a/tests/cpp/integration/req_analyzer.cc +++ b/tests/cpp/integration/req_analyzer.cc @@ -80,13 +80,13 @@ void test_isomorphic_transformed_stores() TEST(ReqAnalyzer, InoutStore) { - legate::Core::perform_registration(); + prepare(); test_inout_store(); } TEST(ReqAnalyzer, IsomorphicTransformedStores) { - legate::Core::perform_registration(); + prepare(); test_isomorphic_transformed_stores(); } diff --git a/tests/cpp/integration/scale_constraints.cc b/tests/cpp/integration/scale_constraints.cc index e383e346ec..7231cd4404 100644 --- a/tests/cpp/integration/scale_constraints.cc +++ b/tests/cpp/integration/scale_constraints.cc @@ -116,25 +116,25 @@ void test_invalid() TEST(ScaleConstraint, 1D) { - legate::Core::perform_registration(); + prepare(); test_scale({{3}, {10}, {29}}); } TEST(ScaleConstraint, 2D) { - legate::Core::perform_registration(); + prepare(); test_scale({{4, 5}, {2, 7}, {10, 30}}); } TEST(ScaleConstraint, 3D) { - legate::Core::perform_registration(); + prepare(); test_scale({{2, 3, 4}, {5, 5, 5}, {10, 15, 20}}); } TEST(ScaleConstraint, Invalid) { - legate::Core::perform_registration(); + prepare(); test_invalid(); } diff --git a/tests/cpp/integration/tree_reduce.cc b/tests/cpp/integration/tree_reduce.cc index c10450a0ad..fb361c3808 100644 --- a/tests/cpp/integration/tree_reduce.cc +++ b/tests/cpp/integration/tree_reduce.cc @@ -89,7 +89,7 @@ void register_tasks() TEST(TreeReduce, AutoProducer) { - legate::Core::perform_registration(); + register_tasks(); auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(library_name); @@ -111,7 +111,7 @@ TEST(TreeReduce, AutoProducer) TEST(TreeReduce, ManualProducer) { - legate::Core::perform_registration(); + register_tasks(); auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(library_name); @@ -133,7 +133,7 @@ TEST(TreeReduce, ManualProducer) TEST(TreeReduce, ManualProducerMultiLevel) { - legate::Core::perform_registration(); + register_tasks(); auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(library_name); @@ -155,7 +155,7 @@ TEST(TreeReduce, ManualProducerMultiLevel) TEST(TreeReduce, ManualProducerUnbound) { - legate::Core::perform_registration(); + register_tasks(); auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(library_name); @@ -174,7 +174,7 @@ TEST(TreeReduce, ManualProducerUnbound) TEST(TreeReduce, ManualProducerSingle) { - legate::Core::perform_registration(); + register_tasks(); auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(library_name); @@ -192,7 +192,7 @@ TEST(TreeReduce, ManualProducerSingle) TEST(TreeReduce, AutoProducerSingle) { - legate::Core::perform_registration(); + register_tasks(); auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(library_name); diff --git a/tests/cpp/integration/tree_reduce_unique.cc b/tests/cpp/integration/tree_reduce_unique.cc index c2f01dfe94..ed7400ee19 100644 --- a/tests/cpp/integration/tree_reduce_unique.cc +++ b/tests/cpp/integration/tree_reduce_unique.cc @@ -106,7 +106,7 @@ void register_tasks() TEST(TreeReduce, Unique) { - legate::Core::perform_registration(); + register_tasks(); auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(library_name); diff --git a/tests/cpp/integration/weighted.cc b/tests/cpp/integration/weighted.cc index 5c6f5b1d30..9aa8f7d85b 100644 --- a/tests/cpp/integration/weighted.cc +++ b/tests/cpp/integration/weighted.cc @@ -102,7 +102,7 @@ void test_weighted(uint32_t num_stores) // Test case with single unbound store TEST(Integration, WeightedSingle) { - legate::Core::perform_registration(); + prepare(); test_weighted(1); } @@ -110,7 +110,7 @@ TEST(Integration, WeightedSingle) // Test case with multiple unbound stores TEST(Integration, WeightedMultiple) { - legate::Core::perform_registration(); + prepare(); test_weighted(3); } diff --git a/tests/cpp/unit/buffer.cc b/tests/cpp/unit/buffer.cc index ad4736669f..9cafbe1ae8 100644 --- a/tests/cpp/unit/buffer.cc +++ b/tests/cpp/unit/buffer.cc @@ -92,7 +92,7 @@ void register_tasks() TEST(BufferUnit, CreateBuffer) { - legate::Core::perform_registration(); + register_tasks(); // Todo: need to add tests for REGDMA_MEM test_buffer(1, 10, legate::Memory::SYSTEM_MEM); @@ -103,7 +103,7 @@ TEST(BufferUnit, CreateBuffer) TEST(BufferUnit, NegativeTest) { - legate::Core::perform_registration(); + register_tasks(); test_buffer(1, 0, legate::Memory::SYSTEM_MEM); test_buffer(2, 10, legate::Memory::SYSTEM_MEM, 0); diff --git a/tests/cpp/unit/constraint.cc b/tests/cpp/unit/constraint.cc index 094b769f1d..646b232b1b 100644 --- a/tests/cpp/unit/constraint.cc +++ b/tests/cpp/unit/constraint.cc @@ -37,7 +37,7 @@ void register_tasks() TEST(Variable, BasicMethods) { - legate::Core::perform_registration(); + register_tasks(); auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(library_name); @@ -73,7 +73,7 @@ TEST(Variable, BasicMethods) TEST(Alignment, BasicMethods) { - legate::Core::perform_registration(); + register_tasks(); auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(library_name); @@ -101,7 +101,7 @@ TEST(Alignment, BasicMethods) TEST(Broadcast, BasicMethods) { - legate::Core::perform_registration(); + register_tasks(); auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(library_name); @@ -126,7 +126,7 @@ TEST(Broadcast, BasicMethods) TEST(ImageConstraint, BasicMethods) { - legate::Core::perform_registration(); + register_tasks(); auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(library_name); @@ -149,4 +149,4 @@ TEST(ImageConstraint, BasicMethods) EXPECT_TRUE(std::find(symbols.begin(), symbols.end(), part_func.impl()) != symbols.end()); EXPECT_TRUE(std::find(symbols.begin(), symbols.end(), part_range.impl()) != symbols.end()); } -} // namespace unit \ No newline at end of file +} // namespace unit diff --git a/tests/cpp/unit/registration.cc b/tests/cpp/unit/registration.cc index e22150c781..5a1496a0f7 100644 --- a/tests/cpp/unit/registration.cc +++ b/tests/cpp/unit/registration.cc @@ -49,9 +49,6 @@ void test_out_of_bounds_task_id() EXPECT_THROW(test_registration::CPUVariantTask<1>::register_variants(library), std::out_of_range); } -TEST(Registration, Duplicate) { legate::Core::perform_registration(); } +TEST(Registration, Duplicate) { test_duplicates(); } -TEST(Registration, TaskIDOutOfBounds) -{ - legate::Core::perform_registration(); -} +TEST(Registration, TaskIDOutOfBounds) { test_out_of_bounds_task_id(); } diff --git a/tests/cpp/unit/scoped_allocator.cc b/tests/cpp/unit/scoped_allocator.cc index 0082120691..6a54ba225e 100644 --- a/tests/cpp/unit/scoped_allocator.cc +++ b/tests/cpp/unit/scoped_allocator.cc @@ -109,7 +109,7 @@ TEST(ScopedAllocatorUnit, EmptyAllocate) TEST(ScopedAllocatorUnit, Allocate) { - legate::Core::perform_registration(); + register_tasks(); test_allocator(BufferOpCode::DEALLOCATE, legate::Memory::SYSTEM_MEM, true, 1000, 16); test_allocator(BufferOpCode::DEALLOCATE, legate::Memory::NO_MEMKIND, true, 1000, 16); test_allocator(BufferOpCode::DEALLOCATE, legate::Memory::SYSTEM_MEM, false, 0, 16); @@ -123,7 +123,7 @@ TEST(ScopedAllocatorUnit, Allocate) TEST(ScopedAllocatorUnit, DoubleDeallocate) { - legate::Core::perform_registration(); + register_tasks(); test_allocator(BufferOpCode::DOUBLE_DEALLOCATE, legate::Memory::SYSTEM_MEM, true, 1000); } @@ -134,7 +134,7 @@ TEST(ScopedAllocatorUnit, InvalidDeallocate) EXPECT_THROW(allocator.deallocate(data.data()), std::runtime_error); // invalid deallocate in task launch - legate::Core::perform_registration(); + register_tasks(); test_allocator(BufferOpCode::INVALID_DEALLOCATE, legate::Memory::SYSTEM_MEM, true, 1000); } } // namespace scoped_allocator_test diff --git a/tests/integration/collective/src/library.cc b/tests/integration/collective/src/library.cc index 629118109b..45071a297b 100644 --- a/tests/integration/collective/src/library.cc +++ b/tests/integration/collective/src/library.cc @@ -30,11 +30,5 @@ void registration_callback() extern "C" { -void collective_perform_registration(void) -{ - // Tell the runtime about our registration callback so we hook it - // in before the runtime starts and make it global so that we know - // that this call back is invoked everywhere across all nodes - legate::Core::perform_registration(); -} +void collective_perform_registration(void) { collective::registration_callback(); } } diff --git a/tests/integration/region_manager/src/library.cc b/tests/integration/region_manager/src/library.cc index f32e775ad1..bdcaef30da 100644 --- a/tests/integration/region_manager/src/library.cc +++ b/tests/integration/region_manager/src/library.cc @@ -27,8 +27,5 @@ void registration_callback() extern "C" { -void perform_registration(void) -{ - legate::Core::perform_registration(); -} +void perform_registration(void) { region_manager::registration_callback(); } } diff --git a/tests/integration/registry/src/library.cc b/tests/integration/registry/src/library.cc index 43a55fab43..503fa9baff 100644 --- a/tests/integration/registry/src/library.cc +++ b/tests/integration/registry/src/library.cc @@ -40,5 +40,5 @@ void registration_callback() extern "C" { -void perform_registration(void) { legate::Core::perform_registration(); } +void perform_registration(void) { rg::registration_callback(); } } diff --git a/tests/integration/scoping/examples/test_scoping.py b/tests/integration/scoping/examples/test_scoping.py index 82e4f20dee..b301d7c885 100644 --- a/tests/integration/scoping/examples/test_scoping.py +++ b/tests/integration/scoping/examples/test_scoping.py @@ -92,7 +92,8 @@ def test_cpu_only(): def test_shifted_slices(): m = get_machine() - num_nodes = len(m.get_node_range()) + (node_lo, node_hi) = m.get_node_range() + num_nodes = node_hi - node_lo per_node_count = int((len(m) + num_nodes - 1) / num_nodes) # this will test processor slicing of the machine for i in range(len(m)): diff --git a/tests/integration/scoping/src/library.cc b/tests/integration/scoping/src/library.cc index 53f26a5c63..a88717b434 100644 --- a/tests/integration/scoping/src/library.cc +++ b/tests/integration/scoping/src/library.cc @@ -44,9 +44,9 @@ void map_check(legate::TaskContext& context) int32_t task_count = context.get_launch_domain().get_volume(); int32_t shard_id = legate::Processor::get_executing_processor().address_space(); int32_t task_id = context.get_task_index()[0]; - int32_t per_node_count = context.scalars().at(0).value(); - int32_t proc_count = context.scalars().at(1).value(); - int32_t start_proc_id = context.scalars().at(2).value(); + int32_t per_node_count = context.scalar(0).value(); + int32_t proc_count = context.scalar(1).value(); + int32_t start_proc_id = context.scalar(2).value(); int32_t global_proc_id = task_id * proc_count / task_count + start_proc_id; int32_t calculated_shard_id = global_proc_id / per_node_count; if (shard_id != calculated_shard_id) { @@ -76,12 +76,12 @@ class CpuVariantOnlyTask : public Task { class MapCheckTask : public Task { public: - static void cpu_variant(legate::TaskContext& context) { map_check(context); } -#ifdef LEGATE_USE_OPENMP - static void omp_variant(legate::TaskContext& context) { map_check(context); } + static void cpu_variant(legate::TaskContext context) { map_check(context); } +#if LegateDefined(USE_OPENMP) + static void omp_variant(legate::TaskContext context) { map_check(context); } #endif -#ifdef LEGATE_USE_CUDA - static void gpu_variant(legate::TaskContext& context) { map_check(context); } +#if LegateDefined(USE_CUDA) + static void gpu_variant(legate::TaskContext context) { map_check(context); } #endif }; @@ -98,8 +98,5 @@ void registration_callback() extern "C" { -void perform_registration(void) -{ - legate::Core::perform_registration(); -} +void perform_registration(void) { scoping::registration_callback(); } } diff --git a/tests/integration/tree_reduce/src/library.cc b/tests/integration/tree_reduce/src/library.cc index f285ea9c8a..21e03548a8 100644 --- a/tests/integration/tree_reduce/src/library.cc +++ b/tests/integration/tree_reduce/src/library.cc @@ -36,8 +36,5 @@ void registration_callback() extern "C" { -void perform_registration(void) -{ - legate::Core::perform_registration(); -} +void perform_registration(void) { tree_reduce::registration_callback(); } } From 3cc85d3f9e4e0e21682b8ba48891157b65fd6a39 Mon Sep 17 00:00:00 2001 From: Vladislav Zhurba <53052066+vzhurba01@users.noreply.github.com> Date: Tue, 3 Oct 2023 16:37:36 -0700 Subject: [PATCH 0238/1425] Integrate cpp tests to legate tester (#90) * Integrate cpp tests to legate tester Adds binary and MPI compatibility to tester * Review touch-ups --- legate/tester/args.py | 34 +++++ legate/tester/config.py | 10 +- legate/tester/stages/test_stage.py | 165 +++++++++++++++++++++--- legate/tester/test_plan.py | 13 +- tests/cpp/run.py | 133 ++++--------------- tests/unit/legate/tester/test_config.py | 5 + 6 files changed, 236 insertions(+), 124 deletions(-) diff --git a/legate/tester/args.py b/legate/tester/args.py index 97d02e8350..27385d15b4 100644 --- a/legate/tester/args.py +++ b/legate/tester/args.py @@ -294,6 +294,7 @@ help="Print out the commands that are to be executed", ) + parser.add_argument( "--color", dest="color", @@ -301,3 +302,36 @@ required=False, help="Whether to use color terminal output (if colorama is installed)", ) + + +parser.add_argument( + "--gtest-file", + dest="gtest_file", + default=None, + help="Path to gtest binary", +) + + +parser.add_argument( + "--gtest-tests", + dest="gtest_tests", + nargs="+", + default=None, + help="Explicit list of test names to run against", +) + + +parser.add_argument( + "--mpi-rank", + dest="mpi_rank", + default=None, + help="Runs mpirun with rank if non-zero", +) + + +parser.add_argument( + "--mpi-output-filename", + dest="mpi_output_filename", + default=None, + help="Path to mpirun results", +) diff --git a/legate/tester/config.py b/legate/tester/config.py index 30eba8759c..d31c61c0fa 100644 --- a/legate/tester/config.py +++ b/legate/tester/config.py @@ -48,7 +48,15 @@ def __init__(self, argv: ArgList) -> None: colors.ENABLED = args.color - # which tests to run + # mpi configuration + self.mpi_rank = args.mpi_rank + self.mpi_output_filename = args.mpi_output_filename + + # gtest configuration + self.gtest_file = args.gtest_file + self.gtest_tests = args.gtest_tests + + # python configuration for which tests to run self.examples = False if args.cov_bin else True self.integration = True self.unit = args.unit diff --git a/legate/tester/stages/test_stage.py b/legate/tester/stages/test_stage.py index 732ac466f1..6311eb9942 100644 --- a/legate/tester/stages/test_stage.py +++ b/legate/tester/stages/test_stage.py @@ -12,6 +12,7 @@ from __future__ import annotations import multiprocessing +import os import queue from datetime import datetime from pathlib import Path @@ -19,6 +20,8 @@ from typing_extensions import Protocol +from legate.driver.launcher import LAUNCHER_VAR_PREFIXES + from ...util.colors import yellow from ...util.types import ArgList, EnvDict from ...util.ui import banner, summary @@ -222,7 +225,29 @@ def cov_args(self, config: Config) -> ArgList: return args - def run( + def _run_common( + self, + cmd: ArgList, + test_description: Path, + config: Config, + system: TestSystem, + shard: Shard, + ) -> ProcessResult: + self.delay(shard, config, system) + + result = system.run( + cmd, + test_description, + env=self._env(config, system), + timeout=config.timeout, + ) + log_proc(self.name, result, config, verbose=config.verbose) + + self.shards.put(shard) + + return result + + def run_python( self, test_file: Path, config: Config, @@ -271,19 +296,113 @@ def run( if custom_args: cmd += custom_args - self.delay(shard, config, system) + return self._run_common(cmd, test_file, config, system, shard) - result = system.run( - cmd, - test_file, - env=self._env(config, system), - timeout=config.timeout, + def run_gtest( + self, + test_file: str, + arg_test: str, + config: Config, + system: TestSystem, + *, + custom_args: ArgList | None = None, + ) -> ProcessResult: + """Execute a single test within gtest with appropriate environment and + command-line options for a feature test stage. + + Parameters + ---------- + test_file : str + Test file to execute + + arg_test : str + Test name to be executed + + config: Config + Test runner configuration + + system: TestSystem + Process execution wrapper + + """ + + shard = self.shards.get() + + cov_args = self.cov_args(config) + + stage_args = self.args + self.shard_args(shard, config) + + cmd = ( + [test_file] + + [f"--gtest_filter={arg_test}"] + + stage_args + + cov_args + + config.extra_args ) - log_proc(self.name, result, config, verbose=config.verbose) - self.shards.put(shard) + if custom_args: + cmd += custom_args - return result + return self._run_common(cmd, Path(arg_test), config, system, shard) + + def run_mpi( + self, + test_file: str, + arg_test: str, + config: Config, + system: TestSystem, + *, + custom_args: ArgList | None = None, + ) -> ProcessResult: + """Execute a single test within gtest with appropriate environment and + command-line options for a feature test stage. + + Parameters + ---------- + test_file : str + Test file to execute + + arg_test : str + Test name to be executed + + config: Config + Test runner configuration + + system: TestSystem + Process execution wrapper + + """ + + shard = self.shards.get() + + cov_args = self.cov_args(config) + + stage_args = self.args + self.shard_args(shard, config) + + mpi_args = [] + mpi_args += ["mpirun", "-n", str(config.ranks)] + mpi_args += ["--output-filename", config.mpi_output_filename] + mpi_args += ["--merge-stderr-to-stdout"] + + for var in dict(os.environ): + if var.endswith("PATH") or any( + var.startswith(prefix) for prefix in LAUNCHER_VAR_PREFIXES + ): + mpi_args += ["-x", var] + + cmd = ( + mpi_args + + [test_file] + + [f"--gtest_filter={arg_test}"] + + stage_args + + cov_args + + config.extra_args + ) + + if custom_args: + cmd += custom_args + + return self._run_common(cmd, Path(arg_test), config, system, shard) def _env(self, config: Config, system: TestSystem) -> EnvDict: env = dict(config.env) @@ -308,10 +427,26 @@ def _launch( ) -> list[ProcessResult]: pool = multiprocessing.pool.ThreadPool(self.spec.workers) - jobs = [ - pool.apply_async(self.run, (path, config, system)) - for path in config.test_files - ] + if config.mpi_rank: + jobs = [ + pool.apply_async( + self.run_mpi, (config.gtest_file, arg, config, system) + ) + for arg in config.gtest_tests + ] + elif config.gtest_file: + jobs = [ + pool.apply_async( + self.run_gtest, + (config.gtest_file, arg, config, system), + ) + for arg in config.gtest_tests + ] + else: + jobs = [ + pool.apply_async(self.run_python, (path, config, system)) + for path in config.test_files + ] pool.close() sharded_results = [job.get() for job in jobs] @@ -319,7 +454,7 @@ def _launch( custom = (x for x in CUSTOM_FILES if x.kind == self.kind) custom_results = [ - self.run(Path(x.file), config, system, custom_args=x.args) + self.run_python(Path(x.file), config, system, custom_args=x.args) for x in custom ] diff --git a/legate/tester/test_plan.py b/legate/tester/test_plan.py index 034be761b6..8e69583ee4 100644 --- a/legate/tester/test_plan.py +++ b/legate/tester/test_plan.py @@ -85,12 +85,23 @@ def intro(self) -> str: except RuntimeError: gpus = "N/A" + if self._config.mpi_rank or self._config.gtest_file: + details = ( + f"* Feature stages : {', '.join(yellow(x) for x in self._config.features)}", # noqa E501 + f"* Test files per stage : {yellow(str(len(self._config.gtest_tests)))}", # noqa E501 + f"* TestSystem description : {yellow(str(cpus) + ' cpus')} / {yellow(str(gpus) + ' gpus')}", # noqa E501 + ) + return banner( + f"Test Suite Configuration ({'OpenMPI' if self._config.mpi_rank else 'GTest'})", # noqa E501 + details=details, + ) + details = ( f"* Feature stages : {', '.join(yellow(x) for x in self._config.features)}", # noqa E501 f"* Test files per stage : {yellow(str(len(self._config.test_files)))}", # noqa E501 f"* TestSystem description : {yellow(str(cpus) + ' cpus')} / {yellow(str(gpus) + ' gpus')}", # noqa E501 ) - return banner("Test Suite Configuration", details=details) + return banner("Test Suite Configuration (Python)", details=details) def outro(self, total: int, passed: int) -> str: """An informative banner to display at test run end. diff --git a/tests/cpp/run.py b/tests/cpp/run.py index a927035c6b..d97ea8d316 100755 --- a/tests/cpp/run.py +++ b/tests/cpp/run.py @@ -13,32 +13,16 @@ import argparse -import os import subprocess import sys -LAUNCHER_VAR_PREFIXES = ( - "CONDA_", - "LEGATE_", - "LEGION_", - "LG_", - "REALM_", - "GASNET_", - "PYTHON", - "UCX_", - "NCCL_", - "CUNUMERIC_", - "NVIDIA_", -) - -test_args_dict = { - # Example of usage - # "Alignment.Basic" : ["-logfile", "build/example_file.log"] -} - - -def fetch_test_names(binary_path): - list_command = [binary_path] + ["--gtest_list_tests"] +from legate.tester.config import Config +from legate.tester.test_plan import TestPlan +from legate.tester.test_system import TestSystem + + +def fetch_test_names(gtest_file): + list_command = [gtest_file] + ["--gtest_list_tests"] result = subprocess.check_output(list_command, stderr=subprocess.STDOUT) result = result.decode(sys.stdout.encoding).split("\n") @@ -61,103 +45,38 @@ def fetch_test_names(binary_path): return test_names -def run_test(config, test_name, log, extra_args): - test_command = [] - if config.ranks != 0: - test_command += ["mpirun", "-n", str(config.ranks)] - test_command += ["--output-filename", "build/mpi_result"] - test_command += ["--merge-stderr-to-stdout"] - - def is_launcher_var(name: str) -> bool: - # Whether an environment variable name is relevant for the laucher - return name.endswith("PATH") or any( - name.startswith(prefix) for prefix in LAUNCHER_VAR_PREFIXES - ) - - for var in dict(os.environ): - if is_launcher_var(var): - test_command += ["-x", var] - - test_command += [config.binary_path] - test_command += [f"--gtest_filter={test_name}"] - test_command += ["--cpus", str(config.cpus)] - test_command += extra_args - - if test_name in test_args_dict: - test_command += test_args_dict[test_name] - - task = subprocess.Popen(test_command, stdout=log, stderr=subprocess.STDOUT) - task.communicate() - - return task.returncode - - def main(): parser = argparse.ArgumentParser(description="Run Legate cpp tests.") parser.add_argument( - "--binary-path", - dest="binary_path", + "--gtest-file", + dest="gtest_file", required=False, default="build/cpp_tests", - help="Path to binary under test.", - ) - parser.add_argument( - "--log-path", - dest="log_path", - required=False, - default="build/results.log", - help="Path to output log file.", + help="GTest file under test", ) parser.add_argument( - "--ranks", - dest="ranks", + "--mpi-rank", + dest="mpi_rank", required=False, type=int, default=0, - help="Runs mpirun with rank if non-zero.", - ) - parser.add_argument( - "--cpus", - dest="cpus", - required=False, - type=int, - default=4, - help="Legion cmd argument for CPU processors to create per process.", + help="Runs mpirun with rank if non-zero", ) config, extra_args = parser.parse_known_args() - # Get names - test_names = fetch_test_names(config.binary_path) - - # Run each test with popen - total_count = len(test_names) - failed_count = 0 - failed_tests = [] - with open(config.log_path, "w") as log: - for count, test_name in enumerate(test_names): - return_code = run_test(config, test_name, log, extra_args) - - # Record test result - if return_code: - failed_tests += [test_name] - failed_count += 1 - print( - f"{count+1:3d}/{total_count}: {test_name} ".ljust(50, "."), - "Failed" if return_code else "Passed", - ) - - # Summarize results - print( - f"\n{int((total_count - failed_count) / total_count * 100)}% " - f"tests passed, {failed_count} tests failed out of {total_count}" - ) - if failed_tests: - print("\nThe following tests FAILED:") - for test in failed_tests: - print(f" - {test} (Failed)") - print(f"\nLog file generated: {config.log_path}") - return 1 - return 0 + if config.mpi_rank != 0: + extra_args += ["--mpi-rank", str(config.mpi_rank)] + extra_args += ["--mpi-output-filename", "build/mpi_result"] + extra_args += ["--gtest-file", config.gtest_file] + extra_args += ["--gtest-tests"] + fetch_test_names(config.gtest_file) + + config = Config(extra_args) + + system = TestSystem(dry_run=config.dry_run) + + plan = TestPlan(config, system) + + return plan.execute() if __name__ == "__main__": diff --git a/tests/unit/legate/tester/test_config.py b/tests/unit/legate/tester/test_config.py index c3839f642b..1820df2a24 100644 --- a/tests/unit/legate/tester/test_config.py +++ b/tests/unit/legate/tester/test_config.py @@ -83,6 +83,11 @@ def test_default_init(self) -> None: assert c.cov_args == "run -a --branch" assert c.cov_src_path is None + assert c.gtest_file is None + assert c.gtest_tests is None + assert c.mpi_rank is None + assert c.mpi_output_filename is None + def test_color_arg(self) -> None: m.Config(["test.py", "--color"]) From 7c5ab4bfe0f221c37e2698f33cc8c559cb59ddd3 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Tue, 3 Oct 2023 17:38:28 -0700 Subject: [PATCH 0239/1425] Deduplicate registration calls in tests (#104) --- .../cpp/integration/alignment_constraints.cc | 3 +++ tests/cpp/integration/bloat_constraints.cc | 3 +++ .../cpp/integration/broadcast_constraints.cc | 3 +++ tests/cpp/integration/copy_gather.cc | 3 +++ tests/cpp/integration/copy_gather_scatter.cc | 3 +++ tests/cpp/integration/copy_normal.cc | 3 +++ tests/cpp/integration/copy_scatter.cc | 3 +++ tests/cpp/integration/cpu_communicator.cc | 3 +++ tests/cpp/integration/exception.cc | 3 +++ tests/cpp/integration/fill.cc | 3 +++ tests/cpp/integration/image_constraints.cc | 3 +++ tests/cpp/integration/req_analyzer.cc | 3 +++ tests/cpp/integration/scale_constraints.cc | 3 +++ tests/cpp/integration/tasks/task_simple.cc | 3 +++ tests/cpp/integration/tree_reduce.cc | 3 +++ tests/cpp/integration/weighted.cc | 3 +++ tests/cpp/unit/buffer.cc | 3 +++ tests/cpp/unit/constraint.cc | 3 +++ tests/cpp/unit/library.cc | 25 +++++++++++++------ tests/cpp/unit/registration.cc | 4 +-- tests/cpp/unit/scoped_allocator.cc | 3 +++ 21 files changed, 76 insertions(+), 10 deletions(-) diff --git a/tests/cpp/integration/alignment_constraints.cc b/tests/cpp/integration/alignment_constraints.cc index 6b51ef90cd..a1e003639e 100644 --- a/tests/cpp/integration/alignment_constraints.cc +++ b/tests/cpp/integration/alignment_constraints.cc @@ -70,6 +70,9 @@ struct TransformedTester : public legate::LegateTask> { void prepare() { + static bool prepared = false; + if (prepared) { return; } + prepared = true; auto runtime = legate::Runtime::get_runtime(); auto context = runtime->create_library(library_name); Initializer::register_variants(context); diff --git a/tests/cpp/integration/bloat_constraints.cc b/tests/cpp/integration/bloat_constraints.cc index 4b16351ea8..8b1bb1e79a 100644 --- a/tests/cpp/integration/bloat_constraints.cc +++ b/tests/cpp/integration/bloat_constraints.cc @@ -54,6 +54,9 @@ struct BloatTester : public legate::LegateTask> { void prepare() { + static bool prepared = false; + if (prepared) { return; } + prepared = true; auto runtime = legate::Runtime::get_runtime(); auto context = runtime->create_library(library_name); BloatTester<1>::register_variants(context); diff --git a/tests/cpp/integration/broadcast_constraints.cc b/tests/cpp/integration/broadcast_constraints.cc index 40b365406c..cb15b65ae7 100644 --- a/tests/cpp/integration/broadcast_constraints.cc +++ b/tests/cpp/integration/broadcast_constraints.cc @@ -45,6 +45,9 @@ struct Initializer : public legate::LegateTask { void prepare() { + static bool prepared = false; + if (prepared) { return; } + prepared = true; auto runtime = legate::Runtime::get_runtime(); auto context = runtime->create_library(library_name); TesterTask::register_variants(context, TESTER); diff --git a/tests/cpp/integration/copy_gather.cc b/tests/cpp/integration/copy_gather.cc index 6cee18d8c6..1e38a5573b 100644 --- a/tests/cpp/integration/copy_gather.cc +++ b/tests/cpp/integration/copy_gather.cc @@ -59,6 +59,9 @@ struct CheckGatherTask : public legate::LegateTaskcreate_library(library_name); FillTask<1>::register_variants(context); diff --git a/tests/cpp/integration/copy_gather_scatter.cc b/tests/cpp/integration/copy_gather_scatter.cc index 7814a0f1b5..f79b9672a6 100644 --- a/tests/cpp/integration/copy_gather_scatter.cc +++ b/tests/cpp/integration/copy_gather_scatter.cc @@ -96,6 +96,9 @@ struct GatherScatterSpec { void register_tasks() { + static bool prepared = false; + if (prepared) { return; } + prepared = true; auto runtime = legate::Runtime::get_runtime(); auto library = runtime->create_library(library_name); FillTask<1>::register_variants(library); diff --git a/tests/cpp/integration/copy_normal.cc b/tests/cpp/integration/copy_normal.cc index 89c8244730..fcf2a8e9d7 100644 --- a/tests/cpp/integration/copy_normal.cc +++ b/tests/cpp/integration/copy_normal.cc @@ -107,6 +107,9 @@ struct CheckCopyReductionTask : public legate::LegateTaskcreate_library(library_name); FillTask<1>::register_variants(library); diff --git a/tests/cpp/integration/copy_scatter.cc b/tests/cpp/integration/copy_scatter.cc index d25a1400e2..83fc9ea61e 100644 --- a/tests/cpp/integration/copy_scatter.cc +++ b/tests/cpp/integration/copy_scatter.cc @@ -73,6 +73,9 @@ struct CheckScatterTask : public legate::LegateTaskcreate_library(library_name); FillTask<1>::register_variants(library); diff --git a/tests/cpp/integration/cpu_communicator.cc b/tests/cpp/integration/cpu_communicator.cc index c26dbfa193..bdf3e9d976 100644 --- a/tests/cpp/integration/cpu_communicator.cc +++ b/tests/cpp/integration/cpu_communicator.cc @@ -45,6 +45,9 @@ struct CPUCommunicatorTester : public legate::LegateTask void prepare() { + static bool prepared = false; + if (prepared) { return; } + prepared = true; auto runtime = legate::Runtime::get_runtime(); auto context = runtime->create_library(library_name); CPUCommunicatorTester::register_variants(context, CPU_COMM_TESTER); diff --git a/tests/cpp/integration/exception.cc b/tests/cpp/integration/exception.cc index 989fb91f4b..68dc2c405d 100644 --- a/tests/cpp/integration/exception.cc +++ b/tests/cpp/integration/exception.cc @@ -54,6 +54,9 @@ struct ExceptionTask : public legate::LegateTask { void prepare() { + static bool prepared = false; + if (prepared) { return; } + prepared = true; auto runtime = legate::Runtime::get_runtime(); auto context = runtime->create_library(library_name); ExceptionTask::register_variants(context, EXCEPTION_TASK); diff --git a/tests/cpp/integration/fill.cc b/tests/cpp/integration/fill.cc index a4d8c1b36e..0fc3f8c972 100644 --- a/tests/cpp/integration/fill.cc +++ b/tests/cpp/integration/fill.cc @@ -39,6 +39,9 @@ struct CheckSliceTask : public legate::LegateTask> { void register_tasks() { + static bool prepared = false; + if (prepared) { return; } + prepared = true; auto runtime = legate::Runtime::get_runtime(); auto context = runtime->create_library(library_name); CheckTask<1>::register_variants(context); diff --git a/tests/cpp/integration/image_constraints.cc b/tests/cpp/integration/image_constraints.cc index 98359faa5d..15b0631127 100644 --- a/tests/cpp/integration/image_constraints.cc +++ b/tests/cpp/integration/image_constraints.cc @@ -173,6 +173,9 @@ struct ImageTester : public legate::LegateTask> { void prepare() { + static bool prepared = false; + if (prepared) { return; } + prepared = true; auto runtime = legate::Runtime::get_runtime(); auto context = runtime->create_library(library_name); InitializeFunction<1, true>::register_variants(context); diff --git a/tests/cpp/integration/req_analyzer.cc b/tests/cpp/integration/req_analyzer.cc index 96148a9ae9..e185d6ca90 100644 --- a/tests/cpp/integration/req_analyzer.cc +++ b/tests/cpp/integration/req_analyzer.cc @@ -38,6 +38,9 @@ struct Tester : public legate::LegateTask { void prepare() { + static bool prepared = false; + if (prepared) { return; } + prepared = true; auto runtime = legate::Runtime::get_runtime(); auto context = runtime->create_library(library_name); Tester::register_variants(context); diff --git a/tests/cpp/integration/scale_constraints.cc b/tests/cpp/integration/scale_constraints.cc index 7231cd4404..1b1e756bd5 100644 --- a/tests/cpp/integration/scale_constraints.cc +++ b/tests/cpp/integration/scale_constraints.cc @@ -53,6 +53,9 @@ struct ScaleTester : public legate::LegateTask> { void prepare() { + static bool prepared = false; + if (prepared) { return; } + prepared = true; auto runtime = legate::Runtime::get_runtime(); auto context = runtime->create_library(library_name); ScaleTester<1>::register_variants(context); diff --git a/tests/cpp/integration/tasks/task_simple.cc b/tests/cpp/integration/tasks/task_simple.cc index 17d8964c0d..955754a26a 100644 --- a/tests/cpp/integration/tasks/task_simple.cc +++ b/tests/cpp/integration/tasks/task_simple.cc @@ -20,6 +20,9 @@ Legion::Logger logger(library_name); void register_tasks() { + static bool prepared = false; + if (prepared) { return; } + prepared = true; auto runtime = legate::Runtime::get_runtime(); auto context = runtime->create_library(library_name); HelloTask::register_variants(context); diff --git a/tests/cpp/integration/tree_reduce.cc b/tests/cpp/integration/tree_reduce.cc index fb361c3808..05824fd73d 100644 --- a/tests/cpp/integration/tree_reduce.cc +++ b/tests/cpp/integration/tree_reduce.cc @@ -79,6 +79,9 @@ struct ReduceUnboundTask : public legate::LegateTask { void register_tasks() { + static bool prepared = false; + if (prepared) { return; } + prepared = true; auto runtime = legate::Runtime::get_runtime(); auto context = runtime->create_library(library_name); ProduceNormalTask::register_variants(context); diff --git a/tests/cpp/integration/weighted.cc b/tests/cpp/integration/weighted.cc index 9aa8f7d85b..90778777dd 100644 --- a/tests/cpp/integration/weighted.cc +++ b/tests/cpp/integration/weighted.cc @@ -53,6 +53,9 @@ struct Tester : public legate::LegateTask { void prepare() { + static bool prepared = false; + if (prepared) { return; } + prepared = true; auto runtime = legate::Runtime::get_runtime(); auto library = runtime->create_library(library_name); Initializer::register_variants(library, INIT); diff --git a/tests/cpp/unit/buffer.cc b/tests/cpp/unit/buffer.cc index 9cafbe1ae8..87d68ed540 100644 --- a/tests/cpp/unit/buffer.cc +++ b/tests/cpp/unit/buffer.cc @@ -85,6 +85,9 @@ void test_buffer(int32_t dim, uint64_t bytes, legate::Memory::Kind kind, size_t void register_tasks() { + static bool prepared = false; + if (prepared) { return; } + prepared = true; auto runtime = legate::Runtime::get_runtime(); auto context = runtime->create_library(library_name); BufferTask::register_variants(context); diff --git a/tests/cpp/unit/constraint.cc b/tests/cpp/unit/constraint.cc index 646b232b1b..695b2351aa 100644 --- a/tests/cpp/unit/constraint.cc +++ b/tests/cpp/unit/constraint.cc @@ -30,6 +30,9 @@ struct Initializer : public legate::LegateTask { void register_tasks() { + static bool prepared = false; + if (prepared) { return; } + prepared = true; auto runtime = legate::Runtime::get_runtime(); auto context = runtime->create_library(library_name); Initializer::register_variants(context); diff --git a/tests/cpp/unit/library.cc b/tests/cpp/unit/library.cc index 2c055cc853..81fbc58016 100644 --- a/tests/cpp/unit/library.cc +++ b/tests/cpp/unit/library.cc @@ -14,27 +14,32 @@ #include "legate.h" +namespace test_library { + TEST(Library, Create) { - auto* runtime = legate::Runtime::get_runtime(); - auto lib = runtime->create_library("libA"); - EXPECT_EQ(lib, runtime->find_library("libA")); - EXPECT_EQ(lib, runtime->maybe_find_library("libA").value()); + const char* LIBNAME = "test_library.libA"; + auto* runtime = legate::Runtime::get_runtime(); + auto lib = runtime->create_library(LIBNAME); + EXPECT_EQ(lib, runtime->find_library(LIBNAME)); + EXPECT_EQ(lib, runtime->maybe_find_library(LIBNAME).value()); } TEST(Library, FindOrCreate) { + const char* LIBNAME = "test_library.libB"; + auto* runtime = legate::Runtime::get_runtime(); legate::ResourceConfig config; config.max_tasks = 1; bool created = false; - auto p_lib1 = runtime->find_or_create_library("libA", config, nullptr, &created); + auto p_lib1 = runtime->find_or_create_library(LIBNAME, config, nullptr, &created); EXPECT_TRUE(created); config.max_tasks = 2; - auto p_lib2 = runtime->find_or_create_library("libA", config, nullptr, &created); + auto p_lib2 = runtime->find_or_create_library(LIBNAME, config, nullptr, &created); EXPECT_FALSE(created); EXPECT_EQ(p_lib1, p_lib2); EXPECT_TRUE(p_lib2.valid_task_id(p_lib2.get_task_id(0))); @@ -43,9 +48,13 @@ TEST(Library, FindOrCreate) TEST(Library, FindNonExistent) { + const char* LIBNAME = "test_library.libC"; + auto* runtime = legate::Runtime::get_runtime(); - EXPECT_THROW(runtime->find_library("libB"), std::out_of_range); + EXPECT_THROW(runtime->find_library(LIBNAME), std::out_of_range); - EXPECT_EQ(runtime->maybe_find_library("libB"), std::nullopt); + EXPECT_EQ(runtime->maybe_find_library(LIBNAME), std::nullopt); } + +} // namespace test_library diff --git a/tests/cpp/unit/registration.cc b/tests/cpp/unit/registration.cc index 5a1496a0f7..9e4fd26a09 100644 --- a/tests/cpp/unit/registration.cc +++ b/tests/cpp/unit/registration.cc @@ -33,7 +33,7 @@ struct GPUVariantTask : public legate::LegateTask> { void test_duplicates() { auto* runtime = legate::Runtime::get_runtime(); - auto library = runtime->create_library("libA"); + auto library = runtime->create_library("test_registration.libA"); test_registration::CPUVariantTask<0>::register_variants(library); EXPECT_THROW(test_registration::CPUVariantTask<0>::register_variants(library), std::invalid_argument); @@ -44,7 +44,7 @@ void test_out_of_bounds_task_id() legate::ResourceConfig config; config.max_tasks = 1; auto* runtime = legate::Runtime::get_runtime(); - auto library = runtime->create_library("libA", config); + auto library = runtime->create_library("test_registration.libB", config); EXPECT_THROW(test_registration::CPUVariantTask<1>::register_variants(library), std::out_of_range); } diff --git a/tests/cpp/unit/scoped_allocator.cc b/tests/cpp/unit/scoped_allocator.cc index 6a54ba225e..1125770fce 100644 --- a/tests/cpp/unit/scoped_allocator.cc +++ b/tests/cpp/unit/scoped_allocator.cc @@ -95,6 +95,9 @@ void test_allocator( void register_tasks() { + static bool prepared = false; + if (prepared) { return; } + prepared = true; auto runtime = legate::Runtime::get_runtime(); auto context = runtime->create_library(library_name); ScopedAllocatorTask::register_variants(context); From 2133ac17888d1da83f4e29d5417ed89e5124c7a0 Mon Sep 17 00:00:00 2001 From: Manolis Papadakis Date: Tue, 3 Oct 2023 20:53:33 -0700 Subject: [PATCH 0240/1425] Attach/Detach (#74) * Avoid using naked LogicalRegionField* * Remove unmapping from the public API * Move unmapping to root LogicalRegionField destructor * Using non-literal strings as format strings produces security warning * Print out compile_commands.json for C++ tests, to aid clangd * Fix typo * Wait for inline mapped PhysicalRegion to become valid before returning * Set provenance on InlineLauncher * Support for attach and detach * Testcase for attach * Fix ordering of dimensions * Small test system improvements * Fix collective= argument * Properly test out-of-order-destroyed stores, fix related bugs * Process review comments from #57 * Address review comments * Add negative tests, add missing unbound() check on detach * Give share and ordering args defaults --- src/core/data/detail/logical_region_field.cc | 66 +++++- src/core/data/detail/logical_region_field.h | 7 + src/core/data/detail/logical_store.cc | 32 ++- src/core/data/detail/logical_store.h | 12 +- src/core/data/detail/store.cc | 8 - src/core/data/detail/store.h | 6 - src/core/data/logical_store.cc | 2 + src/core/data/logical_store.h | 12 + src/core/data/store.cc | 2 - src/core/data/store.h | 6 - src/core/mapping/detail/mapping.cc | 7 +- src/core/mapping/detail/mapping.h | 3 +- src/core/operation/detail/req_analyzer.h | 2 +- src/core/partitioning/partition.cc | 2 +- src/core/runtime/detail/field_manager.cc | 55 +++-- src/core/runtime/detail/field_manager.h | 37 +-- src/core/runtime/detail/runtime.cc | 106 ++++++--- src/core/runtime/detail/runtime.h | 13 +- src/core/runtime/runtime.cc | 9 + src/core/runtime/runtime.h | 24 ++ src/core/runtime/tracker.h | 2 +- tests/cpp/CMakeLists.txt | 2 + tests/cpp/integration/attach.cc | 225 +++++++++++++++++++ tests/cpp/integration/inline_map.cc | 24 -- 24 files changed, 514 insertions(+), 150 deletions(-) create mode 100644 tests/cpp/integration/attach.cc diff --git a/src/core/data/detail/logical_region_field.cc b/src/core/data/detail/logical_region_field.cc index 0e2e3be643..06ed96033f 100644 --- a/src/core/data/detail/logical_region_field.cc +++ b/src/core/data/detail/logical_region_field.cc @@ -27,10 +27,31 @@ LogicalRegionField::LogicalRegionField(FieldManager* manager, LogicalRegionField::~LogicalRegionField() { - // Only free the field once the top-level region is deleted. + // Only free associated resources when the top-level region is deleted. if (parent_ == nullptr) { + // This is a misuse of the Legate API, so it should technically throw an exception, but we + // shouldn't throw exceptions in destructors, so we just abort. + if (attachment_shared_) { + log_legate.error() << "stores created by attaching to a buffer with share=true must be " + << "manually detached"; + LEGATE_ABORT; + } perform_invalidation_callbacks(); - manager_->free_field(lr_, fid_, destroyed_out_of_order_); + // We unmap the field immediately. In the case where a LogicalStore is allowed to be destroyed + // out-of-order, this unmapping might happen at different times on different shards. Unmapping + // doesn't go through the Legion pipeline, so from that perspective it's not critical that all + // shards unmap a region in the same order. The only problematic case is when shard A unmaps + // region R and shard B doesn't, then both shards launch a task that uses R (or any region that + // overlaps with R). Then B will unmap/remap around the task, whereas A will not. This shouldn't + // be an issue in Legate, because once a shard has (locally) freed a root RegionField, there + // should be no Stores remaining that use it (or any of its sub-regions). Moreover, the field + // will only start to get reused once all shards have agreed that it's been collected. + if (pr_.is_mapped()) Runtime::get_runtime()->unmap_physical_region(pr_); + Legion::Future can_dealloc = (nullptr == attachment_) + ? Legion::Future() // waiting on this is a noop + : Runtime::get_runtime()->detach( + pr_, false /*flush*/, destroyed_out_of_order_ /*unordered*/); + manager_->free_field(lr_, fid_, can_dealloc, attachment_, destroyed_out_of_order_); } } @@ -46,6 +67,47 @@ Domain LogicalRegionField::domain() const return Runtime::get_runtime()->get_index_space_domain(lr_.get_index_space()); } +RegionField LogicalRegionField::map() +{ + if (parent_ != nullptr) { + if (LegateDefined(LEGATE_USE_DEBUG)) assert(!pr_.exists()); + return parent_->map(); + } + if (!pr_.exists()) { + pr_ = Runtime::get_runtime()->map_region_field(lr_, fid_); + } else if (!pr_.is_mapped()) { + Runtime::get_runtime()->remap_physical_region(pr_); + } + return RegionField(dim(), pr_, fid_); +} + +void LogicalRegionField::attach(Legion::PhysicalRegion pr, void* buffer, bool share) +{ + if (LegateDefined(LEGATE_USE_DEBUG)) { + assert(nullptr == parent_); + assert(nullptr != buffer && pr.exists()); + assert(nullptr == attachment_ && !pr_.exists()); + } + pr_ = pr; + attachment_ = buffer; + attachment_shared_ = share; +} + +void LogicalRegionField::detach() +{ + if (nullptr != parent_) + throw std::invalid_argument("Manual detach must be called on the root store"); + if (!attachment_shared_) + throw std::invalid_argument("Only stores created with share=true can be manually detached"); + assert(nullptr != attachment_ && pr_.exists()); + if (pr_.is_mapped()) Runtime::get_runtime()->unmap_physical_region(pr_); + Legion::Future fut = Runtime::get_runtime()->detach(pr_, true /*flush*/, false /*unordered*/); + fut.get_void_result(true /*silence_warnings*/); + pr_ = Legion::PhysicalRegion(); + attachment_ = nullptr; + attachment_shared_ = false; +} + void LogicalRegionField::allow_out_of_order_destruction() { if (parent_ != nullptr) diff --git a/src/core/data/detail/logical_region_field.h b/src/core/data/detail/logical_region_field.h index 21974b2849..6c2ab80d64 100644 --- a/src/core/data/detail/logical_region_field.h +++ b/src/core/data/detail/logical_region_field.h @@ -16,6 +16,7 @@ #include #include +#include "core/data/detail/store.h" #include "legion.h" #include "core/data/shape.h" @@ -53,6 +54,9 @@ class LogicalRegionField : public std::enable_shared_from_this parent_; + Legion::PhysicalRegion pr_; + void* attachment_{nullptr}; + bool attachment_shared_{false}; bool destroyed_out_of_order_{false}; std::vector> callbacks_{}; }; diff --git a/src/core/data/detail/logical_store.cc b/src/core/data/detail/logical_store.cc index 25e3e33d38..092bafe97e 100644 --- a/src/core/data/detail/logical_store.cc +++ b/src/core/data/detail/logical_store.cc @@ -144,10 +144,10 @@ std::shared_ptr Storage::get_root() return nullptr == parent_ ? shared_from_this() : parent_->get_root(); } -LogicalRegionField* Storage::get_region_field() +std::shared_ptr Storage::get_region_field() { if (LegateDefined(LEGATE_USE_DEBUG)) { assert(Kind::REGION_FIELD == kind_); } - if (region_field_ != nullptr) return region_field_.get(); + if (region_field_ != nullptr) return region_field_; if (nullptr == parent_) { region_field_ = Runtime::get_runtime()->create_region_field(extents_, type_->size()); @@ -155,7 +155,7 @@ LogicalRegionField* Storage::get_region_field() } else region_field_ = parent_->get_child_data(color_); - return region_field_.get(); + return region_field_; } Legion::Future Storage::get_future() const @@ -187,7 +187,7 @@ void Storage::set_future(Legion::Future future) { future_ = future; } RegionField Storage::map() { if (LegateDefined(LEGATE_USE_DEBUG)) { assert(Kind::REGION_FIELD == kind_); } - return Runtime::get_runtime()->map_region_field(get_region_field()); + return get_region_field()->map(); } void Storage::allow_out_of_order_destruction() @@ -343,11 +343,6 @@ LogicalStore::LogicalStore(Shape&& extents, } } -LogicalStore::~LogicalStore() -{ - if (mapped_ != nullptr) mapped_->unmap(); -} - bool LogicalStore::unbound() const { return storage_->unbound(); } const Shape& LogicalStore::extents() const @@ -378,7 +373,10 @@ uint64_t LogicalStore::id() const { return store_id_; } const Storage* LogicalStore::get_storage() const { return storage_.get(); } -LogicalRegionField* LogicalStore::get_region_field() { return storage_->get_region_field(); } +std::shared_ptr LogicalStore::get_region_field() +{ + return storage_->get_region_field(); +} Legion::Future LogicalStore::get_future() { return storage_->get_future(); } @@ -559,7 +557,19 @@ std::shared_ptr LogicalStore::get_physical_store() return mapped_; } -void LogicalStore::allow_out_of_order_destruction() { storage_->allow_out_of_order_destruction(); } +void LogicalStore::detach() +{ + if (transformed()) throw std::invalid_argument("Manual detach must be called on the root store"); + if (has_scalar_storage() || unbound()) + throw std::invalid_argument("Only stores created with share=true can be manually detached"); + get_region_field()->detach(); +} + +void LogicalStore::allow_out_of_order_destruction() +{ + if (Runtime::get_runtime()->consensus_match_required()) + storage_->allow_out_of_order_destruction(); +} Restrictions LogicalStore::compute_restrictions() const { diff --git a/src/core/data/detail/logical_store.h b/src/core/data/detail/logical_store.h index a24877aea8..f98cfdce64 100644 --- a/src/core/data/detail/logical_store.h +++ b/src/core/data/detail/logical_store.h @@ -48,7 +48,7 @@ class Storage : public std::enable_shared_from_this { Storage(const Shape& extents, std::shared_ptr type, bool optimize_scalar); // Create a Future-backed storage. Initialized eagerly. Storage(const Shape& extents, std::shared_ptr type, const Legion::Future& future); - // Create a RegionField-bakced sub-storage. Initialized lazily. + // Create a RegionField-backed sub-storage. Initialized lazily. Storage(Shape&& extents, std::shared_ptr type, std::shared_ptr parent, @@ -71,7 +71,7 @@ class Storage : public std::enable_shared_from_this { std::shared_ptr get_root(); public: - LogicalRegionField* get_region_field(); + std::shared_ptr get_region_field(); Legion::Future get_future() const; void set_region_field(std::shared_ptr&& region_field); void set_future(Legion::Future future); @@ -160,9 +160,6 @@ class LogicalStore : public std::enable_shared_from_this { const std::shared_ptr& storage, std::shared_ptr&& transform); - public: - ~LogicalStore(); - private: LogicalStore(std::shared_ptr impl); @@ -188,7 +185,7 @@ class LogicalStore : public std::enable_shared_from_this { public: const Storage* get_storage() const; - LogicalRegionField* get_region_field(); + std::shared_ptr get_region_field(); Legion::Future get_future(); void set_region_field(std::shared_ptr&& region_field); void set_future(Legion::Future future); @@ -207,6 +204,7 @@ class LogicalStore : public std::enable_shared_from_this { public: std::shared_ptr get_physical_store(); + void detach(); // Informs the runtime that references to this store may be removed in non-deterministic order // (e.g. by an asynchronous garbage collector). // @@ -218,7 +216,7 @@ class LogicalStore : public std::enable_shared_from_this { // drops to 0 (which triggers object destruction) is not deterministic. // // Before passing a store to a garbage collected language, it must first be marked using this - // function, so that the runtime knows to work around the potentially non-determintic removal of + // function, so that the runtime knows to work around the potentially non-deterministic removal of // references. void allow_out_of_order_destruction(); diff --git a/src/core/data/detail/store.cc b/src/core/data/detail/store.cc index 0d8678ea26..d7a88884fc 100644 --- a/src/core/data/detail/store.cc +++ b/src/core/data/detail/store.cc @@ -73,8 +73,6 @@ struct get_domain_fn { Domain RegionField::domain() const { return dim_dispatch(dim_, get_domain_fn{}, pr_); } -void RegionField::unmap() { detail::Runtime::get_runtime()->unmap_physical_region(pr_); } - UnboundRegionField::UnboundRegionField(const Legion::OutputRegion& out, Legion::FieldID fid) : num_elements_( Legion::UntypedDeferredValue(sizeof(size_t), find_memory_kind_for_executing_processor())), @@ -341,12 +339,6 @@ Domain Store::domain() const return result; } -void Store::unmap() -{ - if (is_future_ || is_unbound_store_) return; - region_field_.unmap(); -} - void Store::bind_empty_data() { check_valid_binding(true); diff --git a/src/core/data/detail/store.h b/src/core/data/detail/store.h index ca1f68cf56..4fab0f7416 100644 --- a/src/core/data/detail/store.h +++ b/src/core/data/detail/store.h @@ -46,9 +46,6 @@ class RegionField { public: Domain domain() const; - public: - void unmap(); - public: bool is_readable() const { return readable_; } bool is_writable() const { return writable_; } @@ -192,9 +189,6 @@ class Store { public: Domain domain() const; - public: - void unmap(); - public: bool is_readable() const { return readable_; } bool is_writable() const { return writable_; } diff --git a/src/core/data/logical_store.cc b/src/core/data/logical_store.cc index b35366b550..b2ccbe4ea8 100644 --- a/src/core/data/logical_store.cc +++ b/src/core/data/logical_store.cc @@ -69,6 +69,8 @@ LogicalStore LogicalStore::delinearize(int32_t dim, std::vector&& sizes Store LogicalStore::get_physical_store() const { return Store(impl_->get_physical_store()); } +void LogicalStore::detach() { impl_->detach(); } + LogicalStorePartition::LogicalStorePartition(std::shared_ptr&& impl) : impl_(std::move(impl)) { diff --git a/src/core/data/logical_store.h b/src/core/data/logical_store.h index 21444791fd..bdb7736e54 100644 --- a/src/core/data/logical_store.h +++ b/src/core/data/logical_store.h @@ -327,6 +327,18 @@ class LogicalStore { */ Store get_physical_store() const; + /** + * @brief Detach a store from its attached memory + * + * This should only be called on a store created by attaching to a buffer with share=true. + * + * This call will wait for all operations that use the store (or any sub-store) to complete. + * + * After this call returns, the contents of the attached buffer will be up-to-date, and it is safe + * to deallocate it. The contents of the store are invalid after that point. + */ + void detach(); + public: std::shared_ptr impl() const { return impl_; } diff --git a/src/core/data/store.cc b/src/core/data/store.cc index c4aabb2465..1e9cb59e0a 100644 --- a/src/core/data/store.cc +++ b/src/core/data/store.cc @@ -39,8 +39,6 @@ bool Store::is_future() const { return impl_->is_future(); } bool Store::is_unbound_store() const { return impl_->is_unbound_store(); } -void Store::unmap() { impl_->unmap(); } - Store::Store(const Array& array) : impl_(array.data().impl()) {} void Store::check_accessor_dimension(const int32_t dim) const diff --git a/src/core/data/store.h b/src/core/data/store.h index 6390ec55f8..101add1de2 100644 --- a/src/core/data/store.h +++ b/src/core/data/store.h @@ -318,12 +318,6 @@ class Store { */ bool is_unbound_store() const; - public: - /** - * @brief Releases all inline allocations of the store - */ - void unmap(); - public: /** * @brief Constructs a store out of an array diff --git a/src/core/mapping/detail/mapping.cc b/src/core/mapping/detail/mapping.cc index 1df78c0c2a..99140e101b 100644 --- a/src/core/mapping/detail/mapping.cc +++ b/src/core/mapping/detail/mapping.cc @@ -63,19 +63,17 @@ LegateVariantCode to_variant_code(TaskTarget target) return LEGATE_CPU_VARIANT; } -void DimOrdering::populate_dimension_ordering(const Store* store, +void DimOrdering::populate_dimension_ordering(int32_t dim, std::vector& ordering) const { // TODO: We need to implement the relative dimension ordering switch (kind) { case Kind::C: { - auto dim = store->region_field().dim(); for (int32_t idx = dim - 1; idx >= 0; --idx) ordering.push_back(static_cast(DIM_X + idx)); break; } case Kind::FORTRAN: { - auto dim = store->region_field().dim(); for (int32_t idx = 0; idx < dim; ++idx) ordering.push_back(static_cast(DIM_X + idx)); break; @@ -147,7 +145,8 @@ void StoreMapping::populate_layout_constraints( std::vector dimension_ordering{}; if (policy.layout == InstLayout::AOS) dimension_ordering.push_back(DIM_F); - policy.ordering.impl()->populate_dimension_ordering(stores.front(), dimension_ordering); + policy.ordering.impl()->populate_dimension_ordering(stores.front()->region_field().dim(), + dimension_ordering); if (policy.layout == InstLayout::SOA) dimension_ordering.push_back(DIM_F); layout_constraints.add_constraint( diff --git a/src/core/mapping/detail/mapping.h b/src/core/mapping/detail/mapping.h index 40eb0d586d..6ca18e47d8 100644 --- a/src/core/mapping/detail/mapping.h +++ b/src/core/mapping/detail/mapping.h @@ -45,8 +45,7 @@ struct DimOrdering { } public: - void populate_dimension_ordering(const Store* store, - std::vector& ordering) const; + void populate_dimension_ordering(int32_t dim, std::vector& ordering) const; public: Kind kind; diff --git a/src/core/operation/detail/req_analyzer.h b/src/core/operation/detail/req_analyzer.h index 7413088c92..682509ea53 100644 --- a/src/core/operation/detail/req_analyzer.h +++ b/src/core/operation/detail/req_analyzer.h @@ -123,7 +123,7 @@ class FutureAnalyzer { struct StoreAnalyzer { public: - void insert(const LogicalRegionField* region_field, + void insert(std::shared_ptr region_field, Legion::PrivilegeMode privilege, const ProjectionInfo& proj_info) { diff --git a/src/core/partitioning/partition.cc b/src/core/partitioning/partition.cc index 897193d961..144c71ec2b 100644 --- a/src/core/partitioning/partition.cc +++ b/src/core/partitioning/partition.cc @@ -373,7 +373,7 @@ std::unique_ptr Image::bloat(const Shape& low_offsts, const Shape& hi Legion::LogicalPartition Image::construct(Legion::LogicalRegion region, bool complete) const { if (!has_launch_domain()) { return Legion::LogicalPartition::NO_PART; } - auto* func_rf = func_->get_region_field(); + auto func_rf = func_->get_region_field(); auto func_region = func_rf->region(); auto func_partition = func_partition_->construct(func_region, func_partition_->is_complete_for(func_->get_storage())); diff --git a/src/core/runtime/detail/field_manager.cc b/src/core/runtime/detail/field_manager.cc index 2adf2d477a..63c6b044e5 100644 --- a/src/core/runtime/detail/field_manager.cc +++ b/src/core/runtime/detail/field_manager.cc @@ -31,10 +31,14 @@ std::shared_ptr FieldManager::allocate_field() while (!ordered_free_fields_.empty() || !matches_.empty()) { // If there's a field that every shard is guaranteed to have, re-use that. if (!ordered_free_fields_.empty()) { - const auto& field = ordered_free_fields_.front(); - auto* rf = new LogicalRegionField(this, field.first, field.second); + const auto& info = ordered_free_fields_.front(); + if (nullptr != info.attachment) { + info.can_dealloc.get_void_result(true /*silence_warnings*/); + free(info.attachment); + } + auto* rf = new LogicalRegionField(this, info.region, info.field_id); log_legate.debug( - "Field %u recycled in field manager %p", field.second, static_cast(this)); + "Field %u recycled in field manager %p", info.field_id, static_cast(this)); ordered_free_fields_.pop_front(); return std::shared_ptr(rf); } @@ -62,16 +66,18 @@ std::shared_ptr FieldManager::import_field(const Legion::Log void FieldManager::free_field(const Legion::LogicalRegion& region, Legion::FieldID field_id, + Legion::Future can_dealloc, + void* attachment, bool unordered) { - if (unordered && runtime_->consensus_match_required()) { + if (unordered) { log_legate.debug( "Field %u freed locally in field manager %p", field_id, static_cast(this)); - unordered_free_fields_.push_back(FreeField(region, field_id)); + unordered_free_fields_.push_back(FreeFieldInfo{region, field_id, can_dealloc, attachment}); } else { log_legate.debug( "Field %u freed in-order in field manager %p", field_id, static_cast(this)); - ordered_free_fields_.push_back(FreeField(region, field_id)); + ordered_free_fields_.push_back(FreeFieldInfo{region, field_id, can_dealloc, attachment}); } } @@ -84,49 +90,54 @@ void FieldManager::issue_field_match() field_match_counter_ = 0; // We need to separately record the region that corresponds to each item in this match, because // the match itself only uses a subset of the full region info. - auto& regions = regions_for_match_items_.emplace_back(); + auto& infos = info_for_match_items_.emplace_back(); std::vector input; input.reserve(unordered_free_fields_.size()); - for (const auto& field : unordered_free_fields_) { - MatchItem item{field.first.get_tree_id(), field.second}; + for (const auto& info : unordered_free_fields_) { + MatchItem item{info.region.get_tree_id(), info.field_id}; input.push_back(item); - regions[item] = field.first; + infos[item] = info; } - assert(regions.size() == unordered_free_fields_.size()); + assert(infos.size() == unordered_free_fields_.size()); unordered_free_fields_.clear(); // Dispatch the match and put it on the queue of outstanding matches, but don't block on it yet. // We'll do that when we run out of ordered fields. matches_.push_back(runtime_->issue_consensus_match(std::move(input))); log_legate.debug("Consensus match emitted with %zu local fields in field manager %p", - regions.size(), + infos.size(), static_cast(this)); } void FieldManager::process_next_field_match() { assert(!matches_.empty()); - auto& match = matches_.front(); - auto& regions = regions_for_match_items_.front(); + auto& match = matches_.front(); + auto& infos = info_for_match_items_.front(); match.wait(); log_legate.debug("Consensus match result in field manager %p: %zu/%zu fields matched", static_cast(this), match.output().size(), match.input().size()); + // Ask the runtime to find all unordered operations that have been observed on all shards, and + // dispatch them right now. This will cover any unordered detachments on fields that all shards + // just matched on; if a LogicalRegionField has been destroyed on all shards, that means its + // detachment has also been emitted on all shards. This makes it safe to later block on any of + // those detachments, since they are all guaranteed to be in the task stream now, and will + // eventually complete. + runtime_->progress_unordered_operations(); // Put all the matched fields into the ordered queue, in the same order as the match result, // which is the same order that all shards will see. for (const auto& item : match.output()) { - auto it = regions.find(item); - assert(it != regions.end()); - ordered_free_fields_.push_back(FreeField(it->second, item.fid)); - regions.erase(it); + auto it = infos.find(item); + assert(it != infos.end()); + ordered_free_fields_.push_back(it->second); + infos.erase(it); } // All fields that weren't matched can go back into the unordered queue, to be included in the // next consensus match that we run. - for (const auto& [item, lr] : regions) { - unordered_free_fields_.push_back(FreeField(lr, item.fid)); - } + for (const auto& [item, info] : infos) { unordered_free_fields_.push_back(info); } matches_.pop_front(); - regions_for_match_items_.pop_front(); + info_for_match_items_.pop_front(); } } // namespace legate::detail diff --git a/src/core/runtime/detail/field_manager.h b/src/core/runtime/detail/field_manager.h index ae43299d38..2e3c55060e 100644 --- a/src/core/runtime/detail/field_manager.h +++ b/src/core/runtime/detail/field_manager.h @@ -23,6 +23,22 @@ namespace legate::detail { class LogicalRegionField; class Runtime; +struct FreeFieldInfo { + Legion::LogicalRegion region; + Legion::FieldID field_id; + Legion::Future can_dealloc; + void* attachment; +}; + +struct MatchItem { + Legion::RegionTreeID tid; + Legion::FieldID fid; + friend bool operator<(const MatchItem& l, const MatchItem& r) + { + return std::tie(l.tid, l.fid) < std::tie(r.tid, r.fid); + } +}; + class FieldManager { private: friend LogicalRegionField; @@ -36,7 +52,11 @@ class FieldManager { Legion::FieldID field_id); private: - void free_field(const Legion::LogicalRegion& region, Legion::FieldID field_id, bool unordered); + void free_field(const Legion::LogicalRegion& region, + Legion::FieldID field_id, + Legion::Future can_dealloc, + void* attachment, + bool unordered); void issue_field_match(); void process_next_field_match(); @@ -47,25 +67,16 @@ class FieldManager { uint32_t field_match_counter_{0}; private: - using FreeField = std::pair; // This is a sanitized list of (region,field_id) pairs that is guaranteed to be ordered across all // the shards even with control replication. - std::deque ordered_free_fields_; + std::deque ordered_free_fields_; // This list contains the fields that we know have been freed on this shard, but may not have been // freed yet on other shards. - std::vector unordered_free_fields_; + std::vector unordered_free_fields_; private: - struct MatchItem { - Legion::RegionTreeID tid; - Legion::FieldID fid; - friend bool operator<(const MatchItem& l, const MatchItem& r) - { - return std::tie(l.tid, l.fid) < std::tie(r.tid, r.fid); - } - }; std::deque> matches_; - std::deque> regions_for_match_items_; + std::deque> info_for_match_items_; }; } // namespace legate::detail diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index cb5213d2d4..49ae5f61bb 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -450,6 +450,46 @@ std::shared_ptr Runtime::create_store(const Scalar& scalar) return std::make_shared(std::move(storage)); } +std::shared_ptr Runtime::create_store(const Shape& extents, + std::shared_ptr type, + void* buffer, + bool share, + const mapping::detail::DimOrdering* ordering) +{ + if (type->variable_size()) + throw std::invalid_argument( + "stores created by attaching to a buffer must have fixed-size type"); + if (nullptr == buffer) throw std::invalid_argument("buffer to attach to cannot be NULL"); + + std::shared_ptr store = create_store(extents, type, false /*optimize_scalar*/); + std::shared_ptr rf = store->get_region_field(); + + if (!share) { + auto nbytes = extents.volume() * type->size(); + void* copy = malloc(nbytes); + if (nullptr == copy) throw std::bad_alloc(); + memcpy(copy, buffer, nbytes); + buffer = copy; + } + + Legion::AttachLauncher launcher( + LEGION_EXTERNAL_INSTANCE, rf->region(), rf->region(), false /*restricted*/, false /*mapped*/); + // the value of column_major below is irrelevant; it will be updated using the ordering constraint + launcher.attach_array_soa(buffer, false /*column_major*/, {rf->field_id()}); + launcher.collective = true; // each shard will attach a full local copy of the entire buffer + launcher.provenance = provenance_manager()->get_provenance(); + launcher.constraints.ordering_constraint.ordering.clear(); + ordering->populate_dimension_ordering(store->dim(), + launcher.constraints.ordering_constraint.ordering); + launcher.constraints.ordering_constraint.ordering.push_back(DIM_F); + Legion::PhysicalRegion pr = legion_runtime_->attach_external_resource(legion_context_, launcher); + // no need to wait on the returned PhysicalRegion, since we're not inline-mapping + // but we can keep it around and remap it later if the user asks + rf->attach(pr, buffer, share); + + return store; +} + void Runtime::check_dimensionality(uint32_t dim) { if (dim > LEGATE_MAX_DIM) { @@ -529,52 +569,43 @@ std::shared_ptr Runtime::import_region_field(Legion::Logical return fld_mgr->import_field(region, field_id); } -RegionField Runtime::map_region_field(const LogicalRegionField* rf) +Legion::PhysicalRegion Runtime::map_region_field(Legion::LogicalRegion region, + Legion::FieldID field_id) { - auto root_region = rf->get_root().region(); - auto field_id = rf->field_id(); - - Legion::PhysicalRegion pr; - - RegionFieldID key(root_region, field_id); - auto finder = inline_mapped_.find(key); - if (inline_mapped_.end() == finder) { - Legion::RegionRequirement req(root_region, READ_WRITE, EXCLUSIVE, root_region); - req.add_field(field_id); + Legion::RegionRequirement req(region, READ_WRITE, EXCLUSIVE, region); + req.add_field(field_id); + auto mapper_id = core_library_->get_mapper_id(); + // TODO: We need to pass the metadata about logical store + Legion::InlineLauncher launcher(req, mapper_id); + launcher.provenance = provenance_manager()->get_provenance(); + Legion::PhysicalRegion pr = legion_runtime_->map_region(legion_context_, launcher); + pr.wait_until_valid(true /*silence_warnings*/); + return pr; +} - auto mapper_id = core_library_->get_mapper_id(); - // TODO: We need to pass the metadata about logical store - Legion::InlineLauncher launcher(req, mapper_id); - pr = legion_runtime_->map_region(legion_context_, launcher); - inline_mapped_[key] = pr; - } else - pr = finder->second; - physical_region_refs_.add(pr); - return RegionField(rf->dim(), pr, field_id); +void Runtime::remap_physical_region(Legion::PhysicalRegion pr) +{ + legion_runtime_->remap_region( + legion_context_, pr, provenance_manager()->get_provenance().c_str()); + pr.wait_until_valid(true /*silence_warnings*/); } void Runtime::unmap_physical_region(Legion::PhysicalRegion pr) { - // TODO: Unmapping doesn't go through the Legion pipeline, so from that perspective it's not - // critical that all shards call `unmap_region` in the same order. However, if shard A unmaps - // region R and shard B doesn't, then both shards launch a task that uses R (or any region that - // overlaps with R), then B will unmap/remap around the task, whereas A will not. To be safe, we - // should consider delaying the unmapping until the field has gone through consensus match, or - // have a full consensus matching process just for unmapping. - if (physical_region_refs_.remove(pr)) { - // The last user of this inline mapping was removed, so remove it from our cache and unmap. + if (LegateDefined(LEGATE_USE_DEBUG)) { std::vector fields; pr.get_fields(fields); assert(fields.size() == 1); - RegionFieldID key(pr.get_logical_region(), fields[0]); - auto finder = inline_mapped_.find(key); - assert(finder != inline_mapped_.end() && finder->second == pr); - inline_mapped_.erase(finder); - legion_runtime_->unmap_region(legion_context_, pr); } + legion_runtime_->unmap_region(legion_context_, pr); } -size_t Runtime::num_inline_mapped() const { return inline_mapped_.size(); } +Legion::Future Runtime::detach(Legion::PhysicalRegion pr, bool flush, bool unordered) +{ + assert(pr.exists() && !pr.is_mapped()); + return legion_runtime_->detach_external_resource( + legion_context_, pr, flush, unordered, provenance_manager()->get_provenance().c_str()); +} uint32_t Runtime::field_reuse_freq() const { return field_reuse_freq_; } @@ -583,6 +614,11 @@ bool Runtime::consensus_match_required() const return force_consensus_match_ || Legion::Machine::get_machine().get_address_space_count() > 1; } +void Runtime::progress_unordered_operations() const +{ + legion_runtime_->progress_unordered_operations(legion_context_); +} + RegionManager* Runtime::find_or_create_region_manager(const Domain& shape) { auto finder = region_managers_.find(shape); @@ -1095,8 +1131,6 @@ void Runtime::destroy() // Any STL containers holding Legion handles need to be cleared here, otherwise they cause // trouble when they get destroyed in the Legate runtime's destructor - inline_mapped_.clear(); - physical_region_refs_.clear(); pending_exceptions_.clear(); // We finally deallocate managers diff --git a/src/core/runtime/detail/runtime.h b/src/core/runtime/detail/runtime.h index dc0be9c07b..da35d287fb 100644 --- a/src/core/runtime/detail/runtime.h +++ b/src/core/runtime/detail/runtime.h @@ -146,6 +146,11 @@ class Runtime { std::shared_ptr type, bool optimize_scalar = false); std::shared_ptr create_store(const Scalar& scalar); + std::shared_ptr create_store(const Shape& extents, + std::shared_ptr type, + void* buffer, + bool share, + const mapping::detail::DimOrdering* ordering); private: void check_dimensionality(uint32_t dim); @@ -167,11 +172,13 @@ class Runtime { std::shared_ptr import_region_field(Legion::LogicalRegion region, Legion::FieldID field_id, uint32_t field_size); - RegionField map_region_field(const LogicalRegionField* region_field); + Legion::PhysicalRegion map_region_field(Legion::LogicalRegion region, Legion::FieldID field_id); + void remap_physical_region(Legion::PhysicalRegion pr); void unmap_physical_region(Legion::PhysicalRegion pr); - size_t num_inline_mapped() const; + Legion::Future detach(Legion::PhysicalRegion pr, bool flush, bool unordered); uint32_t field_reuse_freq() const; bool consensus_match_required() const; + void progress_unordered_operations() const; public: RegionManager* find_or_create_region_manager(const Legion::Domain& shape); @@ -317,8 +324,6 @@ class Runtime { private: using RegionFieldID = std::pair; - std::map inline_mapped_; - MultiSet physical_region_refs_; uint64_t next_store_id_{1}; uint64_t next_storage_id_{1}; const uint32_t field_reuse_freq_; diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 7ec611c25b..88e23309fe 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -188,6 +188,15 @@ LogicalStore Runtime::create_store(const Scalar& scalar) return LogicalStore(impl_->create_store(*scalar.impl_)); } +LogicalStore Runtime::create_store(const Shape& extents, + const Type& type, + void* buffer, + bool share, + const mapping::DimOrdering& ordering) +{ + return LogicalStore(impl_->create_store(extents, type.impl(), buffer, share, ordering.impl())); +} + uint32_t Runtime::max_pending_exceptions() const { return impl_->max_pending_exceptions(); } void Runtime::set_max_pending_exceptions(uint32_t max_pending_exceptions) diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 409975f9db..241d3366df 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -378,6 +378,30 @@ class Runtime { * @return Logical store */ LogicalStore create_store(const Scalar& scalar); + /** + * @brief Creates a store by attaching to existing memory. + * + * If `share` is false, then Legate will make an internal copy of the passed buffer. + * + * If `share` is true, then the existing buffer will be reused, and Legate will update it + * according to any modifications made to the returned store. The caller must keep the buffer + * alive until they explicitly call `detach` on the result store. The contents of the attached + * buffer are only guaranteed to be up-to-date after `detach` returns. + * + * @param extents Shape of the store + * @param type Element type + * @param buffer Pointer to the beginning of the memory to attach to; memory must be contiguous, + * and cover the entire contents of the store (at least `extents.volume() * type.size()` bytes) + * @param ordering In what order the elements are laid out in the passed buffer + * @param share Whether to reuse the passed buffer in-place + * + * @return Logical store + */ + LogicalStore create_store(const Shape& extents, + const Type& type, + void* buffer, + bool share = false, + const mapping::DimOrdering& ordering = mapping::DimOrdering::c_order()); public: /** diff --git a/src/core/runtime/tracker.h b/src/core/runtime/tracker.h index d3be930658..1e896b2633 100644 --- a/src/core/runtime/tracker.h +++ b/src/core/runtime/tracker.h @@ -33,7 +33,7 @@ namespace legate { */ struct ProvenanceTracker { /** - * @brief Creates a trakcer that sets a given provenance string for the scope + * @brief Creates a tracker that sets a given provenance string for the scope * * @param provenance Provenance information in string */ diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt index a5eb731e8e..4a30b3666f 100644 --- a/tests/cpp/CMakeLists.txt +++ b/tests/cpp/CMakeLists.txt @@ -28,6 +28,8 @@ set(CMAKE_CXX_FLAGS_MINSIZEREL "-Os") set(CMAKE_CUDA_FLAGS_MINSIZEREL "-Os") set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g") set(CMAKE_CUDA_FLAGS_RELWITHDEBINFO "-O2 -g") +# Needed to integrate with LLVM/clang tooling +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) include(FetchContent) FetchContent_Declare( diff --git a/tests/cpp/integration/attach.cc b/tests/cpp/integration/attach.cc new file mode 100644 index 0000000000..c7766db75e --- /dev/null +++ b/tests/cpp/integration/attach.cc @@ -0,0 +1,225 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include + +#include "core/data/detail/logical_store.h" +#include "legate.h" + +namespace attach { + +static const char* library_name = "test_attach"; + +enum TaskOpCode { + ADDER = 0, + CHECKER = 1, +}; + +static legate::Shape SHAPE_1D{5}; + +static legate::Shape SHAPE_2D{3, 4}; + +void increment_physical_store(const legate::Store& store, int32_t dim) +{ + if (dim == 1) { + auto shape = store.shape<1>(); + auto acc = store.read_write_accessor(shape); + for (legate::PointInRectIterator<1> it(shape); it.valid(); ++it) acc[*it] += 1; + } else { + auto shape = store.shape<2>(); + auto acc = store.read_write_accessor(shape); + for (legate::PointInRectIterator<2> it(shape); it.valid(); ++it) acc[*it] += 1; + } +} + +void check_physical_store(const legate::Store& store, int32_t dim, int64_t counter) +{ + if (dim == 1) { + auto shape = store.shape<1>(); + auto acc = store.read_accessor(shape); + for (int32_t i = 0; i < SHAPE_1D[0]; ++i) EXPECT_EQ(acc[i], counter++); + } else { + auto shape = store.shape<2>(); + auto acc = store.read_accessor(shape); + // Legate should always see elements in the expected order + for (int32_t i = 0; i < SHAPE_2D[0]; ++i) + for (int32_t j = 0; j < SHAPE_2D[1]; ++j) EXPECT_EQ(acc[legate::Point<2>(i, j)], counter++); + } +} + +struct AdderTask : public legate::LegateTask { + static const int32_t TASK_ID = ADDER; + static void cpu_variant(legate::TaskContext context) + { + auto output = context.output(0).data(); + int32_t dim = context.scalar(0).value(); + increment_physical_store(output, dim); + } +}; + +struct CheckerTask : public legate::LegateTask { + static const int32_t TASK_ID = CHECKER; + static void cpu_variant(legate::TaskContext context) + { + auto input = context.input(0).data(); + int32_t dim = context.scalar(0).value(); + int64_t counter = context.scalar(1).value(); + check_physical_store(input, dim, counter); + } +}; + +void register_tasks() +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->create_library(library_name); + AdderTask::register_variants(context); + CheckerTask::register_variants(context); +} + +int64_t* make_buffer(int32_t dim, bool fortran) +{ + int64_t* buffer; + int64_t counter = 0; + if (dim == 1) { + buffer = new int64_t[SHAPE_1D.volume()]; + for (int32_t i = 0; i < SHAPE_1D[0]; ++i) buffer[i] = counter++; + } else { + buffer = new int64_t[SHAPE_2D.volume()]; + for (int32_t i = 0; i < SHAPE_2D[0]; ++i) + for (int32_t j = 0; j < SHAPE_2D[1]; ++j) + if (fortran) + buffer[j * SHAPE_2D[0] + i] = counter++; + else + buffer[i * SHAPE_2D[1] + j] = counter++; + } + return buffer; +} + +void check_and_delete_buffer(int64_t* buffer, int32_t dim, bool fortran, int64_t counter) +{ + if (dim == 1) + for (int32_t i = 0; i < SHAPE_1D[0]; ++i) EXPECT_EQ(buffer[i], counter++); + else + for (int32_t i = 0; i < SHAPE_2D[0]; ++i) + for (int32_t j = 0; j < SHAPE_2D[1]; ++j) + if (fortran) + EXPECT_EQ(buffer[j * SHAPE_2D[0] + i], counter++); + else + EXPECT_EQ(buffer[i * SHAPE_2D[1] + j], counter++); + delete buffer; +} + +void test_body( + int32_t dim, bool fortran, bool unordered, bool share, bool use_tasks, bool use_inline) +{ + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + int64_t counter = 0; + int64_t* buffer = make_buffer(dim, fortran); + auto l_store = runtime->create_store(dim == 1 ? SHAPE_1D : SHAPE_2D, + legate::int64(), + buffer, + share, + fortran ? legate::mapping::DimOrdering::fortran_order() + : legate::mapping::DimOrdering::c_order()); + if (unordered) l_store.impl()->allow_out_of_order_destruction(); + if (!share) check_and_delete_buffer(buffer, dim, fortran, counter); + for (int32_t iter = 0; iter < 2; ++iter) { + if (use_tasks) { + auto task = runtime->create_task(context, ADDER, {1}); + task.add_input(l_store); + task.add_output(l_store); + task.add_scalar_arg(dim); + runtime->submit(std::move(task)); + ++counter; + } + if (use_inline) { + auto p_store = l_store.get_physical_store(); + increment_physical_store(p_store, dim); + ++counter; + } + } + if (use_tasks) { + auto task = runtime->create_task(context, CHECKER, {1}); + task.add_input(l_store); + task.add_scalar_arg(dim); + task.add_scalar_arg(counter); + runtime->submit(std::move(task)); + } + if (use_inline) { + auto p_store = l_store.get_physical_store(); + check_physical_store(p_store, dim, counter); + } + if (share) { + l_store.detach(); + check_and_delete_buffer(buffer, dim, fortran, counter); + } +} + +TEST(Attach, Positive) +{ + legate::Core::perform_registration(); + // Do multiple operations in a row, with stores collected in-between, in hopes of triggering + // consensus match. + // TODO: Also try keeping multiple stores alive at one time. + for (auto& [dim, fortran] : std::vector>{ + std::make_pair(1, false), std::make_pair(2, false), std::make_pair(2, true)}) + for (bool unordered : {false, true}) + for (bool share : {false, true}) + for (bool use_tasks : {false, true}) + for (bool use_inline : {false, true}) + test_body(dim, fortran, unordered, share, use_tasks, use_inline); +} + +TEST(Attach, Negative) +{ + legate::Core::perform_registration(); + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + // Trying to detach a store without an attachment + EXPECT_THROW(runtime->create_store(legate::Scalar(42)).detach(), std::invalid_argument); + EXPECT_THROW(runtime->create_store(SHAPE_2D, legate::int64()).detach(), std::invalid_argument); + EXPECT_THROW(runtime->create_store(legate::int64()).detach(), std::invalid_argument); + + // Trying to attach to a NULL buffer + EXPECT_THROW(runtime->create_store(SHAPE_2D, legate::int64(), nullptr, true), + std::invalid_argument); + + // Trying to detach a non-shared attachment + EXPECT_THROW( + runtime->create_store(SHAPE_1D, legate::int64(), new int64_t[SHAPE_1D.volume()]).detach(), + std::invalid_argument); + + // Trying to detach a sub-store + auto l_store = runtime->create_store( + SHAPE_1D, legate::int64(), new int64_t[SHAPE_1D.volume()], true /*share*/); + EXPECT_THROW(l_store.project(0, 1).detach(), std::invalid_argument); + // We have to properly detach this, to avoid the abort in the destructor + l_store.detach(); +} + +// TODO: GTest implements death tests by forking, and that is not going to work well with Realm +// and MPI. +TEST(DISABLED_AttachDeathTest, MissingManualDetach) +{ + legate::Core::perform_registration(); + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + + // Forgetting to detach a shared attachment + EXPECT_DEATH(runtime->create_store( + SHAPE_1D, legate::int64(), new int64_t[SHAPE_1D.volume()], true /*share*/), + "must be manually detached"); +} + +} // namespace attach diff --git a/tests/cpp/integration/inline_map.cc b/tests/cpp/integration/inline_map.cc index 8a1f4c1ff0..126184494e 100644 --- a/tests/cpp/integration/inline_map.cc +++ b/tests/cpp/integration/inline_map.cc @@ -41,18 +41,6 @@ void register_tasks() AdderTask::register_variants(context); } -void test_mapped_regions_leak() -{ - auto runtime = legate::Runtime::get_runtime(); - { - auto l_store = runtime->create_store({5}, legate::int64()); - auto p_store = l_store.get_physical_store(); - EXPECT_FALSE(p_store.is_future()); - EXPECT_EQ(runtime->impl()->num_inline_mapped(), 1); - } - EXPECT_EQ(runtime->impl()->num_inline_mapped(), 0); -} - void test_inline_map_future() { auto runtime = legate::Runtime::get_runtime(); @@ -95,16 +83,6 @@ void test_inline_map_and_task() EXPECT_EQ(acc[2], 43); } -void test_inline_map_unmap() -{ - auto runtime = legate::Runtime::get_runtime(); - auto logical_store = runtime->create_store({1, 3}, legate::int64()); - auto store = logical_store.get_physical_store(); - store.unmap(); -} - -TEST(InlineMap, MappedRegionsLeak) { test_mapped_regions_leak(); } - TEST(InlineMap, Future) { test_inline_map_future(); } TEST(InlineMap, RegionAndSlice) { test_inline_map_region_and_slice(); } @@ -115,6 +93,4 @@ TEST(InlineMap, WithTask) test_inline_map_and_task(); } -TEST(InlineMap, Unmap) { test_inline_map_unmap(); } - } // namespace inline_map From 08dc7960aef6923675ba652f16b74d3155ffdb9b Mon Sep 17 00:00:00 2001 From: Manolis Papadakis Date: Tue, 3 Oct 2023 22:44:05 -0700 Subject: [PATCH 0241/1425] Fixes for new registration callback system (#105) --- src/core/runtime/detail/library.cc | 4 ---- src/core/runtime/detail/library.h | 2 ++ tests/cpp/integration/attach.cc | 6 +++--- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/core/runtime/detail/library.cc b/src/core/runtime/detail/library.cc index 496dc71479..1e64b0d7d3 100644 --- a/src/core/runtime/detail/library.cc +++ b/src/core/runtime/detail/library.cc @@ -119,8 +119,6 @@ const std::string& Library::get_task_name(int64_t local_task_id) const return find_task(local_task_id)->name(); } -namespace { - void register_mapper_callback(const Legion::RegistrationCallbackArgs& args) { const std::string library_name(static_cast(args.buffer.get_ptr())); @@ -131,8 +129,6 @@ void register_mapper_callback(const Legion::RegistrationCallbackArgs& args) Legion::Runtime::get_runtime()->add_mapper(library->get_mapper_id(), legion_mapper); } -} // namespace - void Library::register_mapper(std::unique_ptr mapper, bool in_callback) { // Hold the pointer to the mapper to keep it alive diff --git a/src/core/runtime/detail/library.h b/src/core/runtime/detail/library.h index 4a5dd97e11..71f22d4a45 100644 --- a/src/core/runtime/detail/library.h +++ b/src/core/runtime/detail/library.h @@ -24,6 +24,8 @@ namespace legate::detail { class Runtime; +void register_mapper_callback(const Legion::RegistrationCallbackArgs& args); + class Library { private: class ResourceIdScope { diff --git a/tests/cpp/integration/attach.cc b/tests/cpp/integration/attach.cc index c7766db75e..e82c98c7ec 100644 --- a/tests/cpp/integration/attach.cc +++ b/tests/cpp/integration/attach.cc @@ -167,7 +167,7 @@ void test_body( TEST(Attach, Positive) { - legate::Core::perform_registration(); + register_tasks(); // Do multiple operations in a row, with stores collected in-between, in hopes of triggering // consensus match. // TODO: Also try keeping multiple stores alive at one time. @@ -182,7 +182,7 @@ TEST(Attach, Positive) TEST(Attach, Negative) { - legate::Core::perform_registration(); + register_tasks(); auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(library_name); @@ -212,7 +212,7 @@ TEST(Attach, Negative) // and MPI. TEST(DISABLED_AttachDeathTest, MissingManualDetach) { - legate::Core::perform_registration(); + register_tasks(); auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(library_name); From 19b872d05747d7b0c71392a4871157542136a621 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Tue, 3 Oct 2023 23:12:44 -0700 Subject: [PATCH 0242/1425] Guard partitioner state dump with an environment variable (#106) * Guard partitioner state dump with an environment variable * Use consistent naming --- src/core/partitioning/detail/partitioner.cc | 4 ++-- src/core/runtime/detail/runtime.cc | 3 +++ src/core/runtime/detail/runtime.h | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/core/partitioning/detail/partitioner.cc b/src/core/partitioning/detail/partitioner.cc index 5a94ec67cf..b67a092eca 100644 --- a/src/core/partitioning/detail/partitioner.cc +++ b/src/core/partitioning/detail/partitioner.cc @@ -211,7 +211,7 @@ std::unique_ptr Partitioner::partition_stores() solver.solve_constraints(); - if (LegateDefined(LEGATE_USE_DEBUG)) { solver.dump(); } + if (Config::log_partitioning_decisions) { solver.dump(); } auto strategy = std::make_unique(); @@ -257,7 +257,7 @@ std::unique_ptr Partitioner::partition_stores() strategy->compute_launch_domains(solver); - if (LegateDefined(LEGATE_USE_DEBUG)) { strategy->dump(); } + if (Config::log_partitioning_decisions) { strategy->dump(); } return strategy; } diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index 49ae5f61bb..1dabe2e555 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -62,6 +62,8 @@ void show_progress(const Legion::Task* task, Legion::Context ctx, Legion::Runtim /*static*/ bool Config::warmup_nccl = false; +/*static*/ bool Config::log_partitioning_decisions = false; + namespace { // This is the unique string name for our library which can be used from both C++ and Python to @@ -1275,6 +1277,7 @@ void parse_config() parse_variable("LEGATE_EMPTY_TASK", Config::use_empty_task); parse_variable("LEGATE_SYNC_STREAM_VIEW", Config::synchronize_stream_view); parse_variable("LEGATE_LOG_MAPPING", Config::log_mapping_decisions); + parse_variable("LEGATE_LOG_PARTITIONING", Config::log_partitioning_decisions); parse_variable("LEGATE_WARMUP_NCCL", Config::warmup_nccl); } diff --git a/src/core/runtime/detail/runtime.h b/src/core/runtime/detail/runtime.h index da35d287fb..115b645e3d 100644 --- a/src/core/runtime/detail/runtime.h +++ b/src/core/runtime/detail/runtime.h @@ -52,6 +52,7 @@ struct Config { static bool use_empty_task; static bool synchronize_stream_view; static bool log_mapping_decisions; + static bool log_partitioning_decisions; static bool has_socket_mem; static bool warmup_nccl; }; From 8a51088c73ccdfe6086565448695bdce19e0cd73 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 4 Oct 2023 00:20:58 -0700 Subject: [PATCH 0243/1425] Support for tunable parameters (#108) * Support for tunable parameters * Minor fix to the attach test * More fix for the attach test --- src/core/runtime/detail/library.cc | 15 +++++ src/core/runtime/detail/library.h | 2 + src/core/runtime/library.cc | 5 ++ src/core/runtime/library.h | 10 ++++ tests/cpp/integration/attach.cc | 23 +++---- tests/cpp/integration/tunable.cc | 96 ++++++++++++++++++++++++++++++ 6 files changed, 140 insertions(+), 11 deletions(-) create mode 100644 tests/cpp/integration/tunable.cc diff --git a/src/core/runtime/detail/library.cc b/src/core/runtime/detail/library.cc index 1e64b0d7d3..2eadb31952 100644 --- a/src/core/runtime/detail/library.cc +++ b/src/core/runtime/detail/library.cc @@ -119,6 +119,21 @@ const std::string& Library::get_task_name(int64_t local_task_id) const return find_task(local_task_id)->name(); } +std::unique_ptr Library::get_tunable(int64_t tunable_id, std::shared_ptr type) +{ + if (type->variable_size()) { + throw std::invalid_argument("Tunable variables must have fixed-size types"); + } + auto result = Runtime::get_runtime()->get_tunable(mapper_id_, tunable_id, type->size()); + size_t extents = 0; + const void* buffer = result.get_buffer(Memory::Kind::SYSTEM_MEM, &extents); + if (extents != type->size()) { + throw std::invalid_argument("Size mismatch: expected " + std::to_string(type->size()) + + " bytes but got " + std::to_string(extents) + " bytes"); + } + return std::make_unique(std::move(type), buffer, true); +} + void register_mapper_callback(const Legion::RegistrationCallbackArgs& args) { const std::string library_name(static_cast(args.buffer.get_ptr())); diff --git a/src/core/runtime/detail/library.h b/src/core/runtime/detail/library.h index 71f22d4a45..74a93c91bd 100644 --- a/src/core/runtime/detail/library.h +++ b/src/core/runtime/detail/library.h @@ -15,6 +15,7 @@ #include #include +#include "core/data/detail/scalar.h" #include "core/mapping/mapping.h" #include "core/runtime/resource.h" #include "core/task/task_info.h" @@ -95,6 +96,7 @@ class Library { public: const std::string& get_task_name(int64_t local_task_id) const; + std::unique_ptr get_tunable(int64_t tunable_id, std::shared_ptr type); void register_mapper(std::unique_ptr mapper, bool in_callback); Legion::Mapping::Mapper* get_legion_mapper() const { return legion_mapper_; } diff --git a/src/core/runtime/library.cc b/src/core/runtime/library.cc index 9cfbe1b55b..787bb2aea9 100644 --- a/src/core/runtime/library.cc +++ b/src/core/runtime/library.cc @@ -85,6 +85,11 @@ const std::string& Library::get_task_name(int64_t local_task_id) const return impl_->get_task_name(local_task_id); } +Scalar Library::get_tunable(int64_t tunable_id, Type type) +{ + return Scalar{impl_->get_tunable(tunable_id, type.impl())}; +} + void Library::register_mapper(std::unique_ptr mapper) { impl_->register_mapper(std::move(mapper), false /*in_callback*/); diff --git a/src/core/runtime/library.h b/src/core/runtime/library.h index 41e9dddafa..1ce3c6cf58 100644 --- a/src/core/runtime/library.h +++ b/src/core/runtime/library.h @@ -15,6 +15,7 @@ #include #include +#include "core/data/scalar.h" #include "core/runtime/resource.h" #include "core/task/task_info.h" #include "core/utilities/typedefs.h" @@ -79,6 +80,15 @@ class Library { * @return Name of the task */ const std::string& get_task_name(int64_t local_task_id) const; + /** + * @brief Retrieves a tunable parameter + * + * @param tunable_id ID of the tunable parameter + * @param type Type of the tunable value + * + * @return The value of tunable parameter in a `Scalar` + */ + Scalar get_tunable(int64_t tunable_id, Type type); /** * @brief Registers a library specific reduction operator. * diff --git a/tests/cpp/integration/attach.cc b/tests/cpp/integration/attach.cc index e82c98c7ec..d902eab170 100644 --- a/tests/cpp/integration/attach.cc +++ b/tests/cpp/integration/attach.cc @@ -46,13 +46,13 @@ void check_physical_store(const legate::Store& store, int32_t dim, int64_t count if (dim == 1) { auto shape = store.shape<1>(); auto acc = store.read_accessor(shape); - for (int32_t i = 0; i < SHAPE_1D[0]; ++i) EXPECT_EQ(acc[i], counter++); + for (size_t i = 0; i < SHAPE_1D[0]; ++i) EXPECT_EQ(acc[i], counter++); } else { auto shape = store.shape<2>(); auto acc = store.read_accessor(shape); // Legate should always see elements in the expected order - for (int32_t i = 0; i < SHAPE_2D[0]; ++i) - for (int32_t j = 0; j < SHAPE_2D[1]; ++j) EXPECT_EQ(acc[legate::Point<2>(i, j)], counter++); + for (size_t i = 0; i < SHAPE_2D[0]; ++i) + for (size_t j = 0; j < SHAPE_2D[1]; ++j) EXPECT_EQ(acc[legate::Point<2>(i, j)], counter++); } } @@ -79,6 +79,9 @@ struct CheckerTask : public legate::LegateTask { void register_tasks() { + static bool prepared = false; + if (prepared) { return; } + prepared = true; auto runtime = legate::Runtime::get_runtime(); auto context = runtime->create_library(library_name); AdderTask::register_variants(context); @@ -91,11 +94,11 @@ int64_t* make_buffer(int32_t dim, bool fortran) int64_t counter = 0; if (dim == 1) { buffer = new int64_t[SHAPE_1D.volume()]; - for (int32_t i = 0; i < SHAPE_1D[0]; ++i) buffer[i] = counter++; + for (size_t i = 0; i < SHAPE_1D[0]; ++i) buffer[i] = counter++; } else { buffer = new int64_t[SHAPE_2D.volume()]; - for (int32_t i = 0; i < SHAPE_2D[0]; ++i) - for (int32_t j = 0; j < SHAPE_2D[1]; ++j) + for (size_t i = 0; i < SHAPE_2D[0]; ++i) + for (size_t j = 0; j < SHAPE_2D[1]; ++j) if (fortran) buffer[j * SHAPE_2D[0] + i] = counter++; else @@ -107,10 +110,10 @@ int64_t* make_buffer(int32_t dim, bool fortran) void check_and_delete_buffer(int64_t* buffer, int32_t dim, bool fortran, int64_t counter) { if (dim == 1) - for (int32_t i = 0; i < SHAPE_1D[0]; ++i) EXPECT_EQ(buffer[i], counter++); + for (size_t i = 0; i < SHAPE_1D[0]; ++i) EXPECT_EQ(buffer[i], counter++); else - for (int32_t i = 0; i < SHAPE_2D[0]; ++i) - for (int32_t j = 0; j < SHAPE_2D[1]; ++j) + for (size_t i = 0; i < SHAPE_2D[0]; ++i) + for (size_t j = 0; j < SHAPE_2D[1]; ++j) if (fortran) EXPECT_EQ(buffer[j * SHAPE_2D[0] + i], counter++); else @@ -184,7 +187,6 @@ TEST(Attach, Negative) { register_tasks(); auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library(library_name); // Trying to detach a store without an attachment EXPECT_THROW(runtime->create_store(legate::Scalar(42)).detach(), std::invalid_argument); @@ -214,7 +216,6 @@ TEST(DISABLED_AttachDeathTest, MissingManualDetach) { register_tasks(); auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library(library_name); // Forgetting to detach a shared attachment EXPECT_DEATH(runtime->create_store( diff --git a/tests/cpp/integration/tunable.cc b/tests/cpp/integration/tunable.cc new file mode 100644 index 0000000000..7ff726d469 --- /dev/null +++ b/tests/cpp/integration/tunable.cc @@ -0,0 +1,96 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include + +#include "legate.h" + +namespace tunable { + +static const char* library_name = "test_tunable"; + +const std::vector TUNABLES = { + legate::Scalar{false}, + legate::Scalar{int8_t{12}}, + legate::Scalar{int32_t{456}}, + legate::Scalar{uint16_t{78}}, + legate::Scalar{uint64_t{91011}}, + legate::Scalar{double{123.0}}, + legate::Scalar{complex{10.0F, 20.0F}}, +}; + +class LibraryMapper : public legate::mapping::Mapper { + void set_machine(const legate::mapping::MachineQueryInterface* machine) override {} + legate::mapping::TaskTarget task_target( + const legate::mapping::Task& task, + const std::vector& options) override + { + return options.front(); + } + std::vector store_mappings( + const legate::mapping::Task& task, + const std::vector& options) override + { + return {}; + } + legate::Scalar tunable_value(legate::TunableID tunable_id) override + { + return (tunable_id >= 0 && tunable_id < TUNABLES.size()) ? TUNABLES[tunable_id] + : legate::Scalar{}; + } +}; + +void prepare() +{ + static bool prepared = false; + if (prepared) { return; } + prepared = true; + auto runtime = legate::Runtime::get_runtime(); + runtime->create_library( + library_name, legate::ResourceConfig{}, std::make_unique()); +} + +struct scalar_eq_fn { + template + bool operator()(const legate::Scalar& lhs, const legate::Scalar& rhs) + { + using VAL = legate::legate_type_of; + return lhs.value() == rhs.value(); + } +}; + +TEST(Tunable, Valid) +{ + prepare(); + auto runtime = legate::Runtime::get_runtime(); + auto library = runtime->find_library(library_name); + + int64_t tunable_id = 0; + for (const auto& to_compare : TUNABLES) { + auto dtype = to_compare.type(); + EXPECT_TRUE(legate::type_dispatch( + dtype.code(), scalar_eq_fn{}, library.get_tunable(tunable_id++, dtype), to_compare)); + } +} + +TEST(Tunable, Invalid) +{ + prepare(); + auto runtime = legate::Runtime::get_runtime(); + auto library = runtime->find_library(library_name); + + EXPECT_THROW(library.get_tunable(0, legate::string_type()), std::invalid_argument); + EXPECT_THROW(library.get_tunable(0, legate::int64()), std::invalid_argument); + EXPECT_THROW(library.get_tunable(TUNABLES.size(), legate::bool_()), std::invalid_argument); +} + +} // namespace tunable From 2a317a192949458ae2e57b12d9321d5a17d880a3 Mon Sep 17 00:00:00 2001 From: Vladislav Zhurba <53052066+vzhurba01@users.noreply.github.com> Date: Wed, 4 Oct 2023 09:53:52 -0700 Subject: [PATCH 0244/1425] Add missing argv[0] to Config call (#110) Legate Tester assumes the first arg is the file name and trims it off. --- tests/cpp/run.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/cpp/run.py b/tests/cpp/run.py index d97ea8d316..b74a243e4d 100755 --- a/tests/cpp/run.py +++ b/tests/cpp/run.py @@ -62,15 +62,15 @@ def main(): default=0, help="Runs mpirun with rank if non-zero", ) - config, extra_args = parser.parse_known_args() + args, extra_args = parser.parse_known_args() - if config.mpi_rank != 0: - extra_args += ["--mpi-rank", str(config.mpi_rank)] + if args.mpi_rank != 0: + extra_args += ["--mpi-rank", str(args.mpi_rank)] extra_args += ["--mpi-output-filename", "build/mpi_result"] - extra_args += ["--gtest-file", config.gtest_file] - extra_args += ["--gtest-tests"] + fetch_test_names(config.gtest_file) + extra_args += ["--gtest-file", args.gtest_file] + extra_args += ["--gtest-tests"] + fetch_test_names(args.gtest_file) - config = Config(extra_args) + config = Config([sys.argv[0]] + extra_args) system = TestSystem(dry_run=config.dry_run) From 689066e01de513fa13bab1bcb7b5d6131567f5ec Mon Sep 17 00:00:00 2001 From: Vladislav Zhurba <53052066+vzhurba01@users.noreply.github.com> Date: Wed, 4 Oct 2023 09:57:06 -0700 Subject: [PATCH 0245/1425] Use TEST_F instead of global env (#103) --- tests/cpp/CMakeLists.txt | 1 + .../cpp/integration/alignment_constraints.cc | 13 +++--- tests/cpp/integration/attach.cc | 31 ++++++++----- tests/cpp/integration/bloat_constraints.cc | 11 +++-- .../cpp/integration/broadcast_constraints.cc | 9 ++-- tests/cpp/integration/consensus_match.cc | 5 ++- tests/cpp/integration/copy_failure.cc | 13 +++--- tests/cpp/integration/copy_gather.cc | 13 +++--- tests/cpp/integration/copy_gather_scatter.cc | 13 +++--- tests/cpp/integration/copy_normal.cc | 7 ++- tests/cpp/integration/copy_scatter.cc | 11 +++-- tests/cpp/integration/cpu_communicator.cc | 5 ++- tests/cpp/integration/exception.cc | 9 ++-- tests/cpp/integration/field_reuse.cc | 5 ++- tests/cpp/integration/fill.cc | 11 +++-- tests/cpp/integration/image_constraints.cc | 17 ++++--- tests/cpp/integration/inline_map.cc | 9 ++-- tests/cpp/integration/inout.cc | 5 ++- tests/cpp/integration/machine_scope.cc | 5 ++- tests/cpp/integration/manual_simple.cc | 5 ++- tests/cpp/integration/multi_scalar_out.cc | 5 ++- tests/cpp/integration/nccl.cu | 5 ++- tests/cpp/integration/provenance.cc | 5 ++- tests/cpp/integration/region_manager.cc | 5 ++- tests/cpp/integration/req_analyzer.cc | 7 ++- tests/cpp/integration/scale_constraints.cc | 11 +++-- tests/cpp/integration/tree_reduce.cc | 15 ++++--- tests/cpp/integration/tree_reduce_unique.cc | 5 ++- tests/cpp/integration/tunable.cc | 7 ++- tests/cpp/integration/weighted.cc | 7 ++- tests/cpp/main.cc | 16 ++----- tests/cpp/unit/buffer.cc | 7 ++- tests/cpp/unit/constraint.cc | 15 +++++-- tests/cpp/unit/death_example.cc | 35 +++++++++++++++ tests/cpp/unit/library.cc | 9 ++-- tests/cpp/unit/logical_array_creation.cc | 19 ++++---- tests/cpp/unit/logical_store.cc | 9 ++-- tests/cpp/unit/machine.cc | 7 ++- tests/cpp/unit/mapping.cc | 7 ++- tests/cpp/unit/registration.cc | 7 ++- tests/cpp/unit/scalar.cc | 25 ++++++----- tests/cpp/unit/scoped_allocator.cc | 11 +++-- tests/cpp/unit/span.cc | 7 ++- tests/cpp/unit/type.cc | 30 +++++++------ tests/cpp/utilities/utilities.h | 44 +++++++++++++++++++ 45 files changed, 361 insertions(+), 157 deletions(-) create mode 100644 tests/cpp/unit/death_example.cc create mode 100644 tests/cpp/utilities/utilities.h diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt index 4a30b3666f..d6918c2008 100644 --- a/tests/cpp/CMakeLists.txt +++ b/tests/cpp/CMakeLists.txt @@ -67,6 +67,7 @@ endif() add_executable(cpp_tests ${main_SRC} ${tasks_SRC} ${integration_SRC} ${unit_SRC}) +target_include_directories(cpp_tests PRIVATE ${PROJECT_SOURCE_DIR}) target_link_libraries(cpp_tests PRIVATE legate::core PRIVATE GTest::gtest) if(Legion_USE_CUDA) target_link_libraries(cpp_tests PRIVATE NCCL::NCCL) diff --git a/tests/cpp/integration/alignment_constraints.cc b/tests/cpp/integration/alignment_constraints.cc index a1e003639e..f5af7c2354 100644 --- a/tests/cpp/integration/alignment_constraints.cc +++ b/tests/cpp/integration/alignment_constraints.cc @@ -13,9 +13,12 @@ #include #include "legate.h" +#include "utilities/utilities.h" namespace alignment_constraints { +using Alignment = DefaultFixture; + static const char* library_name = "test_alignment_constraints"; enum TaskIDs { @@ -229,31 +232,31 @@ void test_invalid_alignment() EXPECT_THROW(runtime->submit(std::move(task)), std::invalid_argument); } -TEST(Alignment, Basic) +TEST_F(Alignment, Basic) { prepare(); test_alignment(); } -TEST(Alignment, WithBroadcast) +TEST_F(Alignment, WithBroadcast) { prepare(); test_alignment_and_broadcast(); } -TEST(Alignment, WithTransform) +TEST_F(Alignment, WithTransform) { prepare(); test_alignment_transformed(); } -TEST(Alignment, Redundant) +TEST_F(Alignment, Redundant) { prepare(); test_redundant_alignment(); } -TEST(Alignment, Invalid) +TEST_F(Alignment, Invalid) { prepare(); test_invalid_alignment(); diff --git a/tests/cpp/integration/attach.cc b/tests/cpp/integration/attach.cc index d902eab170..99ee1b4348 100644 --- a/tests/cpp/integration/attach.cc +++ b/tests/cpp/integration/attach.cc @@ -14,9 +14,13 @@ #include "core/data/detail/logical_store.h" #include "legate.h" +#include "utilities/utilities.h" namespace attach { +using Attach = DefaultFixture; +using AttachDeathTest = DeathTestFixture; + static const char* library_name = "test_attach"; enum TaskOpCode { @@ -168,7 +172,7 @@ void test_body( } } -TEST(Attach, Positive) +TEST_F(Attach, Positive) { register_tasks(); // Do multiple operations in a row, with stores collected in-between, in hopes of triggering @@ -183,7 +187,7 @@ TEST(Attach, Positive) test_body(dim, fortran, unordered, share, use_tasks, use_inline); } -TEST(Attach, Negative) +TEST_F(Attach, Negative) { register_tasks(); auto runtime = legate::Runtime::get_runtime(); @@ -210,17 +214,22 @@ TEST(Attach, Negative) l_store.detach(); } -// TODO: GTest implements death tests by forking, and that is not going to work well with Realm -// and MPI. -TEST(DISABLED_AttachDeathTest, MissingManualDetach) +TEST_F(AttachDeathTest, MissingManualDetach) { - register_tasks(); - auto runtime = legate::Runtime::get_runtime(); - // Forgetting to detach a shared attachment - EXPECT_DEATH(runtime->create_store( - SHAPE_1D, legate::int64(), new int64_t[SHAPE_1D.volume()], true /*share*/), - "must be manually detached"); + // TODO: We can't check the actual error message, because it's reported using logging, and even + // though that's mirrored on stderr, it doesn't seem to be included in the death message. + EXPECT_DEATH( + { + legate::start(argc_, argv_); + register_tasks(); + auto runtime = legate::Runtime::get_runtime(); + auto context = runtime->find_library(library_name); + runtime->create_store( + SHAPE_1D, legate::int64(), new int64_t[SHAPE_1D.volume()], true /*share*/); + legate::finish(); + }, + ""); } } // namespace attach diff --git a/tests/cpp/integration/bloat_constraints.cc b/tests/cpp/integration/bloat_constraints.cc index 8b1bb1e79a..4490315c86 100644 --- a/tests/cpp/integration/bloat_constraints.cc +++ b/tests/cpp/integration/bloat_constraints.cc @@ -13,9 +13,12 @@ #include #include "legate.h" +#include "utilities/utilities.h" namespace bloat_constraints { +using BloatConstraint = DefaultFixture; + static const char* library_name = "test_bloat_constraints"; static legate::Logger logger(library_name); @@ -134,25 +137,25 @@ void test_invalid() } } -TEST(BloatConstraint, 1D) +TEST_F(BloatConstraint, 1D) { prepare(); test_bloat({{10}, {2}, {4}}); } -TEST(BloatConstraint, 2D) +TEST_F(BloatConstraint, 2D) { prepare(); test_bloat({{9, 9}, {2, 3}, {3, 4}}); } -TEST(BloatConstraint, 3D) +TEST_F(BloatConstraint, 3D) { prepare(); test_bloat({{10, 10, 10}, {2, 3, 4}, {4, 3, 2}}); } -TEST(BloatConstraint, Invalid) +TEST_F(BloatConstraint, Invalid) { prepare(); test_invalid(); diff --git a/tests/cpp/integration/broadcast_constraints.cc b/tests/cpp/integration/broadcast_constraints.cc index cb15b65ae7..9964e37d34 100644 --- a/tests/cpp/integration/broadcast_constraints.cc +++ b/tests/cpp/integration/broadcast_constraints.cc @@ -13,9 +13,12 @@ #include #include "legate.h" +#include "utilities/utilities.h" namespace broadcast_constraints { +using Broadcast = DefaultFixture; + static const char* library_name = "test_broadcast_constraints"; constexpr size_t EXT_SMALL = 10; @@ -123,19 +126,19 @@ void test_invalid_broadcast() EXPECT_THROW(runtime->submit(std::move(task)), std::invalid_argument); } -TEST(Broadcast, Basic) +TEST_F(Broadcast, Basic) { prepare(); test_normal_store(); } -TEST(Broadcast, WithPromotion) +TEST_F(Broadcast, WithPromotion) { prepare(); test_promoted_store(); } -TEST(Broadcast, Invalid) +TEST_F(Broadcast, Invalid) { prepare(); test_invalid_broadcast(); diff --git a/tests/cpp/integration/consensus_match.cc b/tests/cpp/integration/consensus_match.cc index 87cb4d65b6..9a164e0baf 100644 --- a/tests/cpp/integration/consensus_match.cc +++ b/tests/cpp/integration/consensus_match.cc @@ -18,9 +18,12 @@ #include "core/runtime/detail/runtime.h" #include "legate.h" +#include "utilities/utilities.h" namespace consensus_match { +using Integration = DefaultFixture; + static const char* library_name = "consensus_match"; void register_tasks() @@ -36,7 +39,7 @@ struct Thing { bool operator==(const Thing& other) const { return flag == other.flag && number == other.number; } }; -TEST(Integration, ConsensusMatch) +TEST_F(Integration, ConsensusMatch) { auto runtime = legate::Runtime::get_runtime(); register_tasks(); diff --git a/tests/cpp/integration/copy_failure.cc b/tests/cpp/integration/copy_failure.cc index 0bdc6392a9..981fbc983c 100644 --- a/tests/cpp/integration/copy_failure.cc +++ b/tests/cpp/integration/copy_failure.cc @@ -13,9 +13,12 @@ #include #include "legate.h" +#include "utilities/utilities.h" namespace copy_failure { +using Copy = DefaultFixture; + void test_invalid_stores() { auto runtime = legate::Runtime::get_runtime(); @@ -117,14 +120,14 @@ void test_dimension_mismatch_failure() std::invalid_argument); } -TEST(Copy, FailureInvalidStores) { test_invalid_stores(); } +TEST_F(Copy, FailureInvalidStores) { test_invalid_stores(); } -TEST(Copy, FailureDifferentTypes) { test_type_check_failure(); } +TEST_F(Copy, FailureDifferentTypes) { test_type_check_failure(); } -TEST(Copy, FailureDifferentShapes) { test_shape_check_failure(); } +TEST_F(Copy, FailureDifferentShapes) { test_shape_check_failure(); } -TEST(Copy, FailureNonPointTypes) { test_non_point_types_failure(); } +TEST_F(Copy, FailureNonPointTypes) { test_non_point_types_failure(); } -TEST(Copy, FailureDimensionMismatch) { test_dimension_mismatch_failure(); } +TEST_F(Copy, FailureDimensionMismatch) { test_dimension_mismatch_failure(); } } // namespace copy_failure diff --git a/tests/cpp/integration/copy_gather.cc b/tests/cpp/integration/copy_gather.cc index 1e38a5573b..9fd7e9977b 100644 --- a/tests/cpp/integration/copy_gather.cc +++ b/tests/cpp/integration/copy_gather.cc @@ -14,9 +14,12 @@ #include "copy_util.inl" #include "legate.h" +#include "utilities/utilities.h" namespace copy_gather { +using Copy = DefaultFixture; + static const char* library_name = "test_copy_gather"; static legate::Logger logger(library_name); @@ -148,33 +151,33 @@ void test_gather(const GatherSpec& spec) check_gather_output(library, src, tgt, ind); } -TEST(Copy, Gather2Dto1D) +TEST_F(Copy, Gather2Dto1D) { register_tasks(); std::vector shape1d{5}; test_gather(GatherSpec{shape1d, {7, 11}, legate::Scalar(int64_t(123))}); } -TEST(Copy, Gather3Dto2D) +TEST_F(Copy, Gather3Dto2D) { register_tasks(); test_gather(GatherSpec{{3, 7}, {3, 2, 5}, legate::Scalar(uint32_t(456))}); } -TEST(Copy, Gather1Dto3D) +TEST_F(Copy, Gather1Dto3D) { register_tasks(); std::vector shape1d{5}; test_gather(GatherSpec{{2, 5, 4}, shape1d, legate::Scalar(789.0)}); } -TEST(Copy, Gather2Dto2D) +TEST_F(Copy, Gather2Dto2D) { register_tasks(); test_gather(GatherSpec{{4, 5}, {10, 11}, legate::Scalar(int64_t(12))}); } -TEST(Copy, Gather2Dto3D) +TEST_F(Copy, Gather2Dto3D) { register_tasks(); test_gather(GatherSpec{{100, 100, 100}, {10, 10}, legate::Scalar(7.0)}); diff --git a/tests/cpp/integration/copy_gather_scatter.cc b/tests/cpp/integration/copy_gather_scatter.cc index f79b9672a6..993df7e13f 100644 --- a/tests/cpp/integration/copy_gather_scatter.cc +++ b/tests/cpp/integration/copy_gather_scatter.cc @@ -14,9 +14,12 @@ #include "copy_util.inl" #include "legate.h" +#include "utilities/utilities.h" namespace copy_gather_scatter { +using Copy = DefaultFixture; + static const char* library_name = "test_copy_gather_scatter"; static legate::Logger logger(library_name); @@ -178,7 +181,7 @@ void test_gather_scatter(const GatherScatterSpec& spec) check_gather_scatter_output(library, src, tgt, src_ind, tgt_ind, spec.init); } -TEST(Copy, GatherScatter1Dto3Dvia2D) +TEST_F(Copy, GatherScatter1Dto3Dvia2D) { register_tasks(); std::vector shape1d{5}; @@ -186,7 +189,7 @@ TEST(Copy, GatherScatter1Dto3Dvia2D) shape1d, {7, 11}, {10, 10, 10}, legate::Scalar(int64_t(123)), legate::Scalar(int64_t(42))}); } -TEST(Copy, GatherScatter2Dto1Dvia3D) +TEST_F(Copy, GatherScatter2Dto1Dvia3D) { register_tasks(); std::vector shape1d{1000}; @@ -194,7 +197,7 @@ TEST(Copy, GatherScatter2Dto1Dvia3D) {3, 7}, {3, 6, 5}, shape1d, legate::Scalar(uint32_t(456)), legate::Scalar(uint32_t(42))}); } -TEST(Copy, GatherScatter3Dto2Dvia1D) +TEST_F(Copy, GatherScatter3Dto2Dvia1D) { register_tasks(); std::vector shape1d{100}; @@ -202,7 +205,7 @@ TEST(Copy, GatherScatter3Dto2Dvia1D) {4, 5, 2}, shape1d, {50, 50}, legate::Scalar(int64_t(12)), legate::Scalar(int64_t(42))}); } -TEST(Copy, GatherScatter3Dto3Dvia3D) +TEST_F(Copy, GatherScatter3Dto3Dvia3D) { register_tasks(); test_gather_scatter(GatherScatterSpec{{10, 10, 10}, @@ -212,7 +215,7 @@ TEST(Copy, GatherScatter3Dto3Dvia3D) legate::Scalar(int64_t(42))}); } -TEST(Copy, GatherScatter2Dto3Dvia2D) +TEST_F(Copy, GatherScatter2Dto3Dvia2D) { register_tasks(); test_gather_scatter(GatherScatterSpec{ diff --git a/tests/cpp/integration/copy_normal.cc b/tests/cpp/integration/copy_normal.cc index fcf2a8e9d7..5fd63b8b52 100644 --- a/tests/cpp/integration/copy_normal.cc +++ b/tests/cpp/integration/copy_normal.cc @@ -14,6 +14,7 @@ #include "core/type/type_info.h" #include "legate.h" +#include "utilities/utilities.h" #include "copy_util.inl" @@ -27,6 +28,8 @@ extern void silence_unused_function_warnings() namespace copy_normal { +using Copy = DefaultFixture; + static const char* library_name = "test_copy_normal"; static constexpr int32_t TEST_MAX_DIM = 3; @@ -209,7 +212,7 @@ void test_normal_copy_reduction(const NormalCopyReductionSpec& spec) check_copy_reduction_output(library, input, output, seed); } -TEST(Copy, Single) +TEST_F(Copy, Single) { register_tasks(); test_normal_copy({{4, 7}, legate::int64(), legate::Scalar(int64_t(12))}); @@ -217,7 +220,7 @@ TEST(Copy, Single) test_normal_copy({{1}, legate::int64(), legate::Scalar(int64_t(12))}); } -TEST(Copy, SingleReduction) +TEST_F(Copy, SingleReduction) { register_tasks(); test_normal_copy_reduction( diff --git a/tests/cpp/integration/copy_scatter.cc b/tests/cpp/integration/copy_scatter.cc index 83fc9ea61e..63a006cf9a 100644 --- a/tests/cpp/integration/copy_scatter.cc +++ b/tests/cpp/integration/copy_scatter.cc @@ -14,9 +14,12 @@ #include "copy_util.inl" #include "legate.h" +#include "utilities/utilities.h" namespace copy_scatter { +using Copy = DefaultFixture; + static const char* library_name = "test_copy_scatter"; static legate::Logger logger(library_name); @@ -172,7 +175,7 @@ void test_scatter(const ScatterSpec& spec) // Note that the volume of indirection field should be smaller than that of the target to avoid // duplicate updates on the same element, whose semantics is undefined. -TEST(Copy, Scatter1Dto2D) +TEST_F(Copy, Scatter1Dto2D) { register_tasks(); std::vector shape1d{5}; @@ -180,21 +183,21 @@ TEST(Copy, Scatter1Dto2D) ScatterSpec{shape1d, {7, 11}, legate::Scalar(int64_t(123)), legate::Scalar(int64_t(42))}); } -TEST(Copy, Scatter2Dto3D) +TEST_F(Copy, Scatter2Dto3D) { register_tasks(); test_scatter( ScatterSpec{{3, 7}, {3, 6, 5}, legate::Scalar(uint32_t(456)), legate::Scalar(uint32_t(42))}); } -TEST(Copy, Scatter2Dto2D) +TEST_F(Copy, Scatter2Dto2D) { register_tasks(); test_scatter( ScatterSpec{{4, 5}, {10, 11}, legate::Scalar(int64_t(12)), legate::Scalar(int64_t(42))}); } -TEST(Copy, Scatter3Dto2D) +TEST_F(Copy, Scatter3Dto2D) { register_tasks(); test_scatter( diff --git a/tests/cpp/integration/cpu_communicator.cc b/tests/cpp/integration/cpu_communicator.cc index bdf3e9d976..c2887929cf 100644 --- a/tests/cpp/integration/cpu_communicator.cc +++ b/tests/cpp/integration/cpu_communicator.cc @@ -14,9 +14,12 @@ #include "core/comm/coll.h" #include "legate.h" +#include "utilities/utilities.h" namespace cpu_communicator { +using Integration = DefaultFixture; + const char* library_name = "test_cpu_communicator"; enum TaskIDs { @@ -88,7 +91,7 @@ void test_cpu_communicator_manual(int32_t ndim) } // Test case with single unbound store -TEST(Integration, CPUCommunicator) +TEST_F(Integration, CPUCommunicator) { prepare(); diff --git a/tests/cpp/integration/exception.cc b/tests/cpp/integration/exception.cc index 68dc2c405d..7816f93ad3 100644 --- a/tests/cpp/integration/exception.cc +++ b/tests/cpp/integration/exception.cc @@ -13,9 +13,12 @@ #include #include "legate.h" +#include "utilities/utilities.h" namespace exception { +using Exception = DefaultFixture; + const char* library_name = "test_exception"; static legate::Logger logger(library_name); @@ -177,14 +180,14 @@ void test_pending() // Finish the test with a pending exception to check if the runtime cleans things up correctly } -TEST(Exception, Single) +TEST_F(Exception, Single) { prepare(); test_single(); } -TEST(Exception, Multi) +TEST_F(Exception, Multi) { prepare(); @@ -192,7 +195,7 @@ TEST(Exception, Multi) test_multi(false /* use_auto_task */); } -TEST(Exception, Pending) +TEST_F(Exception, Pending) { prepare(); diff --git a/tests/cpp/integration/field_reuse.cc b/tests/cpp/integration/field_reuse.cc index 150c453289..d816de6336 100644 --- a/tests/cpp/integration/field_reuse.cc +++ b/tests/cpp/integration/field_reuse.cc @@ -19,9 +19,12 @@ #include "core/data/detail/logical_store.h" #include "core/runtime/detail/runtime.h" #include "legate.h" +#include "utilities/utilities.h" namespace field_reuse { +using Integration = DefaultFixture; + static const char* library_name = "field_reuse"; void register_tasks() @@ -39,7 +42,7 @@ void check_field_is_new(Legion::FieldID fid) EXPECT_EQ(unique_fields.size(), prev_size + 1); } -TEST(Integration, FieldReuse) +TEST_F(Integration, FieldReuse) { // TODO: Also test the reuse of a field originally returned by an unbounded-output task. diff --git a/tests/cpp/integration/fill.cc b/tests/cpp/integration/fill.cc index 0fc3f8c972..0f8a5314b8 100644 --- a/tests/cpp/integration/fill.cc +++ b/tests/cpp/integration/fill.cc @@ -13,9 +13,12 @@ #include #include "legate.h" +#include "utilities/utilities.h" namespace fill { +using Fill = DefaultFixture; + static const char* library_name = "test_fill"; constexpr size_t SIZE = 10; @@ -180,7 +183,7 @@ void test_invalid() EXPECT_THROW(runtime->issue_fill(store1, scalar_v), std::invalid_argument); } -TEST(Fill, Index) +TEST_F(Fill, Index) { register_tasks(); test_fill_index(1, SIZE); @@ -191,7 +194,7 @@ TEST(Fill, Index) test_fill_index(3, 1); } -TEST(Fill, Single) +TEST_F(Fill, Single) { register_tasks(); test_fill_single(1, SIZE); @@ -202,7 +205,7 @@ TEST(Fill, Single) test_fill_single(3, 1); } -TEST(Fill, Slice) +TEST_F(Fill, Slice) { register_tasks(); test_fill_slice(1, SIZE); @@ -210,6 +213,6 @@ TEST(Fill, Slice) test_fill_slice(3, SIZE); } -TEST(Fill, Invalid) { test_invalid(); } +TEST_F(Fill, Invalid) { test_invalid(); } } // namespace fill diff --git a/tests/cpp/integration/image_constraints.cc b/tests/cpp/integration/image_constraints.cc index 15b0631127..52b1f9f4bd 100644 --- a/tests/cpp/integration/image_constraints.cc +++ b/tests/cpp/integration/image_constraints.cc @@ -14,9 +14,12 @@ #include "core/data/detail/logical_store.h" #include "legate.h" +#include "utilities/utilities.h" namespace image_constraints { +using ImageConstraint = DefaultFixture; + static const char* library_name = "test_image_constraints"; static const int32_t TEST_MAX_DIM = 3; @@ -277,43 +280,43 @@ void test_invalid() EXPECT_THROW(runtime->submit(std::move(task)), std::invalid_argument); } -TEST(ImageConstraint, Point1D) +TEST_F(ImageConstraint, Point1D) { prepare(); test_image({{9}, {100}, false}); } -TEST(ImageConstraint, Point2D) +TEST_F(ImageConstraint, Point2D) { prepare(); test_image({{4, 4}, {10, 10}, false}); } -TEST(ImageConstraint, Point3D) +TEST_F(ImageConstraint, Point3D) { prepare(); test_image({{2, 3, 4}, {5, 5, 5}, false}); } -TEST(ImageConstraint, Rect1D) +TEST_F(ImageConstraint, Rect1D) { prepare(); test_image({{9}, {100}, true}); } -TEST(ImageConstraint, Rect2D) +TEST_F(ImageConstraint, Rect2D) { prepare(); test_image({{4, 4}, {10, 10}, true}); } -TEST(ImageConstraint, Rect3D) +TEST_F(ImageConstraint, Rect3D) { prepare(); test_image({{2, 3, 4}, {5, 5, 5}, true}); } -TEST(ImageConstraint, Invalid) +TEST_F(ImageConstraint, Invalid) { prepare(); test_invalid(); diff --git a/tests/cpp/integration/inline_map.cc b/tests/cpp/integration/inline_map.cc index 126184494e..67d5b09909 100644 --- a/tests/cpp/integration/inline_map.cc +++ b/tests/cpp/integration/inline_map.cc @@ -14,9 +14,12 @@ #include "core/runtime/detail/runtime.h" #include "legate.h" +#include "utilities/utilities.h" namespace inline_map { +using InlineMap = DefaultFixture; + static const char* library_name = "test_inline_map"; enum TaskOpCode { @@ -83,11 +86,11 @@ void test_inline_map_and_task() EXPECT_EQ(acc[2], 43); } -TEST(InlineMap, Future) { test_inline_map_future(); } +TEST_F(InlineMap, Future) { test_inline_map_future(); } -TEST(InlineMap, RegionAndSlice) { test_inline_map_region_and_slice(); } +TEST_F(InlineMap, RegionAndSlice) { test_inline_map_region_and_slice(); } -TEST(InlineMap, WithTask) +TEST_F(InlineMap, WithTask) { register_tasks(); test_inline_map_and_task(); diff --git a/tests/cpp/integration/inout.cc b/tests/cpp/integration/inout.cc index e80627fc23..11ad3a4c9e 100644 --- a/tests/cpp/integration/inout.cc +++ b/tests/cpp/integration/inout.cc @@ -14,9 +14,12 @@ #include "legate.h" #include "tasks/task_simple.h" +#include "utilities/utilities.h" namespace inout { +using Integration = DefaultFixture; + void test_inout() { auto runtime = legate::Runtime::get_runtime(); @@ -31,7 +34,7 @@ void test_inout() runtime->submit(std::move(task)); } -TEST(Integration, InOut) +TEST_F(Integration, InOut) { task::simple::register_tasks(); test_inout(); diff --git a/tests/cpp/integration/machine_scope.cc b/tests/cpp/integration/machine_scope.cc index 07619c9ab5..dd8abc5672 100644 --- a/tests/cpp/integration/machine_scope.cc +++ b/tests/cpp/integration/machine_scope.cc @@ -13,9 +13,12 @@ #include #include "legate.h" +#include "utilities/utilities.h" namespace machine_scope { +using Integration = DefaultFixture; + static const char* library_name = "machine_scope"; static legate::Logger logger(library_name); @@ -131,7 +134,7 @@ void test_cpu_only(legate::Library library) } } -TEST(Integration, MachineScope) +TEST_F(Integration, MachineScope) { register_tasks(); diff --git a/tests/cpp/integration/manual_simple.cc b/tests/cpp/integration/manual_simple.cc index 32b9f6419d..0f6e6abe35 100644 --- a/tests/cpp/integration/manual_simple.cc +++ b/tests/cpp/integration/manual_simple.cc @@ -14,9 +14,12 @@ #include "legate.h" #include "tasks/task_simple.h" +#include "utilities/utilities.h" namespace manualsimple { +using Integration = DefaultFixture; + void test_auto_task(legate::Library library, legate::LogicalStore store) { auto runtime = legate::Runtime::get_runtime(); @@ -47,7 +50,7 @@ void print_store(legate::LogicalStore store) task::simple::logger.print() << ss.str(); } -TEST(Integration, ManualSimple) +TEST_F(Integration, ManualSimple) { task::simple::register_tasks(); diff --git a/tests/cpp/integration/multi_scalar_out.cc b/tests/cpp/integration/multi_scalar_out.cc index 3eb7bf0f1e..4f86c2bd3f 100644 --- a/tests/cpp/integration/multi_scalar_out.cc +++ b/tests/cpp/integration/multi_scalar_out.cc @@ -14,9 +14,12 @@ #include "legate.h" #include "tasks/task_simple.h" +#include "utilities/utilities.h" namespace multiscalarout { +using Integration = DefaultFixture; + void test_writer_auto(legate::Library library, legate::LogicalStore scalar1, legate::LogicalStore scalar2) @@ -71,7 +74,7 @@ void validate_stores(legate::LogicalStore scalar1, EXPECT_EQ(v2, to_match2); } -TEST(Integration, MultiScalarOut) +TEST_F(Integration, MultiScalarOut) { task::simple::register_tasks(); diff --git a/tests/cpp/integration/nccl.cu b/tests/cpp/integration/nccl.cu index 9d93ef1093..c066a25afa 100644 --- a/tests/cpp/integration/nccl.cu +++ b/tests/cpp/integration/nccl.cu @@ -15,11 +15,14 @@ #include "core/cuda/cuda_help.h" #include "core/cuda/stream_pool.h" #include "legate.h" +#include "utilities/utilities.h" #include namespace nccl { +using Integration = DefaultFixture; + const char* library_name = "test_nccl"; enum TaskIDs { @@ -99,7 +102,7 @@ void test_nccl_manual(int32_t ndim) } // Test case with single unbound store -TEST(Integration, NCCL) +TEST_F(Integration, NCCL) { prepare(); diff --git a/tests/cpp/integration/provenance.cc b/tests/cpp/integration/provenance.cc index 09addbccec..d7e7aa29d8 100644 --- a/tests/cpp/integration/provenance.cc +++ b/tests/cpp/integration/provenance.cc @@ -14,9 +14,12 @@ #include "core/runtime/detail/runtime.h" #include "legate.h" +#include "utilities/utilities.h" namespace provenance { +using Integration = DefaultFixture; + static const char* library_name = "test_provenance"; enum TaskIDs { @@ -132,7 +135,7 @@ void test_manual_tracker(legate::Library library) runtime->submit(std::move(task)); } -TEST(Integration, Provenance) +TEST_F(Integration, Provenance) { register_tasks(); diff --git a/tests/cpp/integration/region_manager.cc b/tests/cpp/integration/region_manager.cc index 356632aa80..c4a6c11778 100644 --- a/tests/cpp/integration/region_manager.cc +++ b/tests/cpp/integration/region_manager.cc @@ -14,10 +14,13 @@ #include "legate.h" #include "tasks/task_region_manager.h" +#include "utilities/utilities.h" namespace region_manager { -TEST(Integration, RegionManager) +using Integration = DefaultFixture; + +TEST_F(Integration, RegionManager) { task::region_manager::register_tasks(); diff --git a/tests/cpp/integration/req_analyzer.cc b/tests/cpp/integration/req_analyzer.cc index e185d6ca90..9938e17250 100644 --- a/tests/cpp/integration/req_analyzer.cc +++ b/tests/cpp/integration/req_analyzer.cc @@ -13,9 +13,12 @@ #include #include "legate.h" +#include "utilities/utilities.h" namespace req_analyzer { +using ReqAnalyzer = DefaultFixture; + static const char* library_name = "test_req_analyzer"; enum TaskIDs { @@ -81,13 +84,13 @@ void test_isomorphic_transformed_stores() runtime->submit(std::move(task)); } -TEST(ReqAnalyzer, InoutStore) +TEST_F(ReqAnalyzer, InoutStore) { prepare(); test_inout_store(); } -TEST(ReqAnalyzer, IsomorphicTransformedStores) +TEST_F(ReqAnalyzer, IsomorphicTransformedStores) { prepare(); test_isomorphic_transformed_stores(); diff --git a/tests/cpp/integration/scale_constraints.cc b/tests/cpp/integration/scale_constraints.cc index 1b1e756bd5..56ec90ae26 100644 --- a/tests/cpp/integration/scale_constraints.cc +++ b/tests/cpp/integration/scale_constraints.cc @@ -14,9 +14,12 @@ #include "core/data/detail/logical_store.h" #include "legate.h" +#include "utilities/utilities.h" namespace scale_constraints { +using ScaleConstraint = DefaultFixture; + static const char* library_name = "test_scale_constraints"; static legate::Logger logger(library_name); @@ -117,25 +120,25 @@ void test_invalid() } } -TEST(ScaleConstraint, 1D) +TEST_F(ScaleConstraint, 1D) { prepare(); test_scale({{3}, {10}, {29}}); } -TEST(ScaleConstraint, 2D) +TEST_F(ScaleConstraint, 2D) { prepare(); test_scale({{4, 5}, {2, 7}, {10, 30}}); } -TEST(ScaleConstraint, 3D) +TEST_F(ScaleConstraint, 3D) { prepare(); test_scale({{2, 3, 4}, {5, 5, 5}, {10, 15, 20}}); } -TEST(ScaleConstraint, Invalid) +TEST_F(ScaleConstraint, Invalid) { prepare(); test_invalid(); diff --git a/tests/cpp/integration/tree_reduce.cc b/tests/cpp/integration/tree_reduce.cc index 05824fd73d..8f06d9fdea 100644 --- a/tests/cpp/integration/tree_reduce.cc +++ b/tests/cpp/integration/tree_reduce.cc @@ -17,9 +17,12 @@ #include #include "legate.h" +#include "utilities/utilities.h" namespace tree_reduce { +using TreeReduce = DefaultFixture; + static const char* library_name = "test_tree_reduce"; static const size_t TILE_SIZE = 10; @@ -90,7 +93,7 @@ void register_tasks() ReduceUnboundTask::register_variants(context); } -TEST(TreeReduce, AutoProducer) +TEST_F(TreeReduce, AutoProducer) { register_tasks(); @@ -112,7 +115,7 @@ TEST(TreeReduce, AutoProducer) EXPECT_FALSE(result.unbound()); } -TEST(TreeReduce, ManualProducer) +TEST_F(TreeReduce, ManualProducer) { register_tasks(); @@ -134,7 +137,7 @@ TEST(TreeReduce, ManualProducer) EXPECT_FALSE(result.unbound()); } -TEST(TreeReduce, ManualProducerMultiLevel) +TEST_F(TreeReduce, ManualProducerMultiLevel) { register_tasks(); @@ -156,7 +159,7 @@ TEST(TreeReduce, ManualProducerMultiLevel) EXPECT_FALSE(result.unbound()); } -TEST(TreeReduce, ManualProducerUnbound) +TEST_F(TreeReduce, ManualProducerUnbound) { register_tasks(); @@ -175,7 +178,7 @@ TEST(TreeReduce, ManualProducerUnbound) EXPECT_FALSE(result.unbound()); } -TEST(TreeReduce, ManualProducerSingle) +TEST_F(TreeReduce, ManualProducerSingle) { register_tasks(); @@ -193,7 +196,7 @@ TEST(TreeReduce, ManualProducerSingle) EXPECT_FALSE(result.unbound()); } -TEST(TreeReduce, AutoProducerSingle) +TEST_F(TreeReduce, AutoProducerSingle) { register_tasks(); diff --git a/tests/cpp/integration/tree_reduce_unique.cc b/tests/cpp/integration/tree_reduce_unique.cc index ed7400ee19..c9c2b53577 100644 --- a/tests/cpp/integration/tree_reduce_unique.cc +++ b/tests/cpp/integration/tree_reduce_unique.cc @@ -17,9 +17,12 @@ #include #include "legate.h" +#include "utilities/utilities.h" namespace tree_reduce_unique { +using TreeReduce = DefaultFixture; + static const char* library_name = "test_tree_reduce_unique"; static const size_t TILE_SIZE = 10; @@ -104,7 +107,7 @@ void register_tasks() CheckTask::register_variants(context); } -TEST(TreeReduce, Unique) +TEST_F(TreeReduce, Unique) { register_tasks(); diff --git a/tests/cpp/integration/tunable.cc b/tests/cpp/integration/tunable.cc index 7ff726d469..fb5c0325f3 100644 --- a/tests/cpp/integration/tunable.cc +++ b/tests/cpp/integration/tunable.cc @@ -13,9 +13,12 @@ #include #include "legate.h" +#include "utilities/utilities.h" namespace tunable { +using Tunable = DefaultFixture; + static const char* library_name = "test_tunable"; const std::vector TUNABLES = { @@ -68,7 +71,7 @@ struct scalar_eq_fn { } }; -TEST(Tunable, Valid) +TEST_F(Tunable, Valid) { prepare(); auto runtime = legate::Runtime::get_runtime(); @@ -82,7 +85,7 @@ TEST(Tunable, Valid) } } -TEST(Tunable, Invalid) +TEST_F(Tunable, Invalid) { prepare(); auto runtime = legate::Runtime::get_runtime(); diff --git a/tests/cpp/integration/weighted.cc b/tests/cpp/integration/weighted.cc index 90778777dd..b93d67ce74 100644 --- a/tests/cpp/integration/weighted.cc +++ b/tests/cpp/integration/weighted.cc @@ -13,9 +13,12 @@ #include #include "legate.h" +#include "utilities/utilities.h" namespace weighted { +using Integration = DefaultFixture; + const char* library_name = "test_weighted"; enum TaskIDs { @@ -103,7 +106,7 @@ void test_weighted(uint32_t num_stores) } // Test case with single unbound store -TEST(Integration, WeightedSingle) +TEST_F(Integration, WeightedSingle) { prepare(); @@ -111,7 +114,7 @@ TEST(Integration, WeightedSingle) } // Test case with multiple unbound stores -TEST(Integration, WeightedMultiple) +TEST_F(Integration, WeightedMultiple) { prepare(); diff --git a/tests/cpp/main.cc b/tests/cpp/main.cc index c3d9e46a2e..0e657e6934 100644 --- a/tests/cpp/main.cc +++ b/tests/cpp/main.cc @@ -12,23 +12,13 @@ #include #include "legate.h" - -class Environment : public ::testing::Environment { - public: - Environment(int argc, char** argv) : argc_(argc), argv_(argv) {} - - void SetUp() override { EXPECT_EQ(legate::start(argc_, argv_), 0); } - void TearDown() override { EXPECT_EQ(legate::finish(), 0); } - - private: - int argc_; - char** argv_; -}; +#include "utilities/utilities.h" int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); - ::testing::AddGlobalTestEnvironment(new Environment(argc, argv)); + DefaultFixture::init(argc, argv); + DeathTestFixture::init(argc, argv); return RUN_ALL_TESTS(); } diff --git a/tests/cpp/unit/buffer.cc b/tests/cpp/unit/buffer.cc index 87d68ed540..f884acbc45 100644 --- a/tests/cpp/unit/buffer.cc +++ b/tests/cpp/unit/buffer.cc @@ -12,9 +12,12 @@ #include #include "legate.h" +#include "utilities/utilities.h" namespace buffer_test { +using BufferUnit = DefaultFixture; + static const char* library_name = "legate.buffer"; constexpr int64_t BUFFER_TASK_ID = 0; @@ -93,7 +96,7 @@ void register_tasks() BufferTask::register_variants(context); } -TEST(BufferUnit, CreateBuffer) +TEST_F(BufferUnit, CreateBuffer) { register_tasks(); @@ -104,7 +107,7 @@ TEST(BufferUnit, CreateBuffer) test_buffer(4, 10, legate::Memory::SYSTEM_MEM); } -TEST(BufferUnit, NegativeTest) +TEST_F(BufferUnit, NegativeTest) { register_tasks(); diff --git a/tests/cpp/unit/constraint.cc b/tests/cpp/unit/constraint.cc index 695b2351aa..fd9cf009b7 100644 --- a/tests/cpp/unit/constraint.cc +++ b/tests/cpp/unit/constraint.cc @@ -14,8 +14,15 @@ #include "core/partitioning/detail/constraint.h" #include "legate.h" +#include "utilities/utilities.h" namespace unit { + +using Variable = DefaultFixture; +using Alignment = DefaultFixture; +using Broadcast = DefaultFixture; +using ImageConstraint = DefaultFixture; + static const char* library_name = "test_constraints"; enum TaskIDs { @@ -38,7 +45,7 @@ void register_tasks() Initializer::register_variants(context); } -TEST(Variable, BasicMethods) +TEST_F(Variable, BasicMethods) { register_tasks(); @@ -74,7 +81,7 @@ TEST(Variable, BasicMethods) EXPECT_TRUE(std::find(symbols.begin(), symbols.end(), part2_imp) != symbols.end()); } -TEST(Alignment, BasicMethods) +TEST_F(Alignment, BasicMethods) { register_tasks(); @@ -102,7 +109,7 @@ TEST(Alignment, BasicMethods) EXPECT_TRUE(std::find(symbols.begin(), symbols.end(), part2.impl()) != symbols.end()); } -TEST(Broadcast, BasicMethods) +TEST_F(Broadcast, BasicMethods) { register_tasks(); @@ -127,7 +134,7 @@ TEST(Broadcast, BasicMethods) EXPECT_TRUE(std::find(symbols.begin(), symbols.end(), part1.impl()) != symbols.end()); } -TEST(ImageConstraint, BasicMethods) +TEST_F(ImageConstraint, BasicMethods) { register_tasks(); diff --git a/tests/cpp/unit/death_example.cc b/tests/cpp/unit/death_example.cc new file mode 100644 index 0000000000..8c26970d21 --- /dev/null +++ b/tests/cpp/unit/death_example.cc @@ -0,0 +1,35 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include + +#include "legate.h" +#include "utilities/utilities.h" + +namespace unit { + +using DeathTestExample = DeathTestFixture; + +void KillProcess(int argc, char** argv) +{ + legate::start(0, NULL); + abort(); +} + +TEST_F(DeathTestExample, Simple) +{ + // We can't check that the subprocess dies with SIGABRT, because we run with REALM_BACKTRACE=1, + // and Realm's signal hanlder doesn't propagate the signal, instead it exits right away + EXPECT_EXIT(KillProcess(argc_, argv_), ::testing::ExitedWithCode(1), ""); +} + +} // namespace unit diff --git a/tests/cpp/unit/library.cc b/tests/cpp/unit/library.cc index 81fbc58016..ce344dac13 100644 --- a/tests/cpp/unit/library.cc +++ b/tests/cpp/unit/library.cc @@ -13,10 +13,13 @@ #include #include "legate.h" +#include "utilities/utilities.h" namespace test_library { -TEST(Library, Create) +using Library = DefaultFixture; +TEST_F(Library, Create) + { const char* LIBNAME = "test_library.libA"; auto* runtime = legate::Runtime::get_runtime(); @@ -25,7 +28,7 @@ TEST(Library, Create) EXPECT_EQ(lib, runtime->maybe_find_library(LIBNAME).value()); } -TEST(Library, FindOrCreate) +TEST_F(Library, FindOrCreate) { const char* LIBNAME = "test_library.libB"; @@ -46,7 +49,7 @@ TEST(Library, FindOrCreate) EXPECT_FALSE(p_lib2.valid_task_id(p_lib2.get_task_id(1))); } -TEST(Library, FindNonExistent) +TEST_F(Library, FindNonExistent) { const char* LIBNAME = "test_library.libC"; diff --git a/tests/cpp/unit/logical_array_creation.cc b/tests/cpp/unit/logical_array_creation.cc index ec609d5890..6871a72647 100644 --- a/tests/cpp/unit/logical_array_creation.cc +++ b/tests/cpp/unit/logical_array_creation.cc @@ -13,9 +13,12 @@ #include #include "legate.h" +#include "utilities/utilities.h" namespace array_test { +using LogicalArray = DefaultFixture; + void test_primitive_array(bool nullable) { auto runtime = legate::Runtime::get_runtime(); @@ -211,24 +214,24 @@ void test_invalid() EXPECT_THROW(runtime->create_array(legate::list_type(legate::int64()), 3), std::invalid_argument); } -TEST(LogicalArray, CreatePrimitiveNonNullable) { test_primitive_array(false); } +TEST_F(LogicalArray, CreatePrimitiveNonNullable) { test_primitive_array(false); } -TEST(LogicalArray, CreatePrimitiveNullable) { test_primitive_array(true); } +TEST_F(LogicalArray, CreatePrimitiveNullable) { test_primitive_array(true); } -TEST(LogicalArray, CreateListNonNullable) { test_list_array(false); } +TEST_F(LogicalArray, CreateListNonNullable) { test_list_array(false); } -TEST(LogicalArray, CreateListNullable) { test_list_array(true); } +TEST_F(LogicalArray, CreateListNullable) { test_list_array(true); } -TEST(LogicalArray, CreateStructNonNullable) { test_struct_array(false); } +TEST_F(LogicalArray, CreateStructNonNullable) { test_struct_array(false); } -TEST(LogicalArray, CreateStructNullable) { test_struct_array(true); } +TEST_F(LogicalArray, CreateStructNullable) { test_struct_array(true); } -TEST(LogicalArray, CreateLike) +TEST_F(LogicalArray, CreateLike) { test_isomorphic(false); test_isomorphic(true); } -TEST(LogicalArray, CreateInvalid) { test_invalid(); } +TEST_F(LogicalArray, CreateInvalid) { test_invalid(); } } // namespace array_test diff --git a/tests/cpp/unit/logical_store.cc b/tests/cpp/unit/logical_store.cc index cd9a823544..17b8e7c030 100644 --- a/tests/cpp/unit/logical_store.cc +++ b/tests/cpp/unit/logical_store.cc @@ -13,10 +13,13 @@ #include #include "legate.h" +#include "utilities/utilities.h" namespace unit { -TEST(Store, Creation) +using Store = DefaultFixture; + +TEST_F(Store, Creation) { // Bound { @@ -41,7 +44,7 @@ TEST(Store, Creation) } } -TEST(Store, Transform) +TEST_F(Store, Transform) { // Bound auto runtime = legate::Runtime::get_runtime(); @@ -71,7 +74,7 @@ TEST(Store, Transform) EXPECT_TRUE(delinearized.transformed()); } -TEST(Store, InvalidTransform) +TEST_F(Store, InvalidTransform) { // Bound { diff --git a/tests/cpp/unit/machine.cc b/tests/cpp/unit/machine.cc index 50d0a63fa3..f07c4b2bc2 100644 --- a/tests/cpp/unit/machine.cc +++ b/tests/cpp/unit/machine.cc @@ -21,10 +21,13 @@ #include "core/utilities/deserializer.h" #include "core/utilities/detail/buffer_builder.h" #include "legate.h" +#include "utilities/utilities.h" namespace unit { -TEST(Machine, ProcessorRange) +using Machine = DefaultFixture; + +TEST_F(Machine, ProcessorRange) { // create nonempty { @@ -132,7 +135,7 @@ TEST(Machine, ProcessorRange) } } -TEST(Machine, MachineDesc) +TEST_F(Machine, MachineDesc) { // test empty MachineDesc { diff --git a/tests/cpp/unit/mapping.cc b/tests/cpp/unit/mapping.cc index c294bda724..1254a457e4 100644 --- a/tests/cpp/unit/mapping.cc +++ b/tests/cpp/unit/mapping.cc @@ -14,10 +14,13 @@ #include "core/mapping/detail/mapping.h" #include "legate.h" +#include "utilities/utilities.h" namespace unit { -TEST(Mapping, DimOrdering) +using Mapping = DefaultFixture; + +TEST_F(Mapping, DimOrdering) { // Default construct { @@ -71,7 +74,7 @@ TEST(Mapping, DimOrdering) } } -TEST(Mapping, InstanceMappingPolicy) +TEST_F(Mapping, InstanceMappingPolicy) { // Empty InstanceMappingPolicy { diff --git a/tests/cpp/unit/registration.cc b/tests/cpp/unit/registration.cc index 9e4fd26a09..bd25b5cbe8 100644 --- a/tests/cpp/unit/registration.cc +++ b/tests/cpp/unit/registration.cc @@ -13,6 +13,9 @@ #include #include "legate.h" +#include "utilities/utilities.h" + +using Registration = DefaultFixture; namespace test_registration { @@ -49,6 +52,6 @@ void test_out_of_bounds_task_id() EXPECT_THROW(test_registration::CPUVariantTask<1>::register_variants(library), std::out_of_range); } -TEST(Registration, Duplicate) { test_duplicates(); } +TEST_F(Registration, Duplicate) { test_duplicates(); } -TEST(Registration, TaskIDOutOfBounds) { test_out_of_bounds_task_id(); } +TEST_F(Registration, TaskIDOutOfBounds) { test_out_of_bounds_task_id(); } diff --git a/tests/cpp/unit/scalar.cc b/tests/cpp/unit/scalar.cc index 4e3839c698..5f73df7e81 100644 --- a/tests/cpp/unit/scalar.cc +++ b/tests/cpp/unit/scalar.cc @@ -15,9 +15,12 @@ #include "core/data/detail/scalar.h" #include "core/utilities/deserializer.h" #include "core/utilities/detail/buffer_builder.h" +#include "utilities/utilities.h" namespace scalar_test { +using ScalarUnit = DefaultFixture; + constexpr bool BOOL_VALUE = true; constexpr int8_t INT8_VALUE = 10; constexpr int16_t INT16_VALUE = -1000; @@ -252,7 +255,7 @@ void check_pack_rect_scalar() check_pack(scalar); } -TEST(ScalarUnit, CreateWithObject) +TEST_F(ScalarUnit, CreateWithObject) { // constructor with Scalar object legate::Scalar scalar1(INT32_VALUE); @@ -270,7 +273,7 @@ TEST(ScalarUnit, CreateWithObject) EXPECT_THROW(scalar2.values(), std::invalid_argument); } -TEST(ScalarUnit, CreateSharedScalar) +TEST_F(ScalarUnit, CreateSharedScalar) { // unowned { @@ -301,7 +304,7 @@ TEST(ScalarUnit, CreateSharedScalar) } } -TEST(ScalarUnit, CreateWithValue) +TEST_F(ScalarUnit, CreateWithValue) { check_type(BOOL_VALUE, legate::bool_()); check_type(INT8_VALUE, legate::int8()); @@ -322,7 +325,7 @@ TEST(ScalarUnit, CreateWithValue) EXPECT_THROW(legate::Scalar(FLOAT16_VALUE, legate::float32()), std::invalid_argument); } -TEST(ScalarUnit, CreateBinary) +TEST_F(ScalarUnit, CreateBinary) { { PaddingStructData value = {BOOL_VALUE, INT32_VALUE, UINT64_VALUE}; @@ -335,7 +338,7 @@ TEST(ScalarUnit, CreateBinary) } } -TEST(ScalarUnit, CreateWithVector) +TEST_F(ScalarUnit, CreateWithVector) { // constructor with arrays std::vector scalar_data{INT32_VALUE, INT32_VALUE}; @@ -359,7 +362,7 @@ TEST(ScalarUnit, CreateWithVector) EXPECT_THROW(scalar.values(), std::invalid_argument); } -TEST(ScalarUnit, CreateWithString) +TEST_F(ScalarUnit, CreateWithString) { // constructor with string auto inputString = STRING_VALUE; @@ -381,7 +384,7 @@ TEST(ScalarUnit, CreateWithString) EXPECT_THROW(scalar.values(), std::invalid_argument); } -TEST(ScalarUnit, CreateWithStructType) +TEST_F(ScalarUnit, CreateWithStructType) { // with struct padding { @@ -396,7 +399,7 @@ TEST(ScalarUnit, CreateWithStructType) } } -TEST(ScalarUnit, CreateWithPoint) +TEST_F(ScalarUnit, CreateWithPoint) { { const int64_t bounds[] = {0}; @@ -422,7 +425,7 @@ TEST(ScalarUnit, CreateWithPoint) EXPECT_THROW(legate::Scalar{legate::Point<10>::ONES()}, std::out_of_range); } -TEST(ScalarUnit, CreateWithRect) +TEST_F(ScalarUnit, CreateWithRect) { { const int64_t lo[] = {1}; @@ -454,7 +457,7 @@ TEST(ScalarUnit, CreateWithRect) std::out_of_range); } -TEST(ScalarUnit, OperatorEqual) +TEST_F(ScalarUnit, OperatorEqual) { legate::Scalar scalar1(INT32_VALUE); legate::Scalar scalar2(UINT64_VALUE); @@ -465,7 +468,7 @@ TEST(ScalarUnit, OperatorEqual) EXPECT_EQ(scalar2.values().size(), scalar1.values().size()); } -TEST(ScalarUnit, Pack) +TEST_F(ScalarUnit, Pack) { // test pack for a fixed array scalar { diff --git a/tests/cpp/unit/scoped_allocator.cc b/tests/cpp/unit/scoped_allocator.cc index 1125770fce..2947719b67 100644 --- a/tests/cpp/unit/scoped_allocator.cc +++ b/tests/cpp/unit/scoped_allocator.cc @@ -12,9 +12,12 @@ #include #include "legate.h" +#include "utilities/utilities.h" namespace scoped_allocator_test { +using ScopedAllocatorUnit = DefaultFixture; + static const char* library_name = "legate.scopedallocator"; constexpr int64_t ALLOCATOR_TASK_ID = 1; @@ -103,14 +106,14 @@ void register_tasks() ScopedAllocatorTask::register_variants(context); } -TEST(ScopedAllocatorUnit, EmptyAllocate) +TEST_F(ScopedAllocatorUnit, EmptyAllocate) { auto allocator = legate::ScopedAllocator(legate::Memory::SYSTEM_MEM, true); void* ptr = allocator.allocate(0); EXPECT_EQ(ptr, nullptr); } -TEST(ScopedAllocatorUnit, Allocate) +TEST_F(ScopedAllocatorUnit, Allocate) { register_tasks(); test_allocator(BufferOpCode::DEALLOCATE, legate::Memory::SYSTEM_MEM, true, 1000, 16); @@ -124,13 +127,13 @@ TEST(ScopedAllocatorUnit, Allocate) // test_allocator(BufferOpCode::DEALLOCATE, legate::Memory::SYSTEM_MEM, false, 1000, -1); } -TEST(ScopedAllocatorUnit, DoubleDeallocate) +TEST_F(ScopedAllocatorUnit, DoubleDeallocate) { register_tasks(); test_allocator(BufferOpCode::DOUBLE_DEALLOCATE, legate::Memory::SYSTEM_MEM, true, 1000); } -TEST(ScopedAllocatorUnit, InvalidDeallocate) +TEST_F(ScopedAllocatorUnit, InvalidDeallocate) { auto allocator = legate::ScopedAllocator(legate::Memory::SYSTEM_MEM, true); std::vector data = {1, 2, 3}; diff --git a/tests/cpp/unit/span.cc b/tests/cpp/unit/span.cc index 62d6fd488f..597b8a355b 100644 --- a/tests/cpp/unit/span.cc +++ b/tests/cpp/unit/span.cc @@ -12,9 +12,12 @@ #include #include "legate.h" +#include "utilities/utilities.h" namespace span_test { +using SpanUnit = DefaultFixture; + constexpr bool BOOL_VALUE = true; constexpr int8_t INT8_VALUE = 10; constexpr int16_t INT16_VALUE = -1000; @@ -55,7 +58,7 @@ void create(T value1, T value2, T value3) EXPECT_EQ(span[DATA_SIZE - 1], value3); } -TEST(SpanUnit, Create) +TEST_F(SpanUnit, Create) { create(BOOL_VALUE, BOOL_VALUE, !BOOL_VALUE); create(INT8_VALUE, static_cast(INT8_VALUE + 1), static_cast(INT8_VALUE + 2)); @@ -77,7 +80,7 @@ TEST(SpanUnit, Create) create(COMPLEX_DOUBLE_VALUE1, COMPLEX_DOUBLE_VALUE2, COMPLEX_DOUBLE_VALUE3); } -TEST(SpanUnit, Subspan) +TEST_F(SpanUnit, Subspan) { auto data_vec = std::vector(DATA_SIZE, UINT64_VALUE); const auto* data = data_vec.data(); diff --git a/tests/cpp/unit/type.cc b/tests/cpp/unit/type.cc index 62bfd990b6..50f0f13562 100644 --- a/tests/cpp/unit/type.cc +++ b/tests/cpp/unit/type.cc @@ -12,8 +12,12 @@ #include #include "legate.h" +#include "utilities/utilities.h" namespace type_test { + +using TypeUnit = DefaultFixture; + constexpr int32_t GLOBAL_OP_ID = 0x1F; const std::vector PRIMITIVE_TYPE = {legate::bool_(), @@ -145,7 +149,7 @@ void test_reduction_op(const legate::Type& type) EXPECT_THROW(type.find_reduction_operator(legate::ReductionOpKind::SUB), std::invalid_argument); } -TEST(TypeUnit, PrimitiveType) +TEST_F(TypeUnit, PrimitiveType) { test_primitive_type(legate::bool_(), "bool"); test_primitive_type(legate::int8(), "int8"); @@ -184,9 +188,9 @@ TEST(TypeUnit, PrimitiveType) EXPECT_THROW(legate::primitive_type(legate::Type::Code::LIST), std::invalid_argument); } -TEST(TypeUnit, StringType) { test_string_type(legate::string_type()); } +TEST_F(TypeUnit, StringType) { test_string_type(legate::string_type()); } -TEST(TypeUnit, BinaryType) +TEST_F(TypeUnit, BinaryType) { test_binary_type(legate::binary_type(123), 123); test_binary_type(legate::binary_type(45), 45); @@ -196,7 +200,7 @@ TEST(TypeUnit, BinaryType) EXPECT_THROW(legate::binary_type(0xFFFFF + 1), std::out_of_range); } -TEST(TypeUnit, FixedArrayType) +TEST_F(TypeUnit, FixedArrayType) { // element type is a primitive type { @@ -234,7 +238,7 @@ TEST(TypeUnit, FixedArrayType) EXPECT_THROW(legate::uint32().as_fixed_array_type(), std::invalid_argument); } -TEST(TypeUnit, StructType) +TEST_F(TypeUnit, StructType) { // aligned { @@ -285,7 +289,7 @@ TEST(TypeUnit, StructType) EXPECT_THROW(legate::uint32().as_struct_type(), std::invalid_argument); } -TEST(TypeUnit, PointType) +TEST_F(TypeUnit, PointType) { for (uint32_t idx = 1; idx <= LEGATE_MAX_DIM; ++idx) { auto type = legate::point_type(idx); @@ -310,7 +314,7 @@ TEST(TypeUnit, PointType) EXPECT_THROW(legate::point_type(LEGATE_MAX_DIM + 1), std::out_of_range); } -TEST(TypeUnit, RectType) +TEST_F(TypeUnit, RectType) { for (uint32_t idx = 1; idx <= LEGATE_MAX_DIM; ++idx) { auto type = legate::rect_type(idx); @@ -338,7 +342,7 @@ TEST(TypeUnit, RectType) EXPECT_THROW(legate::rect_type(LEGATE_MAX_DIM + 1), std::out_of_range); } -TEST(TypeUnit, ListType) +TEST_F(TypeUnit, ListType) { test_list_type(legate::bool_(), "list(bool)"); test_list_type(legate::int8(), "list(int8)"); @@ -369,7 +373,7 @@ TEST(TypeUnit, ListType) EXPECT_THROW(legate::string_type().as_struct_type(), std::invalid_argument); } -TEST(TypeUnit, Uid) +TEST_F(TypeUnit, Uid) { // fixed array type for (uint32_t idx = 0; idx < PRIMITIVE_TYPE.size(); ++idx) { @@ -453,14 +457,14 @@ TEST(TypeUnit, Uid) } } -TEST(TypeUnit, ReductionOperator) +TEST_F(TypeUnit, ReductionOperator) { test_reduction_op(legate::string_type()); test_reduction_op(legate::fixed_array_type(legate::int64(), 10)); test_reduction_op(legate::struct_type(true, legate::int64(), legate::bool_(), legate::float64())); } -TEST(TypeUnit, TypeCodeOf) +TEST_F(TypeUnit, TypeCodeOf) { EXPECT_EQ(legate::legate_type_code_of, legate::Type::Code::NIL); EXPECT_EQ(legate::legate_type_code_of, legate::Type::Code::BOOL); @@ -480,7 +484,7 @@ TEST(TypeUnit, TypeCodeOf) EXPECT_EQ(legate::legate_type_code_of, legate::Type::Code::STRING); } -TEST(TypeUnit, TypeOf) +TEST_F(TypeUnit, TypeOf) { EXPECT_TRUE((std::is_same_v, bool>)); EXPECT_TRUE((std::is_same_v, int8_t>)); @@ -506,7 +510,7 @@ TEST(TypeUnit, TypeOf) EXPECT_TRUE((std::is_same_v, void>)); } -TEST(TypeUnit, TypeUtils) +TEST_F(TypeUnit, TypeUtils) { // is_integral EXPECT_TRUE(legate::is_integral::value); diff --git a/tests/cpp/utilities/utilities.h b/tests/cpp/utilities/utilities.h new file mode 100644 index 0000000000..b2c7a55bf7 --- /dev/null +++ b/tests/cpp/utilities/utilities.h @@ -0,0 +1,44 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#pragma once + +#include +#include "legate.h" + +class DefaultFixture : public ::testing::Test { + public: + static void init(int argc, char** argv) + { + DefaultFixture::argc_ = argc; + DefaultFixture::argv_ = argv; + } + + void SetUp() override { EXPECT_EQ(legate::start(argc_, argv_), 0); } + void TearDown() override { EXPECT_EQ(legate::finish(), 0); } + + private: + inline static int argc_; + inline static char** argv_; +}; + +class DeathTestFixture : public ::testing::Test { + public: + static void init(int argc, char** argv) + { + argc_ = argc; + argv_ = argv; + } + + inline static int argc_; + inline static char** argv_; +}; From d3c92dc510cc04066a8a3136759bbae2f263fa05 Mon Sep 17 00:00:00 2001 From: Manolis Papadakis Date: Wed, 4 Oct 2023 11:01:07 -0700 Subject: [PATCH 0246/1425] Tester improvements for multi-rank (#107) * Rename config.ranks to more accurate ranks_per_node * Consolidate --mpi-rank and --ranks-per-node * More defensive handling of default values * Add GTest filter option to run.py * Add fake executable name, so that all args get parsed --- legate/tester/args.py | 18 ++++-------- legate/tester/config.py | 3 +- legate/tester/stages/_linux/cpu.py | 13 +++++---- legate/tester/stages/_linux/gpu.py | 16 ++++++----- legate/tester/stages/_linux/omp.py | 13 +++++---- legate/tester/stages/test_stage.py | 37 +++++++++++++------------ legate/tester/test_plan.py | 4 +-- tests/cpp/run.py | 29 +++++++++++++------ tests/unit/legate/tester/test_config.py | 3 +- 9 files changed, 72 insertions(+), 64 deletions(-) diff --git a/legate/tester/args.py b/legate/tester/args.py index 27385d15b4..77c800cee1 100644 --- a/legate/tester/args.py +++ b/legate/tester/args.py @@ -188,7 +188,7 @@ feature_opts.add_argument( "--ranks-per-node", - dest="ranks", + dest="ranks_per_node", type=int, default=DEFAULT_RANKS_PER_NODE, help="Number of ranks per node to use", @@ -315,17 +315,9 @@ parser.add_argument( "--gtest-tests", dest="gtest_tests", - nargs="+", - default=None, - help="Explicit list of test names to run against", -) - - -parser.add_argument( - "--mpi-rank", - dest="mpi_rank", - default=None, - help="Runs mpirun with rank if non-zero", + nargs="*", + default=[], + help="Explicit list of test names to run", ) @@ -333,5 +325,5 @@ "--mpi-output-filename", dest="mpi_output_filename", default=None, - help="Path to mpirun results", + help="Directory to dump mpirun output", ) diff --git a/legate/tester/config.py b/legate/tester/config.py index d31c61c0fa..1d36633c67 100644 --- a/legate/tester/config.py +++ b/legate/tester/config.py @@ -49,7 +49,6 @@ def __init__(self, argv: ArgList) -> None: colors.ENABLED = args.color # mpi configuration - self.mpi_rank = args.mpi_rank self.mpi_output_filename = args.mpi_output_filename # gtest configuration @@ -77,7 +76,7 @@ def __init__(self, argv: ArgList) -> None: self.gpu_delay = args.gpu_delay self.ompthreads = args.ompthreads self.numamem = args.numamem - self.ranks = args.ranks + self.ranks_per_node = args.ranks_per_node # test run configuration self.timeout = args.timeout diff --git a/legate/tester/stages/_linux/cpu.py b/legate/tester/stages/_linux/cpu.py index 8ab2d1cea5..0830f4a738 100644 --- a/legate/tester/stages/_linux/cpu.py +++ b/legate/tester/stages/_linux/cpu.py @@ -57,10 +57,10 @@ def shard_args(self, shard: Shard, config: Config) -> ArgList: "--cpu-bind", str(shard), ] - if config.ranks > 1: + if config.ranks_per_node > 1: args += [ "--ranks-per-node", - str(config.ranks), + str(config.ranks_per_node), ] return args @@ -69,16 +69,17 @@ def compute_spec(self, config: Config, system: TestSystem) -> StageSpec: procs = config.cpus + config.utility + int(config.cpu_pin == "strict") workers = adjust_workers( - len(cpus) // (procs * config.ranks), config.requested_workers + len(cpus) // (procs * config.ranks_per_node), + config.requested_workers, ) shards: list[Shard] = [] for i in range(workers): rank_shards = [] - for j in range(config.ranks): + for j in range(config.ranks_per_node): shard_cpus = range( - (j + i * config.ranks) * procs, - (j + i * config.ranks + 1) * procs, + (j + i * config.ranks_per_node) * procs, + (j + i * config.ranks_per_node + 1) * procs, ) shard = chain.from_iterable(cpus[k].ids for k in shard_cpus) rank_shards.append(tuple(sorted(shard))) diff --git a/legate/tester/stages/_linux/gpu.py b/legate/tester/stages/_linux/gpu.py index 1e5683be1f..92f5fe0797 100644 --- a/legate/tester/stages/_linux/gpu.py +++ b/legate/tester/stages/_linux/gpu.py @@ -59,16 +59,16 @@ def shard_args(self, shard: Shard, config: Config) -> ArgList: "--gpu-bind", str(shard), ] - if config.ranks > 1: + if config.ranks_per_node > 1: args += [ "--ranks-per-node", - str(config.ranks), + str(config.ranks_per_node), ] return args def compute_spec(self, config: Config, system: TestSystem) -> StageSpec: N = len(system.gpus) - degree = N // (config.gpus * config.ranks) + degree = N // (config.gpus * config.ranks_per_node) fbsize = min(gpu.total for gpu in system.gpus) / (1 << 20) # MB oversub_factor = int(fbsize // (config.fbmem * config.bloat_factor)) @@ -79,15 +79,17 @@ def compute_spec(self, config: Config, system: TestSystem) -> StageSpec: shards: list[Shard] = [] for i in range(degree): rank_shards = [] - for j in range(config.ranks): + for j in range(config.ranks_per_node): shard_gpus = range( - (j + i * config.ranks) * config.gpus, - (j + i * config.ranks + 1) * config.gpus, + (j + i * config.ranks_per_node) * config.gpus, + (j + i * config.ranks_per_node + 1) * config.gpus, ) shard = tuple(shard_gpus) rank_shards.append(shard) shards.append(Shard(rank_shards)) - shard_factor = workers if config.ranks == 1 else oversub_factor + shard_factor = ( + workers if config.ranks_per_node == 1 else oversub_factor + ) return StageSpec(workers, shards * shard_factor) diff --git a/legate/tester/stages/_linux/omp.py b/legate/tester/stages/_linux/omp.py index 0fd0085bbc..de540951f0 100644 --- a/legate/tester/stages/_linux/omp.py +++ b/legate/tester/stages/_linux/omp.py @@ -61,10 +61,10 @@ def shard_args(self, shard: Shard, config: Config) -> ArgList: "--cpu-bind", str(shard), ] - if config.ranks > 1: + if config.ranks_per_node > 1: args += [ "--ranks-per-node", - str(config.ranks), + str(config.ranks_per_node), ] return args @@ -75,16 +75,17 @@ def compute_spec(self, config: Config, system: TestSystem) -> StageSpec: omps * threads + config.utility + int(config.cpu_pin == "strict") ) workers = adjust_workers( - len(cpus) // (procs * config.ranks), config.requested_workers + len(cpus) // (procs * config.ranks_per_node), + config.requested_workers, ) shards: list[Shard] = [] for i in range(workers): rank_shards = [] - for j in range(config.ranks): + for j in range(config.ranks_per_node): shard_cpus = range( - (j + i * config.ranks) * procs, - (j + i * config.ranks + 1) * procs, + (j + i * config.ranks_per_node) * procs, + (j + i * config.ranks_per_node + 1) * procs, ) shard = chain.from_iterable(cpus[k].ids for k in shard_cpus) rank_shards.append(tuple(sorted(shard))) diff --git a/legate/tester/stages/test_stage.py b/legate/tester/stages/test_stage.py index 6311eb9942..141d99b52f 100644 --- a/legate/tester/stages/test_stage.py +++ b/legate/tester/stages/test_stage.py @@ -380,8 +380,10 @@ def run_mpi( stage_args = self.args + self.shard_args(shard, config) mpi_args = [] - mpi_args += ["mpirun", "-n", str(config.ranks)] - mpi_args += ["--output-filename", config.mpi_output_filename] + mpi_args += ["mpirun", "-n", str(config.ranks_per_node)] + mpi_args += ["--npernode", str(config.ranks_per_node)] + if config.mpi_output_filename: + mpi_args += ["--output-filename", config.mpi_output_filename] mpi_args += ["--merge-stderr-to-stdout"] for var in dict(os.environ): @@ -427,21 +429,22 @@ def _launch( ) -> list[ProcessResult]: pool = multiprocessing.pool.ThreadPool(self.spec.workers) - if config.mpi_rank: - jobs = [ - pool.apply_async( - self.run_mpi, (config.gtest_file, arg, config, system) - ) - for arg in config.gtest_tests - ] - elif config.gtest_file: - jobs = [ - pool.apply_async( - self.run_gtest, - (config.gtest_file, arg, config, system), - ) - for arg in config.gtest_tests - ] + if config.gtest_file: + if config.ranks_per_node > 1: + jobs = [ + pool.apply_async( + self.run_mpi, (config.gtest_file, arg, config, system) + ) + for arg in config.gtest_tests + ] + else: + jobs = [ + pool.apply_async( + self.run_gtest, + (config.gtest_file, arg, config, system), + ) + for arg in config.gtest_tests + ] else: jobs = [ pool.apply_async(self.run_python, (path, config, system)) diff --git a/legate/tester/test_plan.py b/legate/tester/test_plan.py index 8e69583ee4..dd4adc5ffb 100644 --- a/legate/tester/test_plan.py +++ b/legate/tester/test_plan.py @@ -85,14 +85,14 @@ def intro(self) -> str: except RuntimeError: gpus = "N/A" - if self._config.mpi_rank or self._config.gtest_file: + if self._config.gtest_file: details = ( f"* Feature stages : {', '.join(yellow(x) for x in self._config.features)}", # noqa E501 f"* Test files per stage : {yellow(str(len(self._config.gtest_tests)))}", # noqa E501 f"* TestSystem description : {yellow(str(cpus) + ' cpus')} / {yellow(str(gpus) + ' gpus')}", # noqa E501 ) return banner( - f"Test Suite Configuration ({'OpenMPI' if self._config.mpi_rank else 'GTest'})", # noqa E501 + f"Test Suite Configuration ({'OpenMPI' if self._config.ranks_per_node > 1 else 'GTest'})", # noqa E501 details=details, ) diff --git a/tests/cpp/run.py b/tests/cpp/run.py index b74a243e4d..2437ebaef3 100755 --- a/tests/cpp/run.py +++ b/tests/cpp/run.py @@ -21,8 +21,12 @@ from legate.tester.test_system import TestSystem -def fetch_test_names(gtest_file): - list_command = [gtest_file] + ["--gtest_list_tests"] +def fetch_test_names(gtest_file, filter): + list_command = [ + gtest_file, + "--gtest_list_tests", + f"--gtest_filter={filter}", + ] result = subprocess.check_output(list_command, stderr=subprocess.STDOUT) result = result.decode(sys.stdout.encoding).split("\n") @@ -47,6 +51,11 @@ def fetch_test_names(gtest_file): def main(): parser = argparse.ArgumentParser(description="Run Legate cpp tests.") + parser.add_argument( + "--filter", + default="*", + help="Only run tests matching the GTest filter", + ) parser.add_argument( "--gtest-file", dest="gtest_file", @@ -55,20 +64,22 @@ def main(): help="GTest file under test", ) parser.add_argument( - "--mpi-rank", - dest="mpi_rank", + "--ranks-per-node", + dest="ranks_per_node", required=False, type=int, - default=0, - help="Runs mpirun with rank if non-zero", + default=1, + help="Number of ranks per node to use (will use mpirun if > 1)", ) args, extra_args = parser.parse_known_args() - if args.mpi_rank != 0: - extra_args += ["--mpi-rank", str(args.mpi_rank)] + if args.ranks_per_node > 1: + extra_args += ["--ranks-per-node", str(args.ranks_per_node)] extra_args += ["--mpi-output-filename", "build/mpi_result"] extra_args += ["--gtest-file", args.gtest_file] - extra_args += ["--gtest-tests"] + fetch_test_names(args.gtest_file) + extra_args += ["--gtest-tests"] + fetch_test_names( + args.gtest_file, args.filter + ) config = Config([sys.argv[0]] + extra_args) diff --git a/tests/unit/legate/tester/test_config.py b/tests/unit/legate/tester/test_config.py index 1820df2a24..6edc20abdb 100644 --- a/tests/unit/legate/tester/test_config.py +++ b/tests/unit/legate/tester/test_config.py @@ -84,8 +84,7 @@ def test_default_init(self) -> None: assert c.cov_src_path is None assert c.gtest_file is None - assert c.gtest_tests is None - assert c.mpi_rank is None + assert c.gtest_tests == [] assert c.mpi_output_filename is None def test_color_arg(self) -> None: From 511b229ff47511cea41dd2a0de641712089defb5 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 4 Oct 2023 12:22:39 -0700 Subject: [PATCH 0247/1425] Canonicalize `size_t` in the rest of `Scalar`'s ctors (#112) --- src/core/data/scalar.inl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/data/scalar.inl b/src/core/data/scalar.inl index 175d962b19..152cd738fc 100644 --- a/src/core/data/scalar.inl +++ b/src/core/data/scalar.inl @@ -77,13 +77,13 @@ Scalar::Scalar(T value, Type type) : impl_(create_impl(type, &value, true)) template Scalar::Scalar(const std::vector& values) : impl_(create_impl( - fixed_array_type(primitive_type(legate_type_code_of), values.size()), values.data(), true)) + fixed_array_type(primitive_type(detail::canonical_type_code_of()), values.size()), values.data(), true)) { } template Scalar::Scalar(const tuple& values) - : impl_(create_impl(fixed_array_type(primitive_type(legate_type_code_of), values.size()), + : impl_(create_impl(fixed_array_type(detail::canonical_type_code_of(), values.size()), values.data().data(), true)) { From bfda370d8a431833b0e9ed82f4b1b5824947ce74 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 4 Oct 2023 14:27:24 -0700 Subject: [PATCH 0248/1425] Remove an unused definition (#113) --- tests/cpp/integration/attach.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/cpp/integration/attach.cc b/tests/cpp/integration/attach.cc index 99ee1b4348..a52e08e62a 100644 --- a/tests/cpp/integration/attach.cc +++ b/tests/cpp/integration/attach.cc @@ -224,7 +224,6 @@ TEST_F(AttachDeathTest, MissingManualDetach) legate::start(argc_, argv_); register_tasks(); auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library(library_name); runtime->create_store( SHAPE_1D, legate::int64(), new int64_t[SHAPE_1D.volume()], true /*share*/); legate::finish(); From ab3d97db6b57cd1460f13e94465c2e6b14de7b15 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Wed, 4 Oct 2023 20:57:18 -0700 Subject: [PATCH 0249/1425] Minor fixes for tests (#114) * Minor fixes for tests * Move the comment to the right branch --- tests/cpp/integration/provenance.cc | 4 ++-- tests/cpp/unit/death_example.cc | 13 ++++++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/cpp/integration/provenance.cc b/tests/cpp/integration/provenance.cc index d7e7aa29d8..075a6b6cc6 100644 --- a/tests/cpp/integration/provenance.cc +++ b/tests/cpp/integration/provenance.cc @@ -108,7 +108,7 @@ void test_provenance_tracker(legate::Library library) auto runtime = legate::Runtime::get_runtime(); // auto task auto task = runtime->create_task(library, PROVENANCE); - std::string provenance = "provenance.cc:104"; + std::string provenance = "provenance.cc:107"; task.add_scalar_arg(legate::Scalar(provenance)); runtime->submit(std::move(task)); } @@ -120,7 +120,7 @@ void test_nested_provenance_tracker(legate::Library library) // The provenance string used by test_provenance_tracker should be popped out at this point auto runtime = legate::Runtime::get_runtime(); auto task = runtime->create_task(library, PROVENANCE); - std::string provenance = "provenance.cc:115"; + std::string provenance = "provenance.cc:118"; task.add_scalar_arg(legate::Scalar(provenance)); runtime->submit(std::move(task)); } diff --git a/tests/cpp/unit/death_example.cc b/tests/cpp/unit/death_example.cc index 8c26970d21..f0be4284ce 100644 --- a/tests/cpp/unit/death_example.cc +++ b/tests/cpp/unit/death_example.cc @@ -27,9 +27,16 @@ void KillProcess(int argc, char** argv) TEST_F(DeathTestExample, Simple) { - // We can't check that the subprocess dies with SIGABRT, because we run with REALM_BACKTRACE=1, - // and Realm's signal hanlder doesn't propagate the signal, instead it exits right away - EXPECT_EXIT(KillProcess(argc_, argv_), ::testing::ExitedWithCode(1), ""); + auto value = getenv("REALM_BACKTRACE"); + bool realm_backtrace = value != nullptr && atoi(value) != 0; + + if (realm_backtrace) { + // We can't check that the subprocess dies with SIGABRT when we run with REALM_BACKTRACE=1, + // because Realm's signal handler doesn't propagate the signal, instead it exits right away + EXPECT_EXIT(KillProcess(argc_, argv_), ::testing::ExitedWithCode(1), ""); + } else { + EXPECT_EXIT(KillProcess(argc_, argv_), ::testing::KilledBySignal(SIGABRT), ""); + } } } // namespace unit From 770662c360ed84572ee4ed9650420d5fb31c0f46 Mon Sep 17 00:00:00 2001 From: Jacob Faibussowitsch Date: Thu, 5 Oct 2023 15:52:25 -0400 Subject: [PATCH 0250/1425] Fix CI job matrix to run tests again (#94) * Fix CI job matrix to run tests again * Run mypy in a standalone test as we don't need to build legate to run it --- .../workflows/ci-gh-cpu-build-and-test.yml | 13 +++--- .../workflows/ci-gh-gpu-build-and-test.yml | 15 ++++--- .github/workflows/ci-gh-lint.yml | 41 +++++++++++++++++++ .github/workflows/gh-test.yml | 3 ++ .../home/coder/.local/bin/build-legate-conda | 2 +- .../coder/.local/bin/run-test-or-analysis | 39 +++++++++++++++++- 6 files changed, 99 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/ci-gh-lint.yml diff --git a/.github/workflows/ci-gh-cpu-build-and-test.yml b/.github/workflows/ci-gh-cpu-build-and-test.yml index d38470fc12..1afa54737c 100644 --- a/.github/workflows/ci-gh-cpu-build-and-test.yml +++ b/.github/workflows/ci-gh-cpu-build-and-test.yml @@ -5,6 +5,10 @@ concurrency: cancel-in-progress: true on: + workflow_run: + workflows: ["Lint legate.core on GH"] + types: + - completed push: branches: - "pull-request/[0-9]+" @@ -16,6 +20,7 @@ jobs: fail-fast: false matrix: cmake-preset: [debug-gcc, debug-sanitizer-gcc] + #if: ${{ github.event.workflow_run.conclusion == 'success' && github.repository_owner == 'nv-legate' }} if: ${{ github.repository_owner == 'nv-legate' }} uses: ./.github/workflows/gh-build.yml @@ -34,10 +39,8 @@ jobs: fail-fast: false matrix: cmake-preset: [debug-gcc, debug-sanitizer-gcc] - include: - - { name: Pytest Unit Tests, test-scope: unit } - - { name: mypy, test-scope: mypy } - name: ${{ matrix.name }} + test-config: [{ name: Pytest Unit Tests, test-scope: unit }] + name: ${{ matrix.test-config.name }} if: ${{ github.repository_owner == 'nv-legate' }} uses: ./.github/workflows/gh-test.yml @@ -47,7 +50,7 @@ jobs: runs-on: ${{ contains(github.repository, 'nv-legate/legate.core') && 'linux-amd64-cpu4' || 'ubuntu-latest' }} sha: ${{ github.sha }} cmake-preset: ${{ matrix.cmake-preset }} - test-scope: ${{ matrix.test-scope }} + test-scope: ${{ matrix.test-config.test-scope }} cleanup: needs: diff --git a/.github/workflows/ci-gh-gpu-build-and-test.yml b/.github/workflows/ci-gh-gpu-build-and-test.yml index 2fac53d919..2f2e6d50ad 100644 --- a/.github/workflows/ci-gh-gpu-build-and-test.yml +++ b/.github/workflows/ci-gh-gpu-build-and-test.yml @@ -5,6 +5,10 @@ concurrency: cancel-in-progress: true on: + workflow_run: + workflows: ["Lint legate.core on GH"] + types: + - completed push: branches: - "pull-request/[0-9]+" @@ -16,6 +20,7 @@ jobs: fail-fast: false matrix: cmake-preset: [debug-gcc, debug-sanitizer-gcc] + #if: ${{ github.event.workflow_run.conclusion == 'success' && github.repository_owner == 'nv-legate' }} if: ${{ github.repository_owner == 'nv-legate' }} uses: ./.github/workflows/gh-build.yml @@ -34,11 +39,9 @@ jobs: fail-fast: false matrix: cmake-preset: [debug-gcc, debug-sanitizer-gcc] - include: - - { name: Pytest Unit Tests, test-scope: unit, runner: linux-amd64-gpu-v100-latest-1 } - - { name: Pytest Unit Tests, test-scope: unit, runner: linux-amd64-2gpu } - - { name: mypy, test-scope: mypy, runner: 'linux-amd64-gpu-v100-latest-1' } - name: ${{ matrix.name }} + runner: [linux-amd64-gpu-v100-latest-1, linux-amd64-2gpu ] + test-config: [{ name: Pytest Unit Tests, test-scope: unit }] + name: ${{ matrix.test-config.name }} if: ${{ github.repository_owner == 'nv-legate' }} uses: ./.github/workflows/gh-test.yml @@ -48,7 +51,7 @@ jobs: runs-on: ${{ matrix.runner }} sha: ${{ github.sha }} cmake-preset: ${{ matrix.cmake-preset }} - test-scope: ${{ matrix.test-scope }} + test-scope: ${{ matrix.test-config.test-scope }} cleanup: needs: diff --git a/.github/workflows/ci-gh-lint.yml b/.github/workflows/ci-gh-lint.yml new file mode 100644 index 0000000000..1f3a755c17 --- /dev/null +++ b/.github/workflows/ci-gh-lint.yml @@ -0,0 +1,41 @@ +name: Lint legate.core on GH + +on: + push: + branches: + - "pull-request/[0-9]+" + - "*branch-*" + +jobs: + mypy: + runs-on: 'ubuntu-latest' + strategy: + fail-fast: true + if: ${{ github.repository_owner == 'nv-legate' }} + permissions: + contents: read # This is required for actions/checkout + + steps: + - name: List machine info + run: uname -a + + - name: List python version + run: python3 --version + + - name: Bootstrap pip + run: | + curl -O https://bootstrap.pypa.io/get-pip.py + python3 get-pip.py + + - name: Install requisite python packages + run: | + python3 -m pip install -U pip setuptools wheel + python3 -m pip install -U numpy mypy traitlets + + - name: Checkout ${{ github.event.repository.name }} (= this repo) + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Run MyPy on the repository + run: python3 -m mypy ./legate diff --git a/.github/workflows/gh-test.yml b/.github/workflows/gh-test.yml index 201fb3f7b2..ce6851877e 100644 --- a/.github/workflows/gh-test.yml +++ b/.github/workflows/gh-test.yml @@ -22,6 +22,9 @@ on: required: true type: string +env: + LEGATE_CORE_CMAKE_PRESET: ${{ inputs.cmake-preset }} + jobs: test: if: github.repository_owner == 'nv-legate' diff --git a/continuous_integration/home/coder/.local/bin/build-legate-conda b/continuous_integration/home/coder/.local/bin/build-legate-conda index 2ec7d0db6e..cf7530ad3c 100755 --- a/continuous_integration/home/coder/.local/bin/build-legate-conda +++ b/continuous_integration/home/coder/.local/bin/build-legate-conda @@ -59,7 +59,7 @@ EOF # Install legate_core C++ libs tar -C "\$PREFIX" --exclude="*.a" --strip-components=1 -xf /tmp/out/legate_core-*-Linux.tar.gz; sed -E -i "s@$HOME/\.conda/envs/legate@\$PREFIX@g" "\$PREFIX/share/Legion/cmake/LegionConfigCommon.cmake"; -sed -E -i "s@$HOME/legate/build/_CPack_Packages/Linux/TGZ/legate_core-(.*)-Linux@\$PREFIX@g" "\$SP_DIR/legion_canonical_cffi.py"; +sed -E -i "s@$HOME/legate/build/${LEGATE_CORE_CMAKE_PRESET}/_CPack_Packages/Linux/TGZ/legate_core-(.*)-Linux@\$PREFIX@g" "\$SP_DIR/legion_canonical_cffi.py"; # Install legate_core Python wheel pip install --no-deps --root / --prefix "\$PREFIX" /tmp/out/legate_core-*.whl; diff --git a/continuous_integration/home/coder/.local/bin/run-test-or-analysis b/continuous_integration/home/coder/.local/bin/run-test-or-analysis index 155288f613..47d44784ae 100755 --- a/continuous_integration/home/coder/.local/bin/run-test-or-analysis +++ b/continuous_integration/home/coder/.local/bin/run-test-or-analysis @@ -17,9 +17,44 @@ run_test_or_analysis() { case "$1" in "unit") echo "Executing unit tests..." + if [[ ${LEGATE_CORE_CMAKE_PRESET} == *-sanitizer-* ]]; then + echo "============================================================================" + echo "x ------------------------------------------------------------------------ x" + echo "Cannot yet handle running Python tests using Address Sanitizer, bailing!" + echo "x ------------------------------------------------------------------------ x" + echo "============================================================================" + return 0 + fi + mamba install -y -n "${DEFAULT_CONDA_ENV:-legate}" -c conda-forge pytest pytest-mock ipython jupyter_client cd ~/legate/tests/unit - pytest + # Leaving this here in case we ever decide to enable this. If we don't do + # this, we get + # ImportError: could not load libasan.6.so + # when loading the Cython extension libs. + if [[ ${LEGATE_CORE_CMAKE_PRESET} == *-sanitizer-gcc ]]; then + # Trying to reproduce this on macOS? See + # https://stackoverflow.com/questions/47619097/address-sanitizing-boost-python-modules + # + # The TL;DR is, however: + # + # asan_lib='/Library/Developer/CommandLineTools/usr/lib/clang/15.0.0/lib/darwin/libclang_rt.asan_osx_dynamic.dylib' + # + # py_paths='/path/to/legate.core.internal' (optional, only if you are using a venv) + # + # DYLD_INSERT_LIBRARIES=$asan_lib PYTHONPATH=$py_paths + # /opt/homebrew/Cellar/python@3.11/3.11.5/Frameworks/Python.framework/Versions/3.11/Resources/Python.app/Contents/MacOS/Python + # /path/to/your/file.py + GCC_ASAN_PRELOAD=$(gcc -print-file-name=libasan.so) + LD_PRELOAD="${GCC_ASAN_PRELOAD}" pytest + elif [[ ${LEGATE_CORE_CMAKE_PRESET} == *-sanitizer-clang ]]; then + echo "Must properly implement Address sanitizer lib injection for Clang-based presets!" + echo "See impls for GCC above" + return 1 + else + pytest + fi + ;; "mypy") echo "Executing mypy..." @@ -36,4 +71,4 @@ run_test_or_analysis() { return 0 } -(run_test_or_analysis "$@"); \ No newline at end of file +(run_test_or_analysis "$@"); From 513256409a4d7ca0b11ced320e97f2534f9795fa Mon Sep 17 00:00:00 2001 From: Parag Kulkarni Date: Fri, 6 Oct 2023 20:40:21 +0530 Subject: [PATCH 0251/1425] Add Release Preset config with TOT. (#115) --- .../workflows/ci-gh-cpu-build-and-test.yml | 4 +- .../workflows/ci-gh-gpu-build-and-test.yml | 4 +- CMakeLists.txt | 14 +++- CMakePresets.json | 3 +- cmake/presets/release.json | 65 +++++++++++++++++++ 5 files changed, 83 insertions(+), 7 deletions(-) create mode 100644 cmake/presets/release.json diff --git a/.github/workflows/ci-gh-cpu-build-and-test.yml b/.github/workflows/ci-gh-cpu-build-and-test.yml index 1afa54737c..4bae6859e5 100644 --- a/.github/workflows/ci-gh-cpu-build-and-test.yml +++ b/.github/workflows/ci-gh-cpu-build-and-test.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - cmake-preset: [debug-gcc, debug-sanitizer-gcc] + cmake-preset: [debug-gcc, debug-sanitizer-gcc, release-gcc] #if: ${{ github.event.workflow_run.conclusion == 'success' && github.repository_owner == 'nv-legate' }} if: ${{ github.repository_owner == 'nv-legate' }} uses: @@ -38,7 +38,7 @@ jobs: strategy: fail-fast: false matrix: - cmake-preset: [debug-gcc, debug-sanitizer-gcc] + cmake-preset: [debug-gcc, debug-sanitizer-gcc, release-gcc] test-config: [{ name: Pytest Unit Tests, test-scope: unit }] name: ${{ matrix.test-config.name }} if: ${{ github.repository_owner == 'nv-legate' }} diff --git a/.github/workflows/ci-gh-gpu-build-and-test.yml b/.github/workflows/ci-gh-gpu-build-and-test.yml index 2f2e6d50ad..febfbaf0cb 100644 --- a/.github/workflows/ci-gh-gpu-build-and-test.yml +++ b/.github/workflows/ci-gh-gpu-build-and-test.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - cmake-preset: [debug-gcc, debug-sanitizer-gcc] + cmake-preset: [debug-gcc, debug-sanitizer-gcc, release-gcc] #if: ${{ github.event.workflow_run.conclusion == 'success' && github.repository_owner == 'nv-legate' }} if: ${{ github.repository_owner == 'nv-legate' }} uses: @@ -38,7 +38,7 @@ jobs: strategy: fail-fast: false matrix: - cmake-preset: [debug-gcc, debug-sanitizer-gcc] + cmake-preset: [debug-gcc, debug-sanitizer-gcc, release-gcc] runner: [linux-amd64-gpu-v100-latest-1, linux-amd64-2gpu ] test-config: [{ name: Pytest Unit Tests, test-scope: unit }] name: ${{ matrix.test-config.name }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 93a88c39a1..d1df076af5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,8 +68,18 @@ legate_include_rapids() # -O3 to -O2 and removes -DNDEBUG # set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g") # set(CMAKE_CUDA_FLAGS_DEBUG "-O0 -g") -set(CMAKE_CXX_FLAGS_RELEASE "-O2") -set(CMAKE_CUDA_FLAGS_RELEASE "-O2") +# set(CMAKE_CXX_FLAGS_RELEASE "-O2") +if (NOT CMAKE_CXX_FLAGS_RELEASE) + # not being set by preset + set(CMAKE_CXX_FLAGS_RELEASE "-O2") +endif() + +# set(CMAKE_CUDA_FLAGS_RELEASE "-O2") +if (NOT CMAKE_CUDA_FLAGS_RELEASE) + # not being set by preset + set(CMAKE_CUDA_FLAGS_RELEASE "-O2") +endif() + set(CMAKE_CXX_FLAGS_MINSIZEREL "-Os") set(CMAKE_CUDA_FLAGS_MINSIZEREL "-Os") set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g") diff --git a/CMakePresets.json b/CMakePresets.json index 5df219c5c2..712b115e0a 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -10,6 +10,7 @@ "include": [ "cmake/presets/base.json", "cmake/presets/debug.json", - "cmake/presets/debug_sanitizer.json" + "cmake/presets/debug_sanitizer.json", + "cmake/presets/release.json" ] } diff --git a/cmake/presets/release.json b/cmake/presets/release.json new file mode 100644 index 0000000000..b2d28603ab --- /dev/null +++ b/cmake/presets/release.json @@ -0,0 +1,65 @@ +{ + "version": 4, + "include": [ + "base.json" + ], + "configurePresets": [ + { + "inherits": "base", + "hidden": true, + "name": "base-release", + "environment": { + "LEGATE_RELEASE_PRESET_CMAKE_CXX_FLAGS": "-O2", + "LEGATE_RELEASE_PRESET_CMAKE_CUDA_FLAGS": "--compiler-options='$env{LEGATE_RELEASE_PRESET_CMAKE_CXX_FLAGS}'", + "LEGATE_RELEASE_PRESET_LEGATE_CORE_CXX_FLAGS": "-fstack-protector-strong", + "LEGATE_RELEASE_PRESET_LEGATE_CORE_CUDA_FLAGS": "--compiler-options='$env{LEGATE_RELEASE_PRESET_LEGATE_CORE_CXX_FLAGS}'" + }, + "cacheVariables": { + "CMAKE_BUILD_TYPE": { + "type": "STRING", + "value": "Release" + }, + "CMAKE_CXX_FLAGS_RELEASE": { + "type": "STRING", + "value": "$env{LEGATE_RELEASE_PRESET_CMAKE_CXX_FLAGS}" + }, + "legate_core_CXX_FLAGS": { + "type": "STRING", + "value": "$env{LEGATE_RELEASE_PRESET_LEGATE_CORE_CXX_FLAGS}" + }, + "CMAKE_CUDA_FLAGS_RELEASE": { + "type": "STRING", + "value": "$env{LEGATE_RELEASE_PRESET_CMAKE_CUDA_FLAGS}" + }, + "legate_core_CUDA_FLAGS": { + "type": "STRING", + "value": "$env{LEGATE_RELEASE_PRESET_LEGATE_CORE_CUDA_FLAGS}" + } + } + }, + { + "inherits": "base-release", + "name": "release-clang", + "displayName": "Release Clang Build", + "description": "Release Clang Build" + }, + { + "inherits": "base-release", + "name": "release-gcc", + "displayName": "Release GCC Build", + "description": "Release GCC Build" + } + ], + "buildPresets": [ + { + "inherits": "base", + "name": "release-clang", + "configurePreset": "release-clang" + }, + { + "inherits": "base", + "name": "release-gcc", + "configurePreset": "release-gcc" + } + ] +} \ No newline at end of file From 849930b5fcb5b3bc89c3d276af2857ac64377d3e Mon Sep 17 00:00:00 2001 From: Parag Kulkarni Date: Mon, 9 Oct 2023 17:13:32 +0530 Subject: [PATCH 0252/1425] Sync to legate.core and remove the WAR (#118) --- .github/workflows/gh-build.yml | 2 +- continuous_integration/build-docker-image | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index 6cf5cced6b..a59d3a5e9b 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -35,7 +35,7 @@ jobs: contents: read # This is required for actions/checkout packages: write # This is required to push docker image to ghcr.io - runs-on: ubuntu-latest + runs-on: ${{ inputs.runs-on }} steps: - name: Checkout ${{ inputs.repos-name }} (= this repo) diff --git a/continuous_integration/build-docker-image b/continuous_integration/build-docker-image index 3d5d1cac1b..0c5d884336 100755 --- a/continuous_integration/build-docker-image +++ b/continuous_integration/build-docker-image @@ -27,6 +27,8 @@ set -e # Avoid build errors due to a missing .creds folder mkdir -p "$SOURCE_DIR/.creds" +export DOCKER_BUILDKIT=1 + docker build \ --build-arg BASE_IMAGE="$BASE_IMAGE" \ --build-arg AWS_SESSION_TOKEN="$AWS_SESSION_TOKEN" \ From db4f8a92b65c4ce00b6f038e7656d91f39d4c68f Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Mon, 9 Oct 2023 15:26:34 -0700 Subject: [PATCH 0253/1425] Several fixes for bugs uncovered by the binding work (#119) * Shape for scalar stores * Fix a bug in the delinearizing transform * Handle inout scalar stores correctly * Stop caching volume for storage * Broadcast constraints with no axis argument * Put back debug flags in the legacy debug build * Handle scalar outputs correctly in task parallelization * Stop using deprecated enums * Address review comments * Missing review comment * Fix for copy elision --- CMakeLists.txt | 12 +- src/core/data/accessors.h | 9 +- src/core/data/accessors.inl | 24 ++-- src/core/data/detail/logical_array.cc | 9 +- src/core/data/detail/logical_store.cc | 19 ++-- src/core/data/detail/logical_store.h | 1 - src/core/data/detail/transform.cc | 4 +- src/core/mapping/detail/base_mapper.cc | 14 ++- src/core/operation/detail/copy.cc | 5 +- src/core/operation/detail/fill.cc | 6 +- src/core/operation/detail/gather.cc | 6 +- src/core/operation/detail/reduce.cc | 8 +- src/core/operation/detail/scatter.cc | 6 +- src/core/operation/detail/scatter_gather.cc | 8 +- src/core/operation/detail/task.cc | 17 ++- src/core/partitioning/constraint.cc | 5 + src/core/partitioning/constraint.h | 16 ++- src/core/partitioning/detail/constraint.cc | 16 ++- src/core/partitioning/detail/constraint.h | 6 +- .../partitioning/detail/constraint_solver.cc | 23 ++-- .../partitioning/detail/constraint_solver.h | 7 +- src/core/runtime/detail/runtime.cc | 8 +- src/core/runtime/detail/runtime.h | 3 +- src/core/runtime/runtime.cc | 4 +- src/core/runtime/runtime.h | 3 +- src/core/utilities/tuple.h | 3 +- src/core/utilities/tuple.inl | 6 + src/core/utilities/typedefs.h | 9 +- .../cpp/integration/broadcast_constraints.cc | 27 +++-- tests/cpp/integration/delinearize.cc | 105 ++++++++++++++++++ tests/cpp/integration/inout.cc | 95 ++++++++++++++-- tests/cpp/integration/scalar_out.cc | 80 +++++++++++++ tests/cpp/unit/logical_store.cc | 21 ++++ 33 files changed, 472 insertions(+), 113 deletions(-) create mode 100644 tests/cpp/integration/delinearize.cc create mode 100644 tests/cpp/integration/scalar_out.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index d1df076af5..214014edd4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -66,20 +66,20 @@ legate_include_rapids() # For now we want the optimization flags to match on both normal make and cmake # builds so we override the cmake defaults here for release, this changes # -O3 to -O2 and removes -DNDEBUG -# set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g") -# set(CMAKE_CUDA_FLAGS_DEBUG "-O0 -g") -# set(CMAKE_CXX_FLAGS_RELEASE "-O2") +if(NOT DEFINED CMAKE_CXX_FLAGS_DEBUG) + set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g") +endif() +if(NOT DEFINED CMAKE_CUDA_FLAGS_DEBUG) + set(CMAKE_CUDA_FLAGS_DEBUG "-O0 -g") +endif() if (NOT CMAKE_CXX_FLAGS_RELEASE) # not being set by preset set(CMAKE_CXX_FLAGS_RELEASE "-O2") endif() - -# set(CMAKE_CUDA_FLAGS_RELEASE "-O2") if (NOT CMAKE_CUDA_FLAGS_RELEASE) # not being set by preset set(CMAKE_CUDA_FLAGS_RELEASE "-O2") endif() - set(CMAKE_CXX_FLAGS_MINSIZEREL "-Os") set(CMAKE_CUDA_FLAGS_MINSIZEREL "-Os") set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g") diff --git a/src/core/data/accessors.h b/src/core/data/accessors.h index 8491b71764..19bf4cd33b 100644 --- a/src/core/data/accessors.h +++ b/src/core/data/accessors.h @@ -27,7 +27,7 @@ template class ListArrayAccessor; template -class ListArrayAccessor { +class ListArrayAccessor { public: ListArrayAccessor(ListArray array); virtual ~ListArrayAccessor(); @@ -41,7 +41,7 @@ class ListArrayAccessor { }; template -class ListArrayAccessor { +class ListArrayAccessor { public: ListArrayAccessor(ListArray array); virtual ~ListArrayAccessor(); @@ -63,7 +63,7 @@ template class StringArrayAccessor; template <> -struct StringArrayAccessor : public ListArrayAccessor { +struct StringArrayAccessor : public ListArrayAccessor { StringArrayAccessor(StringArray array); using ListArrayAccessor::operator[]; @@ -71,7 +71,8 @@ struct StringArrayAccessor : public ListArrayAccessor -struct StringArrayAccessor : public ListArrayAccessor { +struct StringArrayAccessor + : public ListArrayAccessor { StringArrayAccessor(StringArray array); using ListArrayAccessor::insert; diff --git a/src/core/data/accessors.inl b/src/core/data/accessors.inl index fd0f175f03..6364b1e955 100644 --- a/src/core/data/accessors.inl +++ b/src/core/data/accessors.inl @@ -18,26 +18,26 @@ namespace legate { template -ListArrayAccessor::ListArrayAccessor(ListArray array) +ListArrayAccessor::ListArrayAccessor(ListArray array) { desc_acc_ = array.descriptor().data().read_accessor, 1>(); vardata_acc_ = array.vardata().data().read_accessor(); } template -ListArrayAccessor::~ListArrayAccessor() +ListArrayAccessor::~ListArrayAccessor() { } template -Span ListArrayAccessor::operator[](const Point<1>& p) +Span ListArrayAccessor::operator[](const Point<1>& p) { auto& desc = desc_acc_[p]; return Span(vardata_acc_.ptr(desc.lo), desc.volume()); } template -ListArrayAccessor::ListArrayAccessor(ListArray array) +ListArrayAccessor::ListArrayAccessor(ListArray array) { desc_shape_ = array.shape<1>(); desc_acc_ = array.descriptor().data().write_accessor, 1>(); @@ -45,7 +45,7 @@ ListArrayAccessor::ListArrayAccessor(ListArray array) } template -ListArrayAccessor::~ListArrayAccessor() +ListArrayAccessor::~ListArrayAccessor() { if (desc_shape_.empty()) { vardata_store_.bind_empty_data(); @@ -71,43 +71,43 @@ ListArrayAccessor::~ListArrayAccessor() } template -void ListArrayAccessor::insert(const legate::Span& value) +void ListArrayAccessor::insert(const legate::Span& value) { check_overflow(); values_.emplace_back(value.begin(), value.end()); } template -void ListArrayAccessor::check_overflow() +void ListArrayAccessor::check_overflow() { if (values_.size() >= desc_shape_.volume()) { throw std::out_of_range("No space left in the array"); } } -StringArrayAccessor::StringArrayAccessor(StringArray array) +StringArrayAccessor::StringArrayAccessor(StringArray array) : ListArrayAccessor(array.as_list_array()) { } -std::string_view StringArrayAccessor::operator[](const Point<1>& p) +std::string_view StringArrayAccessor::operator[](const Point<1>& p) { auto span = ListArrayAccessor::operator[](p); return std::string_view(reinterpret_cast(span.ptr()), span.size()); } -StringArrayAccessor::StringArrayAccessor(StringArray array) +StringArrayAccessor::StringArrayAccessor(StringArray array) : ListArrayAccessor(array.as_list_array()) { } -void StringArrayAccessor::insert(const std::string& value) +void StringArrayAccessor::insert(const std::string& value) { ListArrayAccessor::insert( legate::Span(reinterpret_cast(value.data()), value.size())); } -void StringArrayAccessor::insert(const std::string_view& value) +void StringArrayAccessor::insert(const std::string_view& value) { ListArrayAccessor::insert( legate::Span(reinterpret_cast(value.data()), value.size())); diff --git a/src/core/data/detail/logical_array.cc b/src/core/data/detail/logical_array.cc index 305323570c..3dfd8a69ef 100644 --- a/src/core/data/detail/logical_array.cc +++ b/src/core/data/detail/logical_array.cc @@ -151,7 +151,7 @@ std::unique_ptr BaseLogicalArray::to_launcher_arg( std::unique_ptr null_mask_arg = nullptr; if (nullable()) { auto null_redop = - privilege == REDUCE ? bool_()->find_reduction_operator(ReductionOpKind::MUL) : -1; + privilege == LEGION_REDUCE ? bool_()->find_reduction_operator(ReductionOpKind::MUL) : -1; null_mask_arg = null_mask_->to_launcher_arg( mapping.at(null_mask_), strategy, launch_domain, privilege, null_redop); } @@ -274,7 +274,8 @@ std::unique_ptr ListLogicalArray::to_launcher_arg( Legion::PrivilegeMode privilege, int32_t redop) const { - auto desc_priv = (READ_ONLY == privilege || vardata_->unbound()) ? privilege : READ_WRITE; + auto desc_priv = + (LEGION_READ_ONLY == privilege || vardata_->unbound()) ? privilege : LEGION_READ_WRITE; auto descriptor_arg = descriptor_->to_launcher_arg(mapping, strategy, launch_domain, desc_priv, redop); auto vardata_arg = vardata_->to_launcher_arg(mapping, strategy, launch_domain, privilege, redop); @@ -285,7 +286,7 @@ std::unique_ptr ListLogicalArray::to_launcher_arg( std::unique_ptr ListLogicalArray::to_launcher_arg_for_fixup( const Domain& launch_domain, Legion::PrivilegeMode privilege) const { - auto descriptor_arg = descriptor_->to_launcher_arg_for_fixup(launch_domain, READ_WRITE); + auto descriptor_arg = descriptor_->to_launcher_arg_for_fixup(launch_domain, LEGION_READ_WRITE); auto vardata_arg = vardata_->to_launcher_arg_for_fixup(launch_domain, privilege); return std::make_unique(type(), std::move(descriptor_arg), std::move(vardata_arg)); @@ -424,7 +425,7 @@ std::unique_ptr StructLogicalArray::to_launcher_arg( std::unique_ptr null_mask_arg = nullptr; if (nullable()) { auto null_redop = - privilege == REDUCE ? bool_()->find_reduction_operator(ReductionOpKind::MUL) : -1; + privilege == LEGION_REDUCE ? bool_()->find_reduction_operator(ReductionOpKind::MUL) : -1; null_mask_arg = null_mask_->to_launcher_arg( mapping.at(null_mask_), strategy, launch_domain, privilege, null_redop); } diff --git a/src/core/data/detail/logical_store.cc b/src/core/data/detail/logical_store.cc index 092bafe97e..ba47c262f2 100644 --- a/src/core/data/detail/logical_store.cc +++ b/src/core/data/detail/logical_store.cc @@ -49,11 +49,10 @@ Storage::Storage(const Shape& extents, std::shared_ptr type, bool optimize : storage_id_(Runtime::get_runtime()->get_unique_storage_id()), dim_(extents.size()), extents_(extents), - volume_(extents.volume()), type_(std::move(type)), offsets_(dim_, 0) { - if (optimize_scalar && volume_ == 1) kind_ = Kind::FUTURE; + if (optimize_scalar && extents_.volume() == 1) kind_ = Kind::FUTURE; if (LegateDefined(LEGATE_USE_DEBUG)) { log_legate.debug() << "Create " << to_string(); } } @@ -61,7 +60,6 @@ Storage::Storage(const Shape& extents, std::shared_ptr type, const Legion: : storage_id_(Runtime::get_runtime()->get_unique_storage_id()), dim_(extents.size()), extents_(extents), - volume_(extents.volume()), type_(std::move(type)), kind_(Kind::FUTURE), future_(future), @@ -78,7 +76,6 @@ Storage::Storage(Shape&& extents, : storage_id_(Runtime::get_runtime()->get_unique_storage_id()), dim_(extents.size()), extents_(std::move(extents)), - volume_(extents_.volume()), type_(std::move(type)), level_(parent->level() + 1), parent_(std::move(parent)), @@ -608,7 +605,7 @@ std::shared_ptr LogicalStore::find_or_create_key_partition( key_partition_->satisfies_restrictions(restrictions)) return key_partition_; - if (has_scalar_storage()) { return create_no_partition(); } + if (has_scalar_storage() || volume() == 0) { return create_no_partition(); } Partition* storage_part = nullptr; if (transform_->is_convertible()) @@ -685,9 +682,9 @@ std::unique_ptr LogicalStore::to_launcher_arg(const Variable* variab int32_t redop) { if (has_scalar_storage()) { - if (!launch_domain.is_valid() && REDUCE == privilege) { privilege = READ_WRITE; } - auto read_only = privilege == READ_ONLY; - auto has_storage = privilege == READ_ONLY || privilege == READ_WRITE; + if (!launch_domain.is_valid() && LEGION_REDUCE == privilege) { privilege = LEGION_READ_WRITE; } + auto read_only = privilege == LEGION_READ_ONLY; + auto has_storage = get_future().valid() && privilege != LEGION_REDUCE; return std::make_unique(this, read_only, has_storage, redop); } else if (unbound()) { return std::make_unique(this, strategy.find_field_space(variable)); @@ -698,10 +695,10 @@ std::unique_ptr LogicalStore::to_launcher_arg(const Variable* variab proj_info->is_key = strategy.is_key_partition(variable); proj_info->redop = redop; - if (privilege == REDUCE && store_partition->is_disjoint_for(launch_domain)) { - privilege = READ_WRITE; + if (privilege == LEGION_REDUCE && store_partition->is_disjoint_for(launch_domain)) { + privilege = LEGION_READ_WRITE; } - if (privilege == WRITE_ONLY || privilege == READ_WRITE) { + if (privilege == LEGION_WRITE_ONLY || privilege == LEGION_READ_WRITE) { set_key_partition(variable->operation()->machine(), partition.get()); } return std::make_unique(this, privilege, std::move(proj_info)); diff --git a/src/core/data/detail/logical_store.h b/src/core/data/detail/logical_store.h index f98cfdce64..3bbd963088 100644 --- a/src/core/data/detail/logical_store.h +++ b/src/core/data/detail/logical_store.h @@ -101,7 +101,6 @@ class Storage : public std::enable_shared_from_this { bool destroyed_out_of_order_{false}; int32_t dim_{-1}; Shape extents_; - size_t volume_; std::shared_ptr type_{nullptr}; Kind kind_{Kind::REGION_FIELD}; diff --git a/src/core/data/detail/transform.cc b/src/core/data/detail/transform.cc index 4c23baf255..5d1233e599 100644 --- a/src/core/data/detail/transform.cc +++ b/src/core/data/detail/transform.cc @@ -802,8 +802,8 @@ std::unique_ptr Delinearize::invert(const Partition* partition) const new_offsets.remove_inplace(dim_ + 1); } - new_tile_shape[dim_] *= strides_[dim_]; - new_offsets[dim_] *= strides_[dim_]; + new_tile_shape[dim_] *= strides_[0]; + new_offsets[dim_] *= strides_[0]; return create_tiling( std::move(new_tile_shape), std::move(new_color_shape), std::move(new_offsets)); diff --git a/src/core/mapping/detail/base_mapper.cc b/src/core/mapping/detail/base_mapper.cc index c88f80bd00..dfb201d78c 100644 --- a/src/core/mapping/detail/base_mapper.cc +++ b/src/core/mapping/detail/base_mapper.cc @@ -314,7 +314,7 @@ void BaseMapper::map_task(const Legion::Mapping::MapperContext ctx, std::vector> for_futures; std::vector> for_unbound_stores; std::vector> for_stores; - std::set mapped_futures; + std::map mapped_futures; std::set mapped_regions; for (auto& client_mapping : client_mappings) { @@ -323,12 +323,14 @@ void BaseMapper::map_task(const Legion::Mapping::MapperContext ctx, auto fut_idx = mapping->store()->future_index(); // Only need to map Future-backed Stores corresponding to inputs (i.e. one of task.futures) if (fut_idx >= task.futures.size()) continue; - if (mapped_futures.count(fut_idx) > 0) { + auto finder = mapped_futures.find(fut_idx); + if (finder != mapped_futures.end() && finder->second->policy != mapping->policy) { logger.error("Mapper %s returned duplicate store mappings", get_mapper_name()); LEGATE_ABORT; + } else { + mapped_futures.insert({fut_idx, mapping}); + for_futures.emplace_back(client_mapping.release()); } - mapped_futures.insert(fut_idx); - for_futures.emplace_back(client_mapping.release()); } else if (mapping->for_unbound_store()) { mapped_regions.insert(mapping->store()->unique_region_field_id()); for_unbound_stores.emplace_back(client_mapping.release()); @@ -369,7 +371,7 @@ void BaseMapper::map_task(const Legion::Mapping::MapperContext ctx, // task.futures) if (fut_idx >= task.futures.size()) continue; if (mapped_futures.find(fut_idx) != mapped_futures.end()) continue; - mapped_futures.insert(fut_idx); + mapped_futures.insert({fut_idx, mapping.get()}); for_futures.push_back(std::move(mapping)); } else { auto key = store->unique_region_field_id(); @@ -390,7 +392,7 @@ void BaseMapper::map_task(const Legion::Mapping::MapperContext ctx, assert(mapped_futures.size() <= task.futures.size()); // The launching code should be packing all Store-backing Futures first. if (mapped_futures.size() > 0) { - uint32_t max_mapped_fut = *mapped_futures.rbegin(); + uint32_t max_mapped_fut = mapped_futures.rbegin()->first; assert(mapped_futures.size() == max_mapped_fut + 1); } } diff --git a/src/core/operation/detail/copy.cc b/src/core/operation/detail/copy.cc index 679bc6f8d9..6c981ff78a 100644 --- a/src/core/operation/detail/copy.cc +++ b/src/core/operation/detail/copy.cc @@ -90,8 +90,9 @@ void Copy::launch(Strategy* p_strategy) void Copy::add_to_solver(ConstraintSolver& solver) { solver.add_constraint(std::move(constraint_)); - solver.add_partition_symbol(target_.variable); - solver.add_partition_symbol(source_.variable); + solver.add_partition_symbol(target_.variable, IsOutput::Y); + if (target_.store->has_scalar_storage()) solver.add_constraint(broadcast(target_.variable)); + solver.add_partition_symbol(source_.variable, IsOutput::N); } std::string Copy::to_string() const { return "Copy:" + std::to_string(unique_id_); } diff --git a/src/core/operation/detail/fill.cc b/src/core/operation/detail/fill.cc index 62ed2d5d64..1e8626b618 100644 --- a/src/core/operation/detail/fill.cc +++ b/src/core/operation/detail/fill.cc @@ -65,6 +65,10 @@ void Fill::launch(Strategy* strategy) std::string Fill::to_string() const { return "Fill:" + std::to_string(unique_id_); } -void Fill::add_to_solver(ConstraintSolver& solver) { solver.add_partition_symbol(lhs_var_); } +void Fill::add_to_solver(ConstraintSolver& solver) +{ + solver.add_partition_symbol(lhs_var_, IsOutput::Y); + if (lhs_->has_scalar_storage()) solver.add_constraint(broadcast(lhs_var_)); +} } // namespace legate::detail diff --git a/src/core/operation/detail/gather.cc b/src/core/operation/detail/gather.cc index 8a53c716ba..979c465151 100644 --- a/src/core/operation/detail/gather.cc +++ b/src/core/operation/detail/gather.cc @@ -92,9 +92,9 @@ void Gather::launch(Strategy* p_strategy) void Gather::add_to_solver(ConstraintSolver& solver) { solver.add_constraint(std::move(constraint_)); - solver.add_partition_symbol(target_.variable); - solver.add_partition_symbol(source_.variable); - solver.add_partition_symbol(source_indirect_.variable); + solver.add_partition_symbol(target_.variable, IsOutput::Y); + solver.add_partition_symbol(source_.variable, IsOutput::N); + solver.add_partition_symbol(source_indirect_.variable, IsOutput::N); } std::string Gather::to_string() const { return "Gather:" + std::to_string(unique_id_); } diff --git a/src/core/operation/detail/reduce.cc b/src/core/operation/detail/reduce.cc index f4a4b1be3e..fc9a4090e5 100644 --- a/src/core/operation/detail/reduce.cc +++ b/src/core/operation/detail/reduce.cc @@ -76,13 +76,13 @@ void Reduce::launch(Strategy* p_strategy) for (auto& proj_fn : proj_fns) { auto proj_info = input_partition->create_projection_info(launch_domain, proj_fn); launcher.add_input(to_array_arg( - std::make_unique(input_.get(), READ_ONLY, std::move(proj_info)))); + std::make_unique(input_.get(), LEGION_READ_ONLY, std::move(proj_info)))); } } else { // otherwise we just add an entire input region to the task auto proj_info = input_partition->create_projection_info(launch_domain); launcher.add_input(to_array_arg( - std::make_unique(input_.get(), READ_ONLY, std::move(proj_info)))); + std::make_unique(input_.get(), LEGION_READ_ONLY, std::move(proj_info)))); } // calculating #of sub-tasks in the reduction task @@ -122,8 +122,8 @@ void Reduce::validate() {} void Reduce::add_to_solver(detail::ConstraintSolver& solver) { - solver.add_partition_symbol(output_part_, true); - solver.add_partition_symbol(input_part_); + solver.add_partition_symbol(output_part_, IsOutput::Y); + solver.add_partition_symbol(input_part_, IsOutput::N); } std::string Reduce::to_string() const { return "Reduce:" + std::to_string(unique_id_); } diff --git a/src/core/operation/detail/scatter.cc b/src/core/operation/detail/scatter.cc index 13f020c587..d3d953f580 100644 --- a/src/core/operation/detail/scatter.cc +++ b/src/core/operation/detail/scatter.cc @@ -92,9 +92,9 @@ void Scatter::launch(Strategy* p_strategy) void Scatter::add_to_solver(ConstraintSolver& solver) { solver.add_constraint(std::move(constraint_)); - solver.add_partition_symbol(target_.variable); - solver.add_partition_symbol(target_indirect_.variable); - solver.add_partition_symbol(source_.variable); + solver.add_partition_symbol(target_.variable, IsOutput::Y); + solver.add_partition_symbol(target_indirect_.variable, IsOutput::N); + solver.add_partition_symbol(source_.variable, IsOutput::N); } std::string Scatter::to_string() const { return "Scatter:" + std::to_string(unique_id_); } diff --git a/src/core/operation/detail/scatter_gather.cc b/src/core/operation/detail/scatter_gather.cc index 71860cb47d..da4593ac27 100644 --- a/src/core/operation/detail/scatter_gather.cc +++ b/src/core/operation/detail/scatter_gather.cc @@ -113,10 +113,10 @@ void ScatterGather::launch(Strategy* p_strategy) void ScatterGather::add_to_solver(ConstraintSolver& solver) { solver.add_constraint(std::move(constraint_)); - solver.add_partition_symbol(target_.variable); - solver.add_partition_symbol(target_indirect_.variable); - solver.add_partition_symbol(source_.variable); - solver.add_partition_symbol(source_indirect_.variable); + solver.add_partition_symbol(target_.variable, IsOutput::Y); + solver.add_partition_symbol(target_indirect_.variable, IsOutput::N); + solver.add_partition_symbol(source_.variable, IsOutput::N); + solver.add_partition_symbol(source_indirect_.variable, IsOutput::N); } std::string ScatterGather::to_string() const diff --git a/src/core/operation/detail/task.cc b/src/core/operation/detail/task.cc index e5f304534d..80b0d9a3ce 100644 --- a/src/core/operation/detail/task.cc +++ b/src/core/operation/detail/task.cc @@ -83,15 +83,17 @@ void Task::launch_task(Strategy* p_strategy) auto launch_domain = strategy.launch_domain(this); for (auto& [arr, mapping] : inputs_) { - launcher.add_input(arr->to_launcher_arg(mapping, strategy, launch_domain, READ_ONLY, -1)); + launcher.add_input( + arr->to_launcher_arg(mapping, strategy, launch_domain, LEGION_READ_ONLY, -1)); } for (auto& [arr, mapping] : outputs_) { - launcher.add_output(arr->to_launcher_arg(mapping, strategy, launch_domain, WRITE_ONLY, -1)); + launcher.add_output( + arr->to_launcher_arg(mapping, strategy, launch_domain, LEGION_WRITE_ONLY, -1)); } uint32_t idx = 0; for (auto& [arr, mapping] : reductions_) { launcher.add_reduction( - arr->to_launcher_arg(mapping, strategy, launch_domain, REDUCE, reduction_ops_[idx++])); + arr->to_launcher_arg(mapping, strategy, launch_domain, LEGION_REDUCE, reduction_ops_[idx++])); } // Add by-value scalars @@ -288,13 +290,16 @@ void AutoTask::add_to_solver(detail::ConstraintSolver& solver) { for (auto& constraint : constraints_) solver.add_constraint(std::move(constraint)); for (auto& [_, mapping] : outputs_) { - for (auto& [_, symb] : mapping) solver.add_partition_symbol(symb, true); + for (auto& [store, symb] : mapping) { + solver.add_partition_symbol(symb, IsOutput::Y); + if (store->has_scalar_storage()) solver.add_constraint(broadcast(symb)); + } } for (auto& [_, mapping] : inputs_) { - for (auto& [_, symb] : mapping) solver.add_partition_symbol(symb, false); + for (auto& [_, symb] : mapping) solver.add_partition_symbol(symb, IsOutput::N); } for (auto& [_, mapping] : reductions_) { - for (auto& [_, symb] : mapping) solver.add_partition_symbol(symb, false); + for (auto& [_, symb] : mapping) solver.add_partition_symbol(symb, IsOutput::N); } } diff --git a/src/core/partitioning/constraint.cc b/src/core/partitioning/constraint.cc index e273443f60..465c5e068c 100644 --- a/src/core/partitioning/constraint.cc +++ b/src/core/partitioning/constraint.cc @@ -30,6 +30,11 @@ Constraint align(Variable lhs, Variable rhs) return Constraint(detail::align(lhs.impl(), rhs.impl()).release()); } +Constraint broadcast(Variable variable) +{ + return Constraint{detail::broadcast(variable.impl()).release()}; +} + Constraint broadcast(Variable variable, const tuple& axes) { return Constraint(detail::broadcast(variable.impl(), tuple(axes)).release()); diff --git a/src/core/partitioning/constraint.h b/src/core/partitioning/constraint.h index 206a60d76c..01182f99fa 100644 --- a/src/core/partitioning/constraint.h +++ b/src/core/partitioning/constraint.h @@ -85,12 +85,26 @@ Constraint align(Variable lhs, Variable rhs); /** * @ingroup partitioning - * @brief Creates a broadcast constraint on a variable + * @brief Creates a broadcast constraint on a variable. + * + * This constraint prevents all dimensions of the store from being partitioned. + * + * @param variable Partition symbol to constrain + * + * @return Broadcast constraint + */ +[[nodiscard]] Constraint broadcast(Variable variable); + +/** + * @ingroup partitioning + * @brief Creates a broadcast constraint on a variable. * * @param variable Partition symbol to constrain * @param axes List of dimensions to broadcast * * @return Broadcast constraint + * + * @throw std::invalid_argument If the list of axes is empty */ Constraint broadcast(Variable variable, const tuple& axes); diff --git a/src/core/partitioning/detail/constraint.cc b/src/core/partitioning/detail/constraint.cc index d24d48627e..b1b6d8504c 100644 --- a/src/core/partitioning/detail/constraint.cc +++ b/src/core/partitioning/detail/constraint.cc @@ -79,6 +79,8 @@ std::string Alignment::to_string() const return std::move(ss).str(); } +Broadcast::Broadcast(const Variable* variable) : variable_{variable} {} + Broadcast::Broadcast(const Variable* variable, const tuple& axes) : variable_(variable), axes_(axes) { @@ -91,6 +93,7 @@ void Broadcast::find_partition_symbols(std::vector& partition_s void Broadcast::validate() const { + if (axes_.empty()) return; auto store = variable_->operation()->find_store(variable_); for (auto axis : axes_.data()) { if (axis < 0 || axis >= store->dim()) @@ -102,7 +105,11 @@ void Broadcast::validate() const std::string Broadcast::to_string() const { std::stringstream ss; - ss << "Broadcast(" << variable_->to_string() << ", " << axes_.to_string() << ")"; + if (axes_.empty()) { + ss << "Broadcast(" << variable_->to_string() << ")"; + } else { + ss << "Broadcast(" << variable_->to_string() << ", " << axes_.to_string() << ")"; + } return std::move(ss).str(); } @@ -246,9 +253,14 @@ std::unique_ptr align(const Variable* lhs, const Variable* rhs) return std::make_unique(lhs, rhs); } -std::unique_ptr broadcast(const Variable* variable, const tuple& axes) +[[nodiscard]] std::unique_ptr broadcast(const Variable* variable) +{ + return std::make_unique(variable); +} +std::unique_ptr broadcast(const Variable* variable, const tuple& axes) { + if (axes.empty()) { throw std::invalid_argument("List of axes to broadcast must not be empty"); } return std::make_unique(variable, axes); } diff --git a/src/core/partitioning/detail/constraint.h b/src/core/partitioning/detail/constraint.h index d9108a6360..8319807baa 100644 --- a/src/core/partitioning/detail/constraint.h +++ b/src/core/partitioning/detail/constraint.h @@ -170,6 +170,7 @@ class Alignment : public Constraint { class Broadcast : public Constraint { public: + explicit Broadcast(const Variable* variable); Broadcast(const Variable* variable, const tuple& axes); public: @@ -197,7 +198,8 @@ class Broadcast : public Constraint { private: const Variable* variable_; - tuple axes_; + // Broadcast all dimensions when empty + tuple axes_{}; }; class ImageConstraint : public Constraint { @@ -313,6 +315,8 @@ class BloatConstraint : public Constraint { std::unique_ptr align(const Variable* lhs, const Variable* rhs); +[[nodiscard]] std::unique_ptr broadcast(const Variable* variable); + std::unique_ptr broadcast(const Variable* variable, const tuple& axes); std::unique_ptr image(const Variable* var_function, const Variable* var_range); diff --git a/src/core/partitioning/detail/constraint_solver.cc b/src/core/partitioning/detail/constraint_solver.cc index 4656174a9d..0b6ab296de 100644 --- a/src/core/partitioning/detail/constraint_solver.cc +++ b/src/core/partitioning/detail/constraint_solver.cc @@ -47,6 +47,10 @@ struct UnionFindEntry { end->size += other->size; return self; } + void restrict_all() + { + for (auto& restriction : restrictions.data()) restriction = Restriction::FORBID; + } const Variable* partition_symbol; Restrictions restrictions; @@ -82,11 +86,10 @@ ConstraintSolver::~ConstraintSolver() for (auto equiv_class : equiv_classes_) delete equiv_class; } -void ConstraintSolver::add_partition_symbol(const Variable* partition_symbol, bool is_output) +void ConstraintSolver::add_partition_symbol(const Variable* partition_symbol, IsOutput is_output) { partition_symbols_.insert(partition_symbol); - is_output_.insert({*partition_symbol, is_output}); - is_dependent_.insert({*partition_symbol, false}); + is_output_.insert({*partition_symbol, static_cast(is_output)}); } void ConstraintSolver::add_constraint(std::unique_ptr constraint) @@ -103,16 +106,16 @@ void ConstraintSolver::solve_constraints() const auto& all_symbols = partition_symbols(); entries.reserve(all_symbols.size()); - auto initialize = [&entries, &table](const auto& all_symbols) { + [](auto& entries, auto& table, auto& is_dependent, const auto& all_symbols) { for (auto& symb : all_symbols) { // TODO: partition symbols can be independent of any stores of the operation // (e.g., when a symbol subsumes a union of two other symbols) auto store = symb->operation()->find_store(symb); entries.emplace_back(symb, store.get()); table.insert({*symb, &entries.back()}); + is_dependent.insert({*symb, false}); } - }; - initialize(all_symbols); + }(entries, table, is_dependent_, all_symbols); // Unify equivalence classes based on alignment constraints auto handle_alignment = [&table](const Alignment* alignment) { @@ -144,11 +147,17 @@ void ConstraintSolver::solve_constraints() auto* variable = broadcast->variable(); auto& axes = broadcast->axes(); auto* equiv_class = table.at(*variable); + if (axes.empty()) { + equiv_class->restrict_all(); + return; + } for (uint32_t idx = 0; idx < axes.size(); ++idx) { uint32_t axis = axes[idx]; // TODO: We want to check the axis eagerly and raise an exception // if it is out of bounds - if (axis >= equiv_class->restrictions.size()) continue; + if (LegateDefined(LEGATE_USE_DEBUG)) { + assert(0 <= axis && axis < equiv_class->restrictions.size()); + } equiv_class->restrictions[axes[idx]] = Restriction::FORBID; } }; diff --git a/src/core/partitioning/detail/constraint_solver.h b/src/core/partitioning/detail/constraint_solver.h index fd0f4e5eaf..5de234ce64 100644 --- a/src/core/partitioning/detail/constraint_solver.h +++ b/src/core/partitioning/detail/constraint_solver.h @@ -24,13 +24,18 @@ namespace legate::detail { class Strategy; +enum class IsOutput : bool { + Y = true, + N = false, +}; + struct ConstraintSolver { public: ConstraintSolver(); ~ConstraintSolver(); public: - void add_partition_symbol(const Variable* partition_symbol, bool is_output = false); + void add_partition_symbol(const Variable* partition_symbol, IsOutput is_output); void add_constraint(std::unique_ptr constraint); public: diff --git a/src/core/runtime/detail/runtime.cc b/src/core/runtime/detail/runtime.cc index 1dabe2e555..b9847e03b9 100644 --- a/src/core/runtime/detail/runtime.cc +++ b/src/core/runtime/detail/runtime.cc @@ -444,9 +444,11 @@ std::shared_ptr Runtime::create_store(const Shape& extents, return std::make_shared(std::move(storage)); } -std::shared_ptr Runtime::create_store(const Scalar& scalar) +std::shared_ptr Runtime::create_store(const Scalar& scalar, const Shape& extents) { - Shape extents{1}; + if (extents.volume() != 1) { + throw std::invalid_argument{"Scalar stores must have a shape of volume 1"}; + } auto future = create_future(scalar.data(), scalar.size()); auto storage = std::make_shared(extents, scalar.type(), future); return std::make_shared(std::move(storage)); @@ -574,7 +576,7 @@ std::shared_ptr Runtime::import_region_field(Legion::Logical Legion::PhysicalRegion Runtime::map_region_field(Legion::LogicalRegion region, Legion::FieldID field_id) { - Legion::RegionRequirement req(region, READ_WRITE, EXCLUSIVE, region); + Legion::RegionRequirement req(region, LEGION_READ_WRITE, EXCLUSIVE, region); req.add_field(field_id); auto mapper_id = core_library_->get_mapper_id(); // TODO: We need to pass the metadata about logical store diff --git a/src/core/runtime/detail/runtime.h b/src/core/runtime/detail/runtime.h index 115b645e3d..e7e31ba6c0 100644 --- a/src/core/runtime/detail/runtime.h +++ b/src/core/runtime/detail/runtime.h @@ -146,7 +146,8 @@ class Runtime { std::shared_ptr create_store(const Shape& extents, std::shared_ptr type, bool optimize_scalar = false); - std::shared_ptr create_store(const Scalar& scalar); + [[nodiscard]] std::shared_ptr create_store(const Scalar& scalar, + const Shape& extents); std::shared_ptr create_store(const Shape& extents, std::shared_ptr type, void* buffer, diff --git a/src/core/runtime/runtime.cc b/src/core/runtime/runtime.cc index 88e23309fe..330e6a8c4a 100644 --- a/src/core/runtime/runtime.cc +++ b/src/core/runtime/runtime.cc @@ -183,9 +183,9 @@ LogicalStore Runtime::create_store(const Shape& extents, return LogicalStore(impl_->create_store(extents, type.impl(), optimize_scalar)); } -LogicalStore Runtime::create_store(const Scalar& scalar) +LogicalStore Runtime::create_store(const Scalar& scalar, const Shape& extents) { - return LogicalStore(impl_->create_store(*scalar.impl_)); + return LogicalStore{impl_->create_store(*scalar.impl_, extents)}; } LogicalStore Runtime::create_store(const Shape& extents, diff --git a/src/core/runtime/runtime.h b/src/core/runtime/runtime.h index 241d3366df..943e3a0a1e 100644 --- a/src/core/runtime/runtime.h +++ b/src/core/runtime/runtime.h @@ -374,10 +374,11 @@ class Runtime { * @brief Creates a normal store out of a `Scalar` object * * @param scalar Value of the scalar to create a store with + * @param extents Shape of the store. The volume must be 1. * * @return Logical store */ - LogicalStore create_store(const Scalar& scalar); + [[nodiscard]] LogicalStore create_store(const Scalar& scalar, const Shape& extents = {1}); /** * @brief Creates a store by attaching to existing memory. * diff --git a/src/core/utilities/tuple.h b/src/core/utilities/tuple.h index 84ecc36cf9..a2480b27f5 100644 --- a/src/core/utilities/tuple.h +++ b/src/core/utilities/tuple.h @@ -90,7 +90,8 @@ class tuple { friend std::ostream& operator<<(std::ostream& out, const tuple<_T>& tpl); public: - const std::vector& data() const; + [[nodiscard]] std::vector& data(); + [[nodiscard]] const std::vector& data() const; private: std::vector data_{}; diff --git a/src/core/utilities/tuple.inl b/src/core/utilities/tuple.inl index 0d178a822e..e3905e90f4 100644 --- a/src/core/utilities/tuple.inl +++ b/src/core/utilities/tuple.inl @@ -281,6 +281,12 @@ std::ostream& operator<<(std::ostream& out, const tuple<_T>& tpl) return out; } +template +std::vector& tuple::data() +{ + return data_; +} + template const std::vector& tuple::data() const { diff --git a/src/core/utilities/typedefs.h b/src/core/utilities/typedefs.h index ac387623ac..f794e23970 100644 --- a/src/core/utilities/typedefs.h +++ b/src/core/utilities/typedefs.h @@ -129,7 +129,8 @@ using Domain = Legion::Domain; * for a complete list of supported operators. */ template -using AccessorRO = Legion::FieldAccessor>; +using AccessorRO = + Legion::FieldAccessor>; /** * @brief Write-only accessor @@ -139,7 +140,8 @@ using AccessorRO = Legion::FieldAccessor -using AccessorWO = Legion::FieldAccessor>; +using AccessorWO = + Legion::FieldAccessor>; /** * @brief Read-write accessor @@ -149,7 +151,8 @@ using AccessorWO = Legion::FieldAccessor -using AccessorRW = Legion::FieldAccessor>; +using AccessorRW = + Legion::FieldAccessor>; /** * @brief Reduction accessor diff --git a/tests/cpp/integration/broadcast_constraints.cc b/tests/cpp/integration/broadcast_constraints.cc index 9964e37d34..5d36cdaf94 100644 --- a/tests/cpp/integration/broadcast_constraints.cc +++ b/tests/cpp/integration/broadcast_constraints.cc @@ -48,9 +48,6 @@ struct Initializer : public legate::LegateTask { void prepare() { - static bool prepared = false; - if (prepared) { return; } - prepared = true; auto runtime = legate::Runtime::get_runtime(); auto context = runtime->create_library(library_name); TesterTask::register_variants(context, TESTER); @@ -62,7 +59,7 @@ void test_normal_store() auto runtime = legate::Runtime::get_runtime(); auto context = runtime->find_library(library_name); - auto launch_tester = [&](const std::vector& dims) { + auto launch_tester = [&](const std::vector& dims, bool omit_dims_in_broadcast) { std::vector extents(3, EXT_SMALL); for (auto dim : dims) extents[dim] = EXT_LARGE; auto store = runtime->create_store(extents, legate::int64()); @@ -71,17 +68,22 @@ void test_normal_store() task.add_scalar_arg(legate::Scalar(EXT_LARGE)); task.add_scalar_arg(legate::Scalar(dims)); task.add_scalar_arg(legate::Scalar(false)); - task.add_constraint(legate::broadcast(part, dims)); + if (omit_dims_in_broadcast) { + task.add_constraint(legate::broadcast(part)); + } else { + task.add_constraint(legate::broadcast(part, dims)); + } runtime->submit(std::move(task)); }; - launch_tester({0}); - launch_tester({1}); - launch_tester({2}); - launch_tester({0, 1}); - launch_tester({1, 2}); - launch_tester({0, 2}); - launch_tester({0, 1, 2}); + launch_tester({0}, false); + launch_tester({1}, false); + launch_tester({2}, false); + launch_tester({0, 1}, false); + launch_tester({1, 2}, false); + launch_tester({0, 2}, false); + launch_tester({0, 1, 2}, false); + launch_tester({0, 1, 2}, true); } void test_promoted_store() @@ -122,6 +124,7 @@ void test_invalid_broadcast() auto task = runtime->create_task(context, INITIALIZER); auto store = runtime->create_store({10}, legate::int64()); auto part = task.add_output(store); + EXPECT_THROW(task.add_constraint(legate::broadcast(part, {})), std::invalid_argument); task.add_constraint(legate::broadcast(part, {1})); EXPECT_THROW(runtime->submit(std::move(task)), std::invalid_argument); } diff --git a/tests/cpp/integration/delinearize.cc b/tests/cpp/integration/delinearize.cc new file mode 100644 index 0000000000..9aacc3e250 --- /dev/null +++ b/tests/cpp/integration/delinearize.cc @@ -0,0 +1,105 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include + +#include "legate.h" +#include "utilities/utilities.h" + +namespace delinearize { + +using Integration = DefaultFixture; + +static const char* library_name = "test_delinearize"; + +enum TaskIDs { + ARANGE = 0, + COPY = 1, +}; + +struct Arange : public legate::LegateTask { + static const int32_t TASK_ID = ARANGE; + static void cpu_variant(legate::TaskContext context) + { + auto output = context.output(0).data(); + auto shape = output.shape<1>(); + auto acc = output.write_accessor(); + for (legate::PointInRectIterator<1> it(shape); it.valid(); ++it) { + auto p = *it; + acc[p] = p[0]; + } + } +}; + +struct Copy : public legate::LegateTask { + static const int32_t TASK_ID = COPY; + static void cpu_variant(legate::TaskContext context) + { + auto input = context.input(0).data(); + auto output = context.output(0).data(); + auto shape = output.shape<3>(); + auto out_acc = output.write_accessor(); + auto in_acc = input.read_accessor(); + for (legate::PointInRectIterator<3> it(shape); it.valid(); ++it) { out_acc[*it] = in_acc[*it]; } + } +}; + +void register_tasks() +{ + auto runtime = legate::Runtime::get_runtime(); + auto library = runtime->create_library(library_name); + Arange::register_variants(library); + Copy::register_variants(library); +} + +void test_delinearize() +{ + auto runtime = legate::Runtime::get_runtime(); + auto library = runtime->find_library(library_name); + + auto input = runtime->create_array( + { + 16, + }, + legate::int64()); + auto output = runtime->create_array({1, 8, 2}, legate::int64()); + + { + auto task = runtime->create_task(library, ARANGE); + task.add_output(input); + runtime->submit(std::move(task)); + } + { + auto transformed = input.promote(0, 1).delinearize(1, {8, 2}); + auto task = runtime->create_task(library, COPY); + auto part_in = task.add_input(transformed); + auto part_out = task.add_output(output); + task.add_constraint(legate::align(part_out, part_in)); + runtime->submit(std::move(task)); + } + + auto p_out = output.data().get_physical_store(); + auto acc = p_out.read_accessor(); + auto shape = p_out.shape<3>(); + for (legate::PointInRectIterator<3> it(shape); it.valid(); ++it) { + auto p = *it; + EXPECT_EQ(acc[p], 2 * p[1] + p[2]); + } +} + +TEST_F(Integration, Delinearize) +{ + register_tasks(); + test_delinearize(); +} + +} // namespace delinearize diff --git a/tests/cpp/integration/inout.cc b/tests/cpp/integration/inout.cc index 11ad3a4c9e..10a562058b 100644 --- a/tests/cpp/integration/inout.cc +++ b/tests/cpp/integration/inout.cc @@ -13,30 +13,107 @@ #include #include "legate.h" -#include "tasks/task_simple.h" #include "utilities/utilities.h" namespace inout { using Integration = DefaultFixture; +enum TaskOpCode { + INOUT = 1, +}; + +static const char* library_name = "test_inout"; + +class TesterMapper : public legate::mapping::Mapper { + void set_machine(const legate::mapping::MachineQueryInterface* machine) override {} + legate::mapping::TaskTarget task_target( + const legate::mapping::Task& task, + const std::vector& options) override + { + return options.front(); + } + std::vector store_mappings( + const legate::mapping::Task& task, + const std::vector& options) override + { + std::vector mappings; + auto inputs = task.inputs(); + auto outputs = task.outputs(); + for (auto& input : inputs) { + mappings.push_back( + legate::mapping::StoreMapping::default_mapping(input.data(), options.front())); + } + for (auto& output : outputs) { + mappings.push_back( + legate::mapping::StoreMapping::default_mapping(output.data(), options.front())); + } + return mappings; + } + legate::Scalar tunable_value(legate::TunableID tunable_id) override { return legate::Scalar{}; } +}; + +struct InoutTask : public legate::LegateTask { + static const int32_t TASK_ID = INOUT; + static void cpu_variant(legate::TaskContext context) + { + auto output = context.output(0).data(); + auto shape = output.shape<2>(); + + if (shape.empty()) return; + + auto acc = output.read_write_accessor(shape); + for (legate::PointInRectIterator<2> it(shape); it.valid(); ++it) { + auto p = *it; + EXPECT_EQ(acc[p], 123); + acc[*it] = (p[0] + 1) + (p[1] + 1) * 1000; + } + } +}; + +void register_tasks() +{ + static bool prepared = false; + if (prepared) { return; } + prepared = true; + auto runtime = legate::Runtime::get_runtime(); + auto library = runtime->create_library( + library_name, legate::ResourceConfig{}, std::make_unique()); + InoutTask::register_variants(library); +} + void test_inout() { auto runtime = legate::Runtime::get_runtime(); - auto context = runtime->find_library(task::simple::library_name); + auto library = runtime->find_library(library_name); + + std::vector stores{ + runtime->create_store({10, 10}, legate::int64()), + runtime->create_store(legate::Scalar(int64_t{0}), legate::Shape{1, 1}), + }; + + for (auto& store : stores) { + runtime->issue_fill(store, legate::Scalar(int64_t(123))); - auto array = runtime->create_array({10, 10}, legate::int64()); - runtime->issue_fill(array.data(), legate::Scalar(int64_t(0))); + auto task = runtime->create_task(library, INOUT); + auto in_part = task.add_input(store); + auto out_part = task.add_output(store); + task.add_constraint(legate::align(in_part, out_part)); + runtime->submit(std::move(task)); - auto task = runtime->create_task(context, task::simple::HELLO); - task.add_input(array, task.find_or_declare_partition(array)); - task.add_output(array, task.find_or_declare_partition(array)); - runtime->submit(std::move(task)); + auto p_out = store.get_physical_store(); + auto acc = p_out.read_accessor(); + auto shape = p_out.shape<2>(); + for (legate::PointInRectIterator<2> it(shape); it.valid(); ++it) { + auto p = *it; + EXPECT_EQ(acc[p], (p[0] + 1) + (p[1] + 1) * 1000); + } + } } TEST_F(Integration, InOut) { - task::simple::register_tasks(); + register_tasks(); test_inout(); } diff --git a/tests/cpp/integration/scalar_out.cc b/tests/cpp/integration/scalar_out.cc new file mode 100644 index 0000000000..1e2a0079e6 --- /dev/null +++ b/tests/cpp/integration/scalar_out.cc @@ -0,0 +1,80 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include + +#include "legate.h" +#include "utilities/utilities.h" + +namespace scalarout { + +using Integration = DefaultFixture; + +static const char* library_name = "test_scalar_out"; + +enum TaskIDs { + COPY = 0, +}; + +struct Copy : public legate::LegateTask { + static const int32_t TASK_ID = COPY; + static void cpu_variant(legate::TaskContext context) + { + auto input = context.input(0).data(); + auto output = context.output(0).data(); + auto shape = output.shape<1>(); + if (shape.empty()) { return; } + auto out_acc = output.write_accessor(); + auto in_acc = input.read_accessor(); + out_acc[0] = in_acc[0]; + } +}; + +void register_tasks() +{ + auto runtime = legate::Runtime::get_runtime(); + auto library = runtime->create_library(library_name); + Copy::register_variants(library); +} + +void test_scalar_out() +{ + auto runtime = legate::Runtime::get_runtime(); + auto library = runtime->find_library(library_name); + + legate::Shape extents{16}; + auto input = runtime->create_store(extents, legate::int64(), false /* optimize_scalar */); + auto output = runtime->create_store(legate::Scalar{int64_t{0}}); + + runtime->issue_fill(input, legate::Scalar{int64_t{123}}); + + { + auto sliced = input.slice(0, legate::Slice{2, 3}); + auto task = runtime->create_task(library, COPY); + auto part_in = task.add_input(sliced); + auto part_out = task.add_output(output); + task.add_constraint(legate::align(part_in, part_out)); + runtime->submit(std::move(task)); + } + + auto p_out = output.get_physical_store(); + auto acc = p_out.read_accessor(); + EXPECT_EQ(acc[0], 123); +} + +TEST_F(Integration, ScalarOut) +{ + register_tasks(); + test_scalar_out(); +} + +} // namespace scalarout diff --git a/tests/cpp/unit/logical_store.cc b/tests/cpp/unit/logical_store.cc index 17b8e7c030..91a388ac6f 100644 --- a/tests/cpp/unit/logical_store.cc +++ b/tests/cpp/unit/logical_store.cc @@ -42,6 +42,27 @@ TEST_F(Store, Creation) EXPECT_FALSE(store.transformed()); EXPECT_THROW(store.extents(), std::invalid_argument); } + + // Scalar + { + auto runtime = legate::Runtime::get_runtime(); + auto store = runtime->create_store(legate::Scalar(int64_t{123})); + EXPECT_FALSE(store.unbound()); + EXPECT_EQ(store.dim(), 1); + EXPECT_EQ(store.extents(), std::vector{1}); + EXPECT_EQ(store.type(), legate::int64()); + EXPECT_FALSE(store.transformed()); + for (const auto& extents : {legate::Shape{1}, legate::Shape{1, 1}, legate::Shape{1, 1, 1}}) { + auto store = runtime->create_store(legate::Scalar(int64_t{123}), extents); + EXPECT_FALSE(store.unbound()); + EXPECT_EQ(store.dim(), extents.size()); + EXPECT_EQ(store.extents(), extents); + EXPECT_EQ(store.type(), legate::int64()); + EXPECT_FALSE(store.transformed()); + } + EXPECT_THROW(runtime->create_store(legate::Scalar(int64_t{123}), {1, 2}), + std::invalid_argument); + } } TEST_F(Store, Transform) From c97afd97ab5fb85c604a963a195d8b78414493c6 Mon Sep 17 00:00:00 2001 From: Jacob Faibussowitsch Date: Wed, 11 Oct 2023 13:39:43 -0400 Subject: [PATCH 0254/1425] - General CI Python Checker Maintenance (#124) - Add vermin (#124) - Configure isort from pyproject.toml --- .github/workflows/ci-gh-lint.yml | 5 +++- .pre-commit-config.yaml | 5 ++++ pyproject.toml | 39 ++++++++++++++++++++++++++++++++ setup.cfg | 35 ++++------------------------ 4 files changed, 53 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ci-gh-lint.yml b/.github/workflows/ci-gh-lint.yml index 1f3a755c17..1ffaf75b29 100644 --- a/.github/workflows/ci-gh-lint.yml +++ b/.github/workflows/ci-gh-lint.yml @@ -30,12 +30,15 @@ jobs: - name: Install requisite python packages run: | python3 -m pip install -U pip setuptools wheel - python3 -m pip install -U numpy mypy traitlets + python3 -m pip install -U numpy mypy traitlets vermin - name: Checkout ${{ github.event.repository.name }} (= this repo) uses: actions/checkout@v3 with: fetch-depth: 0 + - name: Run vermin on the repository + run: vermin --config-file=./setup.cfg ./legate + - name: Run MyPy on the repository run: python3 -m mypy ./legate diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cafd639653..7649fe2705 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,6 +29,11 @@ repos: hooks: - id: cython-lint - id: double-quote-cython-strings + - repo: https://github.com/netromdk/vermin + rev: 'v1.5.2' + hooks: + - id: vermin + args: ['--config-file', './setup.cfg'] - repo: local hooks: - id: legate-defined diff --git a/pyproject.toml b/pyproject.toml index c466da6c52..358f4794c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -101,3 +101,42 @@ ignore_errors = true [tool.pytest.ini_options] cache_dir = "./.cache/pytest" + +[tool.isort] +line_length = 79 +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +combine_as_imports = true +order_by_type = true +known_third_party = ["numpy"] +known_legion = [ + "legion_cffi", + "legion_top" +] +known_first_party = ["legate.core"] +default_section = "THIRDPARTY" +sections = [ + "FUTURE", + "STDLIB", + "THIRDPARTY", + "LEGION", + "FIRSTPARTY", + "LOCALFOLDER" +] +skip = [ + ".eggs", + ".git", + ".mypy_cache", + ".tox", + ".venv", + "_build", + "_skbuild", + "build", + "dist", + "__init__.py" +] +skip_glob = [ + "legion/*", + "install/*" +] diff --git a/setup.cfg b/setup.cfg index 65a47048c0..caaf0a6e65 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,36 +25,11 @@ ignore = # whitespace before : E203 -[isort] -line_length=79 -multi_line_output=3 -include_trailing_comma=True -force_grid_wrap=0 -combine_as_imports=True -order_by_type=True -known_third_party= - numpy -known_legion= - legion_cffi - legion_top -known_first_party= - legate.core -default_section=THIRDPARTY -sections=FUTURE,STDLIB,THIRDPARTY,LEGION,FIRSTPARTY,LOCALFOLDER -skip= - .eggs - .git - .mypy_cache - .tox - .venv - _build - build - dist - __init__.py -skip_glob= - legion/* - install/* - +[vermin] +verbose = 3 +only_show_violations = yes +eval_annotations = yes +targets = 3.9- [options] packages = find: From 83458e6477b9998b12f37387d0b86de5a3706165 Mon Sep 17 00:00:00 2001 From: Jacob Faibussowitsch Date: Thu, 12 Oct 2023 09:49:23 -0400 Subject: [PATCH 0255/1425] Bump mypy version to match CI job and fix errors (#121) --- .github/workflows/ci-gh-lint.yml | 12 +++++++----- .pre-commit-config.yaml | 2 +- pyproject.toml | 1 + 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-gh-lint.yml b/.github/workflows/ci-gh-lint.yml index 1ffaf75b29..445d64a836 100644 --- a/.github/workflows/ci-gh-lint.yml +++ b/.github/workflows/ci-gh-lint.yml @@ -22,6 +22,11 @@ jobs: - name: List python version run: python3 --version + - name: Checkout ${{ github.event.repository.name }} (= this repo) + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Bootstrap pip run: | curl -O https://bootstrap.pypa.io/get-pip.py @@ -30,12 +35,9 @@ jobs: - name: Install requisite python packages run: | python3 -m pip install -U pip setuptools wheel - python3 -m pip install -U numpy mypy traitlets vermin + python3 -m pip install -U numpy traitlets vermin + python3 -m pip install mypy==$(awk '/repo:.*mypy/{getline; print $2}' ./.pre-commit-config.yaml | tr -d \'v) - - name: Checkout ${{ github.event.repository.name }} (= this repo) - uses: actions/checkout@v3 - with: - fetch-depth: 0 - name: Run vermin on the repository run: vermin --config-file=./setup.cfg ./legate diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7649fe2705..143613f82c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: files: \.(cu|cuh|h|cc|inl)$ types_or: [] - repo: https://github.com/pre-commit/mirrors-mypy - rev: 'v1.4.1' + rev: 'v1.5.1' hooks: - id: mypy pass_filenames: false diff --git a/pyproject.toml b/pyproject.toml index 358f4794c1..1aad09a98d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -102,6 +102,7 @@ ignore_errors = true [tool.pytest.ini_options] cache_dir = "./.cache/pytest" + [tool.isort] line_length = 79 multi_line_output = 3 From 25776fa2057d9f9b8f2ee6dde719099c54dfdf87 Mon Sep 17 00:00:00 2001 From: Wonchan Lee Date: Thu, 12 Oct 2023 11:26:37 -0700 Subject: [PATCH 0256/1425] Handle mixed dimensionality cases correctly (#125) * Use the right copyright header * Handle mixed dimensionality cases correctly * Minor tweak to be in line with the modernization PR --- src/core/partitioning/detail/partitioner.cc | 5 +- tests/cpp/integration/consensus_match.cc | 22 +++---- tests/cpp/integration/field_reuse.cc | 22 +++---- tests/cpp/integration/mixed_dim.cc | 71 +++++++++++++++++++++ tests/cpp/integration/tree_reduce.cc | 22 +++---- tests/cpp/integration/tree_reduce_unique.cc | 22 +++---- 6 files changed, 108 insertions(+), 56 deletions(-) create mode 100644 tests/cpp/integration/mixed_dim.cc diff --git a/src/core/partitioning/detail/partitioner.cc b/src/core/partitioning/detail/partitioner.cc index b67a092eca..1ec4b84383 100644 --- a/src/core/partitioning/detail/partitioner.cc +++ b/src/core/partitioning/detail/partitioner.cc @@ -76,10 +76,7 @@ Domain LaunchDomainResolver::resolve_launch_domain() const } else { if (LegateDefined(LEGATE_USE_DEBUG)) { assert(launch_domains_.size() == 1); } auto& launch_domain = *launch_domains_.begin(); - if (unbound_dim_ != UNSET && launch_domain.dim != unbound_dim_) { - int64_t volume = *launch_volumes_.begin(); - return Domain{0, volume - 1}; - } + if (unbound_dim_ != UNSET && launch_domain.dim != unbound_dim_) return {}; return launch_domain; } } diff --git a/tests/cpp/integration/consensus_match.cc b/tests/cpp/integration/consensus_match.cc index 9a164e0baf..027ad099c3 100644 --- a/tests/cpp/integration/consensus_match.cc +++ b/tests/cpp/integration/consensus_match.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/integration/field_reuse.cc b/tests/cpp/integration/field_reuse.cc index d816de6336..62eb7e056e 100644 --- a/tests/cpp/integration/field_reuse.cc +++ b/tests/cpp/integration/field_reuse.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/integration/mixed_dim.cc b/tests/cpp/integration/mixed_dim.cc new file mode 100644 index 0000000000..eda80efec0 --- /dev/null +++ b/tests/cpp/integration/mixed_dim.cc @@ -0,0 +1,71 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include + +#include "legate.h" +#include "utilities/utilities.h" + +namespace mixed_dim { + +using Partitioner = DefaultFixture; + +static const char* library_name = "test_mixed_dim"; + +enum TaskIDs { + TESTER = 0, +}; + +struct Tester : public legate::LegateTask { + static const int32_t TASK_ID = TESTER; + static void cpu_variant(legate::TaskContext context) + { + EXPECT_TRUE(context.is_single_task()); + context.output(0).data().bind_empty_data(); + } +}; + +TEST_F(Partitioner, MixedDim) +{ + auto runtime = legate::Runtime::get_runtime(); + auto library = runtime->create_library(library_name); + Tester::register_variants(library); + + auto test = [&runtime, &library]( + int32_t unbound_ndim, const auto& extents1, const auto& extents2) { + auto unbound = runtime->create_store(legate::int32(), unbound_ndim); + auto normal1 = runtime->create_store(extents1, legate::int64()); + auto normal2 = runtime->create_store(extents2, legate::float64()); + + auto task = runtime->create_task(library, TESTER); + task.add_output(unbound); + task.add_output(normal1); + task.add_output(normal2); + runtime->submit(std::move(task)); + }; + + auto ty = legate::int64(); + auto extents1d = legate::Shape{ + 10, + }; + auto extents2d = legate::Shape{10, 20}; + auto extents3d = legate::Shape{10, 20, 30}; + // Must-be-1D cases + test(2, extents1d, extents2d); + test(3, extents1d, extents2d); + // Cases where normal stores lead to a single launch domain + test(1, extents2d, extents2d); + test(2, extents3d, extents3d); + test(3, extents1d, extents1d); +} + +} // namespace mixed_dim diff --git a/tests/cpp/integration/tree_reduce.cc b/tests/cpp/integration/tree_reduce.cc index 8f06d9fdea..b9b0750f4a 100644 --- a/tests/cpp/integration/tree_reduce.cc +++ b/tests/cpp/integration/tree_reduce.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include diff --git a/tests/cpp/integration/tree_reduce_unique.cc b/tests/cpp/integration/tree_reduce_unique.cc index c9c2b53577..e08c902ad5 100644 --- a/tests/cpp/integration/tree_reduce_unique.cc +++ b/tests/cpp/integration/tree_reduce_unique.cc @@ -1,17 +1,13 @@ -/* Copyright 2023 NVIDIA Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. */ #include From df30a60c1baad17d295dcea1de5132d58d9f152d Mon Sep 17 00:00:00 2001 From: Jacob Faibussowitsch Date: Thu, 12 Oct 2023 16:55:18 -0400 Subject: [PATCH 0257/1425] install.py use the cmake presets (#120) --- install.py | 162 +++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 131 insertions(+), 31 deletions(-) diff --git a/install.py b/install.py index 1f62863fb6..31368ded44 100755 --- a/install.py +++ b/install.py @@ -15,6 +15,7 @@ import multiprocessing import os import platform +import re import shutil import subprocess import sys @@ -269,6 +270,86 @@ def install_legion_jupyter_notebook( ) +def configure_cmake_preset(argparse_args): + def vprint(*args, flush=True, **kwargs): + if argparse_args.verbose: + print(*args, **kwargs, flush=flush) + + def yield_compiler_guesses(): + def try_guess_compiler(gen_fn, *args): + vprint(f"Trying: {gen_fn.__qualname__}{args}") + compiler = gen_fn(*args) + if not compiler: + return + + vprint(compiler, "exists") + try: + stdout = subprocess.check_output( + [compiler, "--version"] + ).decode() + except subprocess.CalledProcessError as cpe: + vprint(cpe) + return + + vprint(stdout) + if re.search(r"clang\s+version", stdout): + vprint("Detected clang") + return "clang" + if re.search( + r"[gG][cC][cC].*Free\s+Software\s+Foundation", + stdout, + re.DOTALL, + ): + vprint("Detected GCC") + return "gcc" + vprint("Detected neither!") + + for gen in ( + (os.environ.get, "CMAKE_CXX_COMPILER", ""), + (os.environ.get, "CXX", ""), + # cmake likes to use cc and c++ to pick the compilers + (shutil.which, "c++"), + (os.environ.get, "CMAKE_C_COMPILER", ""), + (os.environ.get, "CC", ""), + (shutil.which, "cc"), + ): + yield try_guess_compiler(*gen) + + # last resort + for compiler in ("clang", "gcc"): + vprint("Searching for compiler", compiler) + yield shutil.which(compiler) + + raise RuntimeError("Could not find suitable compiler!") + + cmake_preset = [] + for attr, val in dict(vars(argparse_args)).items(): + if not attr.startswith("preset_"): + continue + + vprint("Found preset attr", attr, "=", val, end="") + delattr(argparse_args, attr) + vprint("... deleted") + if val: + cmake_preset = attr.split("_")[1:] # skip "preset_" + vprint("Set cmake preset:", cmake_preset) + + if not cmake_preset: + vprint("No cmake preset attr found, using default") + cmake_preset = ["release"] # default to release if unset + + vprint("Base cmake preset configured:", cmake_preset) + for guess in yield_compiler_guesses(): + if guess: + vprint("Guess successful, using", guess) + cmake_preset.append(guess) + break + + ret = "-".join(cmake_preset) + vprint("Final cmake preset set:", ret) + return ret + + def install( cmake_preset, networks, @@ -290,8 +371,6 @@ def install( cuda_dir, maxdim, maxfields, - debug, - debug_release, check_bounds, clean_first, extra_flags, @@ -344,8 +423,6 @@ def install( print(f"cuda_dir: {cuda_dir}") print(f"maxdim: {maxdim}") print(f"maxfields: {maxfields}") - print(f"debug: {debug}") - print(f"debug_release: {debug_release}") print(f"check_bounds: {check_bounds}") print(f"clean_first: {clean_first}") print(f"extra_flags: {extra_flags}") @@ -445,9 +522,7 @@ def validate_path(path): if clean_first: shutil.rmtree(skbuild_dir, ignore_errors=True) shutil.rmtree(join(legate_core_dir, "dist"), ignore_errors=True) - build_dir = join(legate_core_dir, "build") - if cmake_preset: - build_dir = join(build_dir, cmake_preset) + build_dir = join(legate_core_dir, "build", cmake_preset) shutil.rmtree(build_dir, ignore_errors=True) shutil.rmtree( join(legate_core_dir, "legate_core.egg-info"), @@ -458,12 +533,14 @@ def validate_path(path): pip_install_cmd = [sys.executable, "-m", "pip", "install"] # Use preexisting CMAKE_ARGS from conda if set - cmake_flags = [] + cmake_flags = ["--preset", cmake_preset] - if cmake_preset: - cmake_flags.extend(["--preset", cmake_preset]) - - cmake_flags.extend(cmd_env.get("CMAKE_ARGS", "").split(" ")) + if env_cmake_args := [ + f + for f in cmd_env.get("CMAKE_ARGS", "").split() + if not f.startswith("-DCMAKE_BUILD_TYPE") + ]: + cmake_flags.extend(env_cmake_args) if unknown is None: unknown = [] @@ -498,13 +575,10 @@ def validate_path(path): if verbose: pip_install_cmd += ["-vv"] - if debug or verbose: - cmake_flags += [f"--log-level={'DEBUG' if debug else 'VERBOSE'}"] + if cmake_preset.startswith("debug"): + cmake_flags += ["--log-level=DEBUG"] cmake_flags += f"""\ --DCMAKE_BUILD_TYPE={( - "Debug" if debug else "RelWithDebInfo" if debug_release else "Release" -)} -DBUILD_SHARED_LIBS=ON -DBUILD_MARCH={march} -DLegion_MAX_DIM={str(maxdim)} @@ -584,10 +658,6 @@ def validate_path(path): # execute python -m pip install . execute_command(pip_install_cmd, verbose, cwd=legate_core_dir, env=cmd_env) - print("=" * 90, flush=True) - print("=" * 90, flush=True) - print("=" * 90, flush=True) - print("=" * 90, flush=True) install_legion_python_bindings( verbose, cmake_exe, @@ -609,31 +679,54 @@ def driver(): "PREFIX" in os.environ and os.environ.get("CONDA_BUILD", "0") == "1" ) - parser = argparse.ArgumentParser(description="Install Legate front end.") - parser.add_argument( - "--cmake-preset", + parser = argparse.ArgumentParser( + description="Install Legate front end.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + preset_group_super = parser.add_argument_group( + title="Build Presets", + description="Configure Legate core to a specific build preset", + ) + + # a bit of a hack to make this group have a title and description + preset_group = preset_group_super.add_mutually_exclusive_group() + preset_group.add_argument( + "--release", + dest="preset_release", + action="store_true", required=False, - default=os.environ.get("LEGATE_CORE_CMAKE_PRESET", ""), - help="Set a CMake Preset to use", + default=os.environ.get("RELEASE", "1") == "1", + help="Build Legate and Legion with full optimizations enabled", ) - parser.add_argument( + preset_group.add_argument( "--debug", - dest="debug", + dest="preset_debug", action="store_true", required=False, default=os.environ.get("DEBUG", "0") == "1", help="Build Legate and Legion with no optimizations, and full " "debugging checks.", ) - parser.add_argument( + preset_group.add_argument( + "--debug-sanitizer", + dest="preset_debug_sanitizer", + action="store_true", + required=False, + default=os.environ.get("DEBUG_SANITZER", "0") == "1", + help="Build Legate and Legion with no optimizations, and full " + "debugging checks with sanitizer support.", + ) + preset_group.add_argument( "--debug-release", - dest="debug_release", + dest="preset_debug_release", action="store_true", required=False, default=os.environ.get("DEBUG_RELEASE", "0") == "1", help="Build Legate and Legion with optimizations enabled, but include " "debugging symbols.", ) + parser.add_argument( "--check-bounds", dest="check_bounds", @@ -924,7 +1017,14 @@ def driver(): print(f"Attempted to execute: {args.cmake_exe}") sys.exit(1) - install(unknown=unknown, is_conda=is_conda, **vars(args)) + cmake_preset = configure_cmake_preset(args) + + install( + unknown=unknown, + is_conda=is_conda, + cmake_preset=cmake_preset, + **vars(args), + ) if __name__ == "__main__": From d650d6c04c15a21111acdc04a03d2bb74bf89e07 Mon Sep 17 00:00:00 2001 From: Joy Shen <110752938+joyshennv@users.noreply.github.com> Date: Fri, 13 Oct 2023 14:10:04 +0800 Subject: [PATCH 0258/1425] Expand testcases for null type and binary type. (#78) --- tests/cpp/unit/type.cc | 57 +++++++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/tests/cpp/unit/type.cc b/tests/cpp/unit/type.cc index 50f0f13562..4fc006cf55 100644 --- a/tests/cpp/unit/type.cc +++ b/tests/cpp/unit/type.cc @@ -35,11 +35,11 @@ const std::vector PRIMITIVE_TYPE = {legate::bool_(), legate::complex128()}; template -void test_primitive_type(const legate::Type& type, std::string type_string) +void test_primitive_type(const legate::Type& type, std::string type_string, uint32_t size) { EXPECT_EQ(type.code(), legate::legate_type_code_of); - EXPECT_EQ(type.size(), sizeof(T)); - EXPECT_EQ(type.alignment(), sizeof(T)); + EXPECT_EQ(type.size(), size); + EXPECT_EQ(type.alignment(), size); EXPECT_FALSE(type.variable_size()); EXPECT_TRUE(type.is_primitive()); EXPECT_EQ(type.to_string(), type_string); @@ -151,21 +151,23 @@ void test_reduction_op(const legate::Type& type) TEST_F(TypeUnit, PrimitiveType) { - test_primitive_type(legate::bool_(), "bool"); - test_primitive_type(legate::int8(), "int8"); - test_primitive_type(legate::int16(), "int16"); - test_primitive_type(legate::int32(), "int32"); - test_primitive_type(legate::int64(), "int64"); - test_primitive_type(legate::uint8(), "uint8"); - test_primitive_type(legate::uint16(), "uint16"); - test_primitive_type(legate::uint32(), "uint32"); - test_primitive_type(legate::uint64(), "uint64"); - test_primitive_type<__half>(legate::float16(), "float16"); - test_primitive_type(legate::float32(), "float32"); - test_primitive_type(legate::float64(), "float64"); - test_primitive_type>(legate::complex64(), "complex64"); - test_primitive_type>(legate::complex128(), "complex128"); - + test_primitive_type(legate::null_type(), "null_type", 0); + test_primitive_type(legate::bool_(), "bool", sizeof(bool)); + test_primitive_type(legate::int8(), "int8", sizeof(int8_t)); + test_primitive_type(legate::int16(), "int16", sizeof(int16_t)); + test_primitive_type(legate::int32(), "int32", sizeof(int32_t)); + test_primitive_type(legate::int64(), "int64", sizeof(int64_t)); + test_primitive_type(legate::uint8(), "uint8", sizeof(uint8_t)); + test_primitive_type(legate::uint16(), "uint16", sizeof(uint16_t)); + test_primitive_type(legate::uint32(), "uint32", sizeof(uint32_t)); + test_primitive_type(legate::uint64(), "uint64", sizeof(uint64_t)); + test_primitive_type<__half>(legate::float16(), "float16", sizeof(__half)); + test_primitive_type(legate::float32(), "float32", sizeof(float)); + test_primitive_type(legate::float64(), "float64", sizeof(double)); + test_primitive_type>(legate::complex64(), "complex64", sizeof(complex)); + test_primitive_type>(legate::complex128(), "complex128", sizeof(complex)); + + EXPECT_EQ(legate::null_type(), legate::primitive_type(legate::Type::Code::NIL)); EXPECT_EQ(legate::bool_(), legate::primitive_type(legate::Type::Code::BOOL)); EXPECT_EQ(legate::int8(), legate::primitive_type(legate::Type::Code::INT8)); EXPECT_EQ(legate::int16(), legate::primitive_type(legate::Type::Code::INT16)); @@ -181,11 +183,11 @@ TEST_F(TypeUnit, PrimitiveType) EXPECT_EQ(legate::complex64(), legate::primitive_type(legate::Type::Code::COMPLEX64)); EXPECT_EQ(legate::complex128(), legate::primitive_type(legate::Type::Code::COMPLEX128)); - EXPECT_THROW(legate::primitive_type(legate::Type::Code::BINARY), std::invalid_argument); EXPECT_THROW(legate::primitive_type(legate::Type::Code::FIXED_ARRAY), std::invalid_argument); EXPECT_THROW(legate::primitive_type(legate::Type::Code::STRUCT), std::invalid_argument); EXPECT_THROW(legate::primitive_type(legate::Type::Code::STRING), std::invalid_argument); EXPECT_THROW(legate::primitive_type(legate::Type::Code::LIST), std::invalid_argument); + EXPECT_THROW(legate::primitive_type(legate::Type::Code::BINARY), std::invalid_argument); } TEST_F(TypeUnit, StringType) { test_string_type(legate::string_type()); } @@ -194,7 +196,6 @@ TEST_F(TypeUnit, BinaryType) { test_binary_type(legate::binary_type(123), 123); test_binary_type(legate::binary_type(45), 45); - EXPECT_EQ(legate::binary_type(678).uid(), legate::binary_type(678).uid()); EXPECT_THROW(legate::binary_type(0), std::out_of_range); EXPECT_THROW(legate::binary_type(0xFFFFF + 1), std::out_of_range); @@ -455,6 +456,13 @@ TEST_F(TypeUnit, Uid) EXPECT_NE(list_type1.uid(), static_cast(element_type.code())); EXPECT_NE(list_type2.uid(), static_cast(element_type.code())); } + + // binary type + auto binary_type = legate::binary_type(678); + auto same_binary_type = legate::binary_type(678); + auto diff_binary_type = legate::binary_type(67); + EXPECT_EQ(binary_type.uid(), same_binary_type.uid()); + EXPECT_NE(binary_type.uid(), diff_binary_type.uid()); } TEST_F(TypeUnit, ReductionOperator) @@ -462,6 +470,9 @@ TEST_F(TypeUnit, ReductionOperator) test_reduction_op(legate::string_type()); test_reduction_op(legate::fixed_array_type(legate::int64(), 10)); test_reduction_op(legate::struct_type(true, legate::int64(), legate::bool_(), legate::float64())); + test_reduction_op( + legate::struct_type(false, legate::int64(), legate::bool_(), legate::float64())); + test_reduction_op(legate::binary_type(10)); } TEST_F(TypeUnit, TypeCodeOf) @@ -508,6 +519,7 @@ TEST_F(TypeUnit, TypeOf) EXPECT_TRUE((std::is_same_v, void>)); EXPECT_TRUE((std::is_same_v, void>)); EXPECT_TRUE((std::is_same_v, void>)); + EXPECT_TRUE((std::is_same_v, void>)); } TEST_F(TypeUnit, TypeUtils) @@ -534,6 +546,7 @@ TEST_F(TypeUnit, TypeUtils) EXPECT_FALSE(legate::is_integral::value); EXPECT_FALSE(legate::is_integral::value); EXPECT_FALSE(legate::is_integral::value); + EXPECT_FALSE(legate::is_integral::value); // is_signed EXPECT_TRUE(legate::is_signed::value); @@ -557,6 +570,7 @@ TEST_F(TypeUnit, TypeUtils) EXPECT_FALSE(legate::is_signed::value); EXPECT_FALSE(legate::is_signed::value); EXPECT_FALSE(legate::is_signed::value); + EXPECT_FALSE(legate::is_signed::value); // is_unsigned EXPECT_TRUE(legate::is_unsigned::value); @@ -580,6 +594,7 @@ TEST_F(TypeUnit, TypeUtils) EXPECT_FALSE(legate::is_unsigned::value); EXPECT_FALSE(legate::is_unsigned::value); EXPECT_FALSE(legate::is_unsigned::value); + EXPECT_FALSE(legate::is_unsigned::value); // is_floating_point EXPECT_TRUE(legate::is_floating_point::value); @@ -603,6 +618,7 @@ TEST_F(TypeUnit, TypeUtils) EXPECT_FALSE(legate::is_floating_point::value); EXPECT_FALSE(legate::is_floating_point::value); EXPECT_FALSE(legate::is_floating_point::value); + EXPECT_FALSE(legate::is_floating_point::value); // is_complex EXPECT_TRUE(legate::is_complex::value); @@ -626,6 +642,7 @@ TEST_F(TypeUnit, TypeUtils) EXPECT_FALSE(legate::is_complex::value); EXPECT_FALSE(legate::is_complex::value); EXPECT_FALSE(legate::is_complex::value); + EXPECT_FALSE(legate::is_complex::value); // is_complex_type EXPECT_TRUE(legate::is_complex_type>::value); From befdfc6001ce57f36a83b64059aef9df7fcbc2f9 Mon Sep 17 00:00:00 2001 From: Jacob Faibussowitsch Date: Fri, 13 Oct 2023 14:26:24 -0400 Subject: [PATCH 0259/1425] - Add legate::{Internal}SharedPtr (#84) - Add compressed_pair - Add legate::make_{internal_}shared() --- legate_core_cpp.cmake | 5 + src/core/utilities/compressed_pair.h | 237 +++++++++ src/core/utilities/internal_shared_ptr.h | 160 +++++++ src/core/utilities/internal_shared_ptr.inl | 533 +++++++++++++++++++++ src/core/utilities/memory.h | 3 + src/core/utilities/shared_ptr.h | 119 +++++ src/core/utilities/shared_ptr.inl | 297 ++++++++++++ tests/cpp/CMakeLists.txt | 4 +- tests/cpp/unit/internal_shared_ptr.cc | 295 ++++++++++++ tests/cpp/unit/logical_store.cc | 2 +- tests/cpp/unit/shared_ptr.cc | 470 ++++++++++++++++++ tests/cpp/unit/shared_ptr_util.h | 196 ++++++++ 12 files changed, 2318 insertions(+), 3 deletions(-) create mode 100644 src/core/utilities/compressed_pair.h create mode 100644 src/core/utilities/internal_shared_ptr.h create mode 100644 src/core/utilities/internal_shared_ptr.inl create mode 100644 src/core/utilities/shared_ptr.h create mode 100644 src/core/utilities/shared_ptr.inl create mode 100644 tests/cpp/unit/internal_shared_ptr.cc create mode 100644 tests/cpp/unit/shared_ptr.cc create mode 100644 tests/cpp/unit/shared_ptr_util.h diff --git a/legate_core_cpp.cmake b/legate_core_cpp.cmake index e0487edd32..60ace0a794 100644 --- a/legate_core_cpp.cmake +++ b/legate_core_cpp.cmake @@ -570,6 +570,11 @@ install( src/core/utilities/tuple.h src/core/utilities/tuple.inl src/core/utilities/typedefs.h + src/core/utilities/shared_ptr.h + src/core/utilities/shared_ptr.inl + src/core/utilities/internal_shared_ptr.h + src/core/utilities/internal_shared_ptr.inl + src/core/utilities/compressed_pair.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/legate/core/utilities) ############################################################################## diff --git a/src/core/utilities/compressed_pair.h b/src/core/utilities/compressed_pair.h new file mode 100644 index 0000000000..9aba479cef --- /dev/null +++ b/src/core/utilities/compressed_pair.h @@ -0,0 +1,237 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#pragma once + +#include +#include + +namespace legate { + +namespace detail { + +template +struct compressed_pair_selector; + +template <> +struct compressed_pair_selector : std::integral_constant {}; + +template <> +struct compressed_pair_selector : std::integral_constant {}; + +template <> +struct compressed_pair_selector : std::integral_constant {}; + +template <> +struct compressed_pair_selector : std::integral_constant {}; + +template +class compressed_pair_impl; + +// selector = 0, neither are empty, derive directly from std::pair +template +class compressed_pair_impl : std::pair { + using base_type = std::pair; + + public: + using base_type::base_type; + using typename base_type::first_type; + using typename base_type::second_type; + + [[nodiscard]] first_type& first() noexcept { return static_cast(*this).first; } + [[nodiscard]] const first_type& first() const noexcept + { + return static_cast(*this).first; + } + + [[nodiscard]] second_type& second() noexcept { return static_cast(*this).second; } + [[nodiscard]] const second_type& second() const noexcept + { + return static_cast(*this).second; + } +}; + +// selector = 1, T is empty +template +class compressed_pair_impl : T { + using base_type = T; + + public: + using base_type::base_type; + using first_type = T; + using second_type = U; + + compressed_pair_impl() = default; + + compressed_pair_impl(first_type x, second_type y) : base_type{std::move(x)}, second_{std::move(y)} + { + } + + compressed_pair_impl(second_type x) : second_{std::move(x)} {} + + [[nodiscard]] first_type& first() noexcept { return *this; } + [[nodiscard]] const first_type& first() const noexcept { return *this; } + + [[nodiscard]] second_type& second() noexcept { return second_; } + [[nodiscard]] const second_type& second() const noexcept { return second_; } + + private: + second_type second_; +}; + +// selector = 2, U is empty +template +class compressed_pair_impl : U { + using base_type = U; + + public: + using base_type::base_type; + using first_type = T; + using second_type = U; + + compressed_pair_impl() = default; + + compressed_pair_impl(first_type x, second_type y) : base_type{std::move(y)}, first_{std::move(x)} + { + } + + compressed_pair_impl(first_type x) : first_{std::move(x)} {} + + [[nodiscard]] first_type& first() noexcept { return first_; } + [[nodiscard]] const first_type& first() const noexcept { return first_; } + + [[nodiscard]] second_type& second() noexcept { return *this; } + [[nodiscard]] const second_type& second() const noexcept { return *this; } + + private: + first_type first_; +}; + +// selector = 3, T and U are both empty +template +class compressed_pair_impl : T, U { + using first_base_type = T; + using second_base_type = U; + + public: + using first_type = T; + using second_type = U; + + using first_type::first_type; + using second_type::second_type; + + compressed_pair_impl() = default; + + compressed_pair_impl(first_type x, second_type y) + : first_type{std::move(x)}, second_type{std::move(y)} + { + } + + // Casts are needed to disambiguate case where T or U derive from one another, for example + // + // struct T { }; + // struct U : T { }; + // + // In this case both U and T are able to satisfy "conversion" to T + [[nodiscard]] first_type& first() noexcept { return static_cast(*this); } + [[nodiscard]] const first_type& first() const noexcept + { + return static_cast(*this); + } + + [[nodiscard]] second_type& second() noexcept { return static_cast(*this); } + [[nodiscard]] const second_type& second() const noexcept + { + return static_cast(*this); + } +}; + +} // namespace detail + +template +class compressed_pair + : public detail::compressed_pair_impl< + T, + U, + detail::compressed_pair_selector, std::is_empty_v>::value> { + using base_type = detail::compressed_pair_impl< + T, + U, + detail::compressed_pair_selector, std::is_empty_v>::value>; + + public: + using base_type::base_type; +}; + +// Intel compilers don't implement empty base optimization (yes, you read that right), so these +// tests fail +#if !defined(__INTEL_COMPILER) && !defined(__ICL) + +namespace compressed_pair_test { + +struct Empty {}; + +static_assert(std::is_empty_v); +static_assert(sizeof(Empty) == 1); + +struct Empty2 {}; + +static_assert(std::is_empty::value); +static_assert(sizeof(Empty2) == 1); + +struct NotEmpty { + std::uint64_t d{}; +}; + +static_assert(!std::is_empty_v); +static_assert(sizeof(NotEmpty) > 1); + +struct EmptyMember { + Empty m{}; + Empty2 m2{}; +}; + +static_assert(!std::is_empty_v); +static_assert(sizeof(EmptyMember) > 1); + +// empty-empty should only be 1 byte since both are compressed out +static_assert(std::is_empty_v>); +static_assert(sizeof(compressed_pair) == 1); + +// flipping template param order changes nothing +static_assert(std::is_empty_v>); +static_assert(sizeof(compressed_pair) == 1); + +// empty-not_empty should be less than sum of sizes, since empty is compressed out +static_assert(!std::is_empty_v>); +static_assert(sizeof(compressed_pair) < (sizeof(Empty) + sizeof(NotEmpty))); + +// flipping template param order changes nothing +static_assert(!std::is_empty_v>); +static_assert(sizeof(compressed_pair) < (sizeof(NotEmpty) + sizeof(Empty))); + +// empty_member-not_empty should also be greater than or equal to sum of sizes (g.t. because +// potential padding) because neither is compressed away +static_assert(!std::is_empty_v>); +static_assert(sizeof(compressed_pair) >= + (sizeof(EmptyMember) + sizeof(NotEmpty))); + +// flipping template param order changes nothing +static_assert(!std::is_empty_v>); +static_assert(sizeof(compressed_pair) >= + (sizeof(NotEmpty) + sizeof(EmptyMember))); + +} // namespace compressed_pair_test + +#endif + +} // namespace legate diff --git a/src/core/utilities/internal_shared_ptr.h b/src/core/utilities/internal_shared_ptr.h new file mode 100644 index 0000000000..360457846f --- /dev/null +++ b/src/core/utilities/internal_shared_ptr.h @@ -0,0 +1,160 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#pragma once + +#include +#include +#include + +namespace legate { + +namespace detail { + +class ControlBlockBase { + public: + using ref_count_type = std::uint32_t; + + constexpr ControlBlockBase() noexcept = default; + virtual ~ControlBlockBase() noexcept = default; + + virtual void destroy() noexcept = 0; + virtual void dispose_control_block() noexcept = 0; + [[nodiscard]] virtual void* ptr() noexcept = 0; + [[nodiscard]] virtual const void* ptr() const noexcept = 0; + + [[nodiscard]] ref_count_type strong_ref_cnt() const noexcept; + [[nodiscard]] ref_count_type user_ref_cnt() const noexcept; + + ref_count_type strong_ref() noexcept; + ref_count_type user_ref() noexcept; + ref_count_type strong_deref() noexcept; + ref_count_type user_deref() noexcept; + + protected: + template + static void dispose_control_block_impl_(T* cb_impl); + + private: + ref_count_type strong_refs_{1}; + ref_count_type user_refs_{0}; +}; + +} // namespace detail + +template +class SharedPtr; + +template +class InternalSharedPtr; + +template +void swap(InternalSharedPtr&, InternalSharedPtr&) noexcept; + +template +[[nodiscard]] InternalSharedPtr make_internal_shared(Args&&... args); + +template +class InternalSharedPtr { + using control_block_type = detail::ControlBlockBase; + + public: + using element_type = std::remove_extent_t; + using ref_count_type = typename control_block_type::ref_count_type; + static_assert(!std::is_reference_v); + + // constructors + constexpr InternalSharedPtr() noexcept = default; + + InternalSharedPtr(std::nullptr_t) noexcept; + + template > + InternalSharedPtr(U* ptr, D deleter, A allocator = A{}); + template + explicit InternalSharedPtr(U* ptr); + explicit InternalSharedPtr(element_type* ptr); + + InternalSharedPtr(const InternalSharedPtr& other) noexcept; + InternalSharedPtr& operator=(const InternalSharedPtr& other) noexcept; + InternalSharedPtr(InternalSharedPtr&& other) noexcept; + InternalSharedPtr& operator=(InternalSharedPtr&& other) noexcept; + + template + InternalSharedPtr(const InternalSharedPtr& other) noexcept; + template + InternalSharedPtr& operator=(const InternalSharedPtr& other) noexcept; + template + InternalSharedPtr(InternalSharedPtr&& other) noexcept; + template + InternalSharedPtr& operator=(InternalSharedPtr&& other) noexcept; + + template + InternalSharedPtr(std::unique_ptr&& ptr); + template + InternalSharedPtr& operator=(std::unique_ptr&& ptr); + + ~InternalSharedPtr() noexcept; + + // Modifiers + void swap(InternalSharedPtr& other) noexcept; + friend void ::legate::swap<>(InternalSharedPtr& lhs, InternalSharedPtr& rhs) noexcept; + void reset() noexcept; + void reset(std::nullptr_t) noexcept; + template , typename A = std::allocator> + void reset(U* ptr, D deleter = D{}, A allocator = A{}); + + // Observers + [[nodiscard]] element_type* get() const noexcept; + [[nodiscard]] element_type& operator*() const noexcept; + [[nodiscard]] element_type* operator->() const noexcept; + + [[nodiscard]] ref_count_type use_count() const noexcept; + [[nodiscard]] ref_count_type strong_ref_count() const noexcept; + [[nodiscard]] ref_count_type user_ref_count() const noexcept; + explicit operator bool() const noexcept; + + class SharedPtrAccessTag { + SharedPtrAccessTag() = default; + + friend class SharedPtr; + friend class InternalSharedPtr; + }; + + void user_reference_(SharedPtrAccessTag) noexcept; + void user_dereference_(SharedPtrAccessTag) noexcept; + + private: + // Unfortunately we cannot just friend the make_internal_shared() for U = T, since that would + // constitute a partial function specialization. So instead we just friend them all, because + // friendship makes the world go 'round. + template + friend InternalSharedPtr make_internal_shared(Args&&... args); + + template + friend class InternalSharedPtr; + + struct AllocateSharedTag {}; + + template + InternalSharedPtr(AllocateSharedTag, control_block_type* ctrl_impl, U* ptr) noexcept; + + void maybe_destroy_() noexcept; + void strong_reference_() noexcept; + void strong_dereference_() noexcept; + + control_block_type* ctrl_{}; + element_type* ptr_{}; +}; + +} // namespace legate + +#include "core/utilities/internal_shared_ptr.inl" diff --git a/src/core/utilities/internal_shared_ptr.inl b/src/core/utilities/internal_shared_ptr.inl new file mode 100644 index 0000000000..1c07930c78 --- /dev/null +++ b/src/core/utilities/internal_shared_ptr.inl @@ -0,0 +1,533 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#pragma once + +#include "core/utilities/compressed_pair.h" +#include "core/utilities/internal_shared_ptr.h" + +#include "legate_defines.h" + +#include +#include +#include +#include + +namespace legate { + +namespace detail { + +// ========================================================================================== + +inline typename ControlBlockBase::ref_count_type ControlBlockBase::strong_ref_cnt() const noexcept +{ + return strong_refs_; +} + +inline typename ControlBlockBase::ref_count_type ControlBlockBase::user_ref_cnt() const noexcept +{ + return user_refs_; +} + +inline typename ControlBlockBase::ref_count_type ControlBlockBase::strong_ref() noexcept +{ + return ++strong_refs_; +} + +inline typename ControlBlockBase::ref_count_type ControlBlockBase::user_ref() noexcept +{ + return ++user_refs_; +} + +inline typename ControlBlockBase::ref_count_type ControlBlockBase::strong_deref() noexcept +{ + if (LegateDefined(LEGATE_USE_DEBUG)) assert(strong_refs_ > 0); + return --strong_refs_; +} + +inline typename ControlBlockBase::ref_count_type ControlBlockBase::user_deref() noexcept +{ + if (LegateDefined(LEGATE_USE_DEBUG)) assert(user_refs_ > 0); + return --user_refs_; +} + +// ========================================================================================== + +template +inline void ControlBlockBase::dispose_control_block_impl_(T* cb_impl) +{ + auto alloc = cb_impl->template rebind_alloc(); + using alloc_traits = std::allocator_traits>; + + // cb_impl->~T(); + alloc_traits::destroy(alloc, cb_impl); + // operator delete(cb_impl); + alloc_traits::deallocate(alloc, cb_impl, 1); +} + +// ========================================================================================== + +template +class SeparateControlBlock final : public ControlBlockBase { + public: + using value_type = T; + using deleter_type = Deleter; + using allocator_type = Alloc; + + SeparateControlBlock() = delete; + + SeparateControlBlock(value_type* ptr, deleter_type deleter, allocator_type allocator) noexcept + : ptr_{ptr}, pair_{std::move(deleter), std::move(allocator)} + { + static_assert( + std::is_nothrow_move_constructible_v, + "Deleter must be no-throw move constructible to preserve strong exception guarantee"); + static_assert( + std::is_nothrow_move_constructible_v, + "Allocator must be no-throw move constructible to preserve strong exception guarantee"); + } + + void destroy() noexcept final + { + static_assert(sizeof(value_type) > 0, "Value type must be complete at destruction"); + if (LegateDefined(LEGATE_USE_DEBUG)) assert(ptr_); + deleter_()(ptr_); + } + + void dispose_control_block() noexcept final + { + ControlBlockBase::dispose_control_block_impl_(this); + } + + [[nodiscard]] void* ptr() noexcept final { return static_cast(ptr_); } + [[nodiscard]] const void* ptr() const noexcept final { return static_cast(ptr_); } + + template + [[nodiscard]] auto rebind_alloc() const + { + using rebound_type = typename std::allocator_traits::template rebind_alloc; + return rebound_type{alloc_()}; + } + + private: + [[nodiscard]] allocator_type& alloc_() noexcept { return pair_.second(); } + [[nodiscard]] const allocator_type& alloc_() const noexcept { return pair_.second(); } + + [[nodiscard]] deleter_type& deleter_() noexcept { return pair_.first(); } + [[nodiscard]] const deleter_type& deleter_() const noexcept { return pair_.first(); } + + value_type* ptr_; + compressed_pair pair_; +}; + +template +class InplaceControlBlock final : public ControlBlockBase { + public: + using value_type = T; + using allocator_type = Allocator; + + private: + struct aligned_storage { + constexpr aligned_storage() noexcept = default; + // use this ctor to avoid zero-initializing the array + constexpr aligned_storage(std::nullptr_t) noexcept {} + + [[nodiscard]] void* addr() noexcept { return static_cast(&mem); } + [[nodiscard]] const void* addr() const noexcept { return static_cast(&mem); } + + alignas(alignof(value_type)) std::byte mem[sizeof(value_type)]; + }; + + public: + InplaceControlBlock() = delete; + + template + InplaceControlBlock(allocator_type allocator, Args&&... args) + : pair_{std::move(allocator), nullptr} + { + auto alloc = rebind_alloc(); + + // possibly throwing + std::allocator_traits>::construct( + alloc, static_cast(ptr()), std::forward(args)...); + } + + void destroy() noexcept final + { + static_assert(sizeof(value_type) > 0, "Value type must be complete at destruction"); + auto alloc = rebind_alloc(); + + std::allocator_traits>::destroy(alloc, + static_cast(ptr())); + } + + void dispose_control_block() noexcept final + { + ControlBlockBase::dispose_control_block_impl_(this); + } + + [[nodiscard]] void* ptr() noexcept final { return store_().addr(); } + [[nodiscard]] const void* ptr() const noexcept final { return store_().addr(); } + + template + [[nodiscard]] auto rebind_alloc() const + { + using rebound_type = typename std::allocator_traits::template rebind_alloc; + return rebound_type{alloc_()}; + } + + private: + [[nodiscard]] allocator_type& alloc_() noexcept { return pair_.first(); } + [[nodiscard]] const allocator_type& alloc_() const noexcept { return pair_.first(); } + + [[nodiscard]] aligned_storage& store_() noexcept { return pair_.second(); } + [[nodiscard]] const aligned_storage& store_() const noexcept { return pair_.second(); } + + compressed_pair pair_; +}; + +template +inline U* construct_from_allocator_(Alloc& allocator, P* hint, Args&&... args) +{ + using rebound_type = typename std::allocator_traits::template rebind_alloc; + using rebound_traits = std::allocator_traits; + rebound_type rebound_alloc{allocator}; + + // Don't cast result to U * (implicitly or explicitly). It is an error for the rebound + // allocator to allocate anything other than U *, so we should catch that. + auto result = rebound_traits::allocate(rebound_alloc, 1, hint); + static_assert(std::is_same_v, U*>); + // OK if the preceding lines throw, it is assumed the allocator cleans up after itself + // properly + try { + rebound_traits::construct(rebound_alloc, result, std::forward(args)...); + } catch (...) { + rebound_traits::deallocate(rebound_alloc, result, 1); + throw; + } + return result; +} + +} // namespace detail + +// ========================================================================================== + +template +void InternalSharedPtr::maybe_destroy_() noexcept +{ + if (!use_count()) { + ctrl_->destroy(); + // Do NOT delete, move, re-order, or otherwise modify the following lines under ANY + // circumstances. + // + // They must stay exactly as they are. ctrl_->dispose_control_block() calls the moral + // equivalent of "delete this", and hence any modification of ctrl_ hereafter is strictly + // undefined behavior. + // + // BEGIN DO NOT MODIFY + ctrl_->dispose_control_block(); + ctrl_ = nullptr; + ptr_ = nullptr; + // END DO NOT MODIFY + } +} + +template +void InternalSharedPtr::strong_reference_() noexcept +{ + if (ctrl_) { + if (LegateDefined(LEGATE_USE_DEBUG)) { + assert(get()); + assert(use_count()); + } + ctrl_->strong_ref(); + } +} + +template +void InternalSharedPtr::strong_dereference_() noexcept +{ + if (ctrl_) { + if (LegateDefined(LEGATE_USE_DEBUG)) assert(get()); + ctrl_->strong_deref(); + maybe_destroy_(); + } +} + +template +void InternalSharedPtr::user_reference_(SharedPtrAccessTag) noexcept +{ + if (ctrl_) { + if (LegateDefined(LEGATE_USE_DEBUG)) { + assert(get()); + assert(use_count()); + } + ctrl_->user_ref(); + } +} + +template +void InternalSharedPtr::user_dereference_(SharedPtrAccessTag) noexcept +{ + if (ctrl_) { + if (LegateDefined(LEGATE_USE_DEBUG)) assert(get()); + ctrl_->user_deref(); + maybe_destroy_(); + } +} + +template +template +inline InternalSharedPtr::InternalSharedPtr(AllocateSharedTag, + control_block_type* ctrl_impl, + U* ptr) noexcept + : ctrl_{ctrl_impl}, ptr_{ptr} +{ +} + +// ========================================================================================== + +template +inline InternalSharedPtr::InternalSharedPtr(std::nullptr_t) noexcept +{ +} + +// clang-format off +template +template +inline InternalSharedPtr::InternalSharedPtr(U* ptr, D deleter, A allocator) try : + InternalSharedPtr{ + AllocateSharedTag{}, + detail::construct_from_allocator_>(allocator, ptr, ptr, deleter, allocator), + ptr + } +{ +} +catch (...) +{ + deleter(ptr); + throw; +} +// clang-format on + +template +inline InternalSharedPtr::InternalSharedPtr(element_type* ptr) + : InternalSharedPtr{ptr, std::default_delete{}} +{ +} + +template +template +inline InternalSharedPtr::InternalSharedPtr(U* ptr) + : InternalSharedPtr{ptr, std::default_delete{}} +{ +} + +template +inline InternalSharedPtr::InternalSharedPtr(const InternalSharedPtr& other) noexcept + : InternalSharedPtr{AllocateSharedTag{}, other.ctrl_, other.ptr_} +{ + strong_reference_(); +} + +template +inline InternalSharedPtr& InternalSharedPtr::operator=( + const InternalSharedPtr& other) noexcept +{ + InternalSharedPtr tmp{other}; + + swap(tmp); + return *this; +} + +template +inline InternalSharedPtr::InternalSharedPtr(InternalSharedPtr&& other) noexcept + : InternalSharedPtr{ + AllocateSharedTag{}, std::exchange(other.ctrl_, nullptr), std::exchange(other.ptr_, nullptr)} +{ + // we do not increment ref-counts here, the other pointer gives up ownership entirely (so + // refcounts stay at previous levels) +} + +template +inline InternalSharedPtr& InternalSharedPtr::operator=(InternalSharedPtr&& other) noexcept +{ + InternalSharedPtr tmp{std::move(other)}; + + swap(tmp); + return *this; +} + +template +template +inline InternalSharedPtr::InternalSharedPtr(const InternalSharedPtr& other) noexcept + : InternalSharedPtr{AllocateSharedTag{}, other.ctrl_, other.ptr_} +{ + strong_reference_(); +} + +template +template +inline InternalSharedPtr& InternalSharedPtr::operator=( + const InternalSharedPtr& other) noexcept +{ + InternalSharedPtr tmp{other}; + + swap(tmp); + return *this; +} + +template +template +inline InternalSharedPtr::InternalSharedPtr(InternalSharedPtr&& other) noexcept + : InternalSharedPtr{ + AllocateSharedTag{}, std::exchange(other.ctrl_, nullptr), std::exchange(other.ptr_, nullptr)} +{ +} + +template +template +inline InternalSharedPtr& InternalSharedPtr::operator=(InternalSharedPtr&& other) noexcept +{ + InternalSharedPtr tmp{std::move(other)}; + + swap(tmp); + return *this; +} + +template +template +inline InternalSharedPtr::InternalSharedPtr(std::unique_ptr&& ptr) + : InternalSharedPtr{ptr.get(), std::move(ptr.get_deleter())} +{ + // Release only after we have fully constructed ourselves to preserve strong exception + // guarantee. If the above constructor throws, this has no effect. + ptr.release(); +} + +template +template +inline InternalSharedPtr& InternalSharedPtr::operator=(std::unique_ptr&& ptr) +{ + InternalSharedPtr{std::move(ptr)}.swap(*this); + return *this; +} + +template +inline InternalSharedPtr::~InternalSharedPtr() noexcept +{ + strong_dereference_(); +} + +// ========================================================================================== + +template +inline void InternalSharedPtr::swap(InternalSharedPtr& other) noexcept +{ + using std::swap; + + swap(other.ctrl_, ctrl_); + swap(other.ptr_, ptr_); +} + +template +inline void swap(InternalSharedPtr& lhs, InternalSharedPtr& rhs) noexcept +{ + lhs.swap(rhs); +} + +template +inline void InternalSharedPtr::reset() noexcept +{ + InternalSharedPtr{}.swap(*this); +} + +template +inline void InternalSharedPtr::reset(std::nullptr_t) noexcept +{ + reset(); +} + +template +template +inline void InternalSharedPtr::reset(U* ptr, D deleter, A allocator) +{ + InternalSharedPtr{ptr, std::move(deleter), std::move(allocator)}.swap(*this); +} + +// ========================================================================================== + +template +inline typename InternalSharedPtr::element_type* InternalSharedPtr::get() const noexcept +{ + return ptr_; +} + +template +inline typename InternalSharedPtr::element_type& InternalSharedPtr::operator*() const noexcept +{ + return *ptr_; +} + +template +inline typename InternalSharedPtr::element_type* InternalSharedPtr::operator->() + const noexcept +{ + return get(); +} + +template +inline typename InternalSharedPtr::ref_count_type InternalSharedPtr::use_count() + const noexcept +{ + // SharedPtr's are a subset of InternalSharedPtr (since each one holds an InternalSharedPtr), + // so the number of strong references gives the total unique references held to the pointer. + return strong_ref_count(); +} + +template +inline typename InternalSharedPtr::ref_count_type InternalSharedPtr::strong_ref_count() + const noexcept +{ + return ctrl_ ? ctrl_->strong_ref_cnt() : 0; +} + +template +inline typename InternalSharedPtr::ref_count_type InternalSharedPtr::user_ref_count() + const noexcept +{ + return ctrl_ ? ctrl_->user_ref_cnt() : 0; +} + +template +inline InternalSharedPtr::operator bool() const noexcept +{ + return get() != nullptr; +} + +// ========================================================================================== + +template +inline InternalSharedPtr make_internal_shared(Args&&... args) +{ + using allocator_type = std::allocator; + + auto alloc = allocator_type{}; + auto control_block = + detail::construct_from_allocator_>( + alloc, static_cast(nullptr), alloc, std::forward(args)...); + return {typename InternalSharedPtr::AllocateSharedTag{}, + control_block, + static_cast(control_block->ptr())}; +} + +} // namespace legate diff --git a/src/core/utilities/memory.h b/src/core/utilities/memory.h index 86ab4f2ac6..98648866fd 100644 --- a/src/core/utilities/memory.h +++ b/src/core/utilities/memory.h @@ -36,3 +36,6 @@ struct default_delete { } // namespace legate #include "core/utilities/memory.inl" + +#include "core/utilities/internal_shared_ptr.h" +#include "core/utilities/shared_ptr.h" diff --git a/src/core/utilities/shared_ptr.h b/src/core/utilities/shared_ptr.h new file mode 100644 index 0000000000..9865ef9a15 --- /dev/null +++ b/src/core/utilities/shared_ptr.h @@ -0,0 +1,119 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#pragma once + +#include "core/utilities/internal_shared_ptr.h" + +namespace legate { + +template +class SharedPtr; + +template +void swap(SharedPtr&, SharedPtr&) noexcept; + +template +class SharedPtr { + using internal_ptr_type = InternalSharedPtr; + + public: + using element_type = typename internal_ptr_type::element_type; + using ref_count_type = typename internal_ptr_type::ref_count_type; + + // Constructors + constexpr SharedPtr() noexcept = default; + + SharedPtr(std::nullptr_t) noexcept; + + template > + SharedPtr(U* ptr, Deleter deleter, Alloc allocator = Alloc{}); + template + explicit SharedPtr(U* ptr); + + SharedPtr(const SharedPtr&) noexcept; + SharedPtr& operator=(const SharedPtr&) noexcept; + SharedPtr(SharedPtr&&) noexcept; + SharedPtr& operator=(SharedPtr&&) noexcept; + + template + SharedPtr(const SharedPtr&) noexcept; + template + SharedPtr& operator=(const SharedPtr&) noexcept; + template + SharedPtr(SharedPtr&&) noexcept; + template + SharedPtr& operator=(SharedPtr&&) noexcept; + + explicit SharedPtr(const InternalSharedPtr&) noexcept; + SharedPtr& operator=(const InternalSharedPtr&) noexcept; + explicit SharedPtr(InternalSharedPtr&&) noexcept; + SharedPtr& operator=(InternalSharedPtr&&) noexcept; + + template + explicit SharedPtr(const InternalSharedPtr&) noexcept; + template + SharedPtr& operator=(const InternalSharedPtr&) noexcept; + template + explicit SharedPtr(InternalSharedPtr&&) noexcept; + template + SharedPtr& operator=(InternalSharedPtr&&) noexcept; + + template + SharedPtr(std::unique_ptr&&); + template + SharedPtr& operator=(std::unique_ptr&&); + + ~SharedPtr() noexcept; + + // Modifiers + void swap(SharedPtr&) noexcept; + // must namespace qualify to disambiguate from member function, otherwise clang balks + friend void ::legate::swap<>(SharedPtr&, SharedPtr&) noexcept; + void reset() noexcept; + void reset(std::nullptr_t) noexcept; + template , typename A = std::allocator> + void reset(U* ptr, D deleter = D{}, A allocator = A{}); + + // Observers + [[nodiscard]] element_type* get() const noexcept; + [[nodiscard]] element_type& operator*() const noexcept; + [[nodiscard]] element_type* operator->() const noexcept; + + [[nodiscard]] ref_count_type use_count() const noexcept; + explicit operator bool() const noexcept; + + private: + struct copy_tag {}; + struct move_tag {}; + + template + SharedPtr(copy_tag, const InternalSharedPtr& other) noexcept; + template + SharedPtr(move_tag, InternalSharedPtr&& other, bool from_internal_ptr) noexcept; + template + void assign_(copy_tag, const InternalSharedPtr&) noexcept; + template + void assign_(move_tag, InternalSharedPtr&&) noexcept; + + template + friend class SharedPtr; + + void reference_() noexcept; + void dereference_() noexcept; + + internal_ptr_type ptr_{}; +}; + +} // namespace legate + +#include "core/utilities/shared_ptr.inl" diff --git a/src/core/utilities/shared_ptr.inl b/src/core/utilities/shared_ptr.inl new file mode 100644 index 0000000000..10fb42286d --- /dev/null +++ b/src/core/utilities/shared_ptr.inl @@ -0,0 +1,297 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#pragma once + +#include "core/utilities/shared_ptr.h" + +namespace legate { + +template +inline void SharedPtr::reference_() noexcept +{ + ptr_.user_reference_({}); +} + +template +inline void SharedPtr::dereference_() noexcept +{ + ptr_.user_dereference_({}); +} + +template +template +inline SharedPtr::SharedPtr(copy_tag, const InternalSharedPtr& other) noexcept : ptr_{other} +{ + reference_(); +} + +template +template +inline SharedPtr::SharedPtr(move_tag, + InternalSharedPtr&& other, + bool from_internal_ptr) noexcept + : ptr_{std::move(other)} +{ + // Only update refcount if we are constructing from a bare InternalSharedPointer, since + // the previous owning SharedPtr gives ownership entirely + if (from_internal_ptr) reference_(); +} + +template +template +inline void SharedPtr::assign_(copy_tag, const InternalSharedPtr& other) noexcept +{ + SharedPtr tmp{other}; + + swap(tmp); +} + +template +template +inline void SharedPtr::assign_(move_tag, InternalSharedPtr&& other) noexcept +{ + SharedPtr tmp{std::move(other)}; + + swap(tmp); +} + +// ========================================================================================== + +template +inline SharedPtr::SharedPtr(std::nullptr_t) noexcept +{ +} + +template +template +inline SharedPtr::SharedPtr(U* ptr, Deleter deleter, Alloc allocator) + : ptr_{ptr, std::move(deleter), std::move(allocator)} +{ + reference_(); +} + +template +template +inline SharedPtr::SharedPtr(U* ptr) : SharedPtr{ptr, std::default_delete{}} +{ +} + +template +inline SharedPtr::SharedPtr(const SharedPtr& other) noexcept : SharedPtr{copy_tag{}, other.ptr_} +{ +} + +template +inline SharedPtr& SharedPtr::operator=(const SharedPtr& other) noexcept +{ + assign_(copy_tag{}, other.ptr_); + return *this; +} + +template +inline SharedPtr::SharedPtr(SharedPtr&& other) noexcept + : SharedPtr{move_tag{}, std::move(other.ptr_), false} +{ +} + +template +inline SharedPtr& SharedPtr::operator=(SharedPtr&& other) noexcept +{ + assign_(move_tag{}, std::move(other.ptr_)); + return *this; +} + +template +template +inline SharedPtr::SharedPtr(const SharedPtr& other) noexcept + : SharedPtr{copy_tag{}, other.ptr_} +{ +} + +template +template +inline SharedPtr& SharedPtr::operator=(const SharedPtr& other) noexcept +{ + assign_(copy_tag{}, other.ptr_); + return *this; +} + +template +template +inline SharedPtr::SharedPtr(SharedPtr&& other) noexcept + : SharedPtr{move_tag{}, std::move(other.ptr_), false} +{ +} + +template +template +inline SharedPtr& SharedPtr::operator=(SharedPtr&& other) noexcept +{ + assign_(move_tag{}, std::move(other.ptr_)); + return *this; +} + +template +inline SharedPtr::SharedPtr(const InternalSharedPtr& other) noexcept + : SharedPtr{copy_tag{}, other} +{ +} + +template +inline SharedPtr& SharedPtr::operator=(const InternalSharedPtr& other) noexcept +{ + assign_(copy_tag{}, other); + return *this; +} + +template +inline SharedPtr::SharedPtr(InternalSharedPtr&& other) noexcept + : SharedPtr{move_tag{}, std::move(other), true} +{ +} + +template +inline SharedPtr& SharedPtr::operator=(InternalSharedPtr&& other) noexcept +{ + assign_(move_tag{}, std::move(other)); + return *this; +} + +template +template +inline SharedPtr::SharedPtr(const InternalSharedPtr& other) noexcept + : SharedPtr{copy_tag{}, other} +{ +} + +template +template +inline SharedPtr& SharedPtr::operator=(const InternalSharedPtr& other) noexcept +{ + assign_(copy_tag{}, other); + return *this; +} + +template +template +inline SharedPtr::SharedPtr(InternalSharedPtr&& other) noexcept + : SharedPtr{move_tag{}, std::move(other), true} +{ +} + +template +template +inline SharedPtr& SharedPtr::operator=(InternalSharedPtr&& other) noexcept +{ + assign_(move_tag{}, std::move(other)); + return *this; +} + +template +template +inline SharedPtr::SharedPtr(std::unique_ptr&& ptr) : ptr_{std::move(ptr)} +{ + reference_(); +} + +template +template +inline SharedPtr& SharedPtr::operator=(std::unique_ptr&& ptr) +{ + ptr_ = std::move(ptr); + reference_(); + return *this; +} + +template +inline SharedPtr::~SharedPtr() noexcept +{ + dereference_(); +} + +// ========================================================================================== + +template +inline void SharedPtr::swap(SharedPtr& other) noexcept +{ + ptr_.swap(other.ptr_); +} + +// friend function +template +inline void swap(SharedPtr& lhs, SharedPtr& rhs) noexcept +{ + lhs.swap(rhs); +} + +template +inline void SharedPtr::reset() noexcept +{ + SharedPtr{}.swap(*this); +} + +template +inline void SharedPtr::reset(std::nullptr_t) noexcept +{ + SharedPtr{nullptr}.swap(*this); +} + +template +template +inline void SharedPtr::reset(U* ptr, D deleter, A allocator) +{ + // cannot call ptr_.reset() since we may need to bump the user and strong reference counts + SharedPtr{ptr, std::move(deleter), std::move(allocator)}.swap(*this); +} + +// ========================================================================================== + +template +inline typename SharedPtr::element_type* SharedPtr::get() const noexcept +{ + return ptr_.get(); +} + +template +inline typename SharedPtr::element_type& SharedPtr::operator*() const noexcept +{ + return ptr_.operator*(); +} + +template +inline typename SharedPtr::element_type* SharedPtr::operator->() const noexcept +{ + return ptr_.operator->(); +} + +template +inline typename SharedPtr::ref_count_type SharedPtr::use_count() const noexcept +{ + return ptr_.use_count(); +} + +template +inline SharedPtr::operator bool() const noexcept +{ + return ptr_.operator bool(); +} + +// ========================================================================================== + +template +inline SharedPtr make_shared(Args&&... args) +{ + return SharedPtr{make_internal_shared(std::forward(args)...)}; +} + +} // namespace legate diff --git a/tests/cpp/CMakeLists.txt b/tests/cpp/CMakeLists.txt index d6918c2008..4c211887c7 100644 --- a/tests/cpp/CMakeLists.txt +++ b/tests/cpp/CMakeLists.txt @@ -20,8 +20,8 @@ if (NOT CMAKE_CXX_STANDARD) set(CMAKE_CXX_STANDARD 17) endif() -set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g") -set(CMAKE_CUDA_FLAGS_DEBUG "-O0 -g") +# set(CMAKE_CXX_FLAGS_DEBUG "-O0 -g") +# set(CMAKE_CUDA_FLAGS_DEBUG "-O0 -g") set(CMAKE_CXX_FLAGS_RELEASE "-O2") set(CMAKE_CUDA_FLAGS_RELEASE "-O2") set(CMAKE_CXX_FLAGS_MINSIZEREL "-Os") diff --git a/tests/cpp/unit/internal_shared_ptr.cc b/tests/cpp/unit/internal_shared_ptr.cc new file mode 100644 index 0000000000..2fcbbdbb8c --- /dev/null +++ b/tests/cpp/unit/internal_shared_ptr.cc @@ -0,0 +1,295 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include "core/utilities/internal_shared_ptr.h" + +#include "shared_ptr_util.h" + +template +struct InternalSharedPtrUnit : BasicSharedPtrUnit {}; + +TYPED_TEST_SUITE(InternalSharedPtrUnit, BasicSharedPtrTypeList); + +TYPED_TEST(InternalSharedPtrUnit, CreateBasic) +{ + legate::InternalSharedPtr ptr; + + test_basic_equal(ptr, static_cast(nullptr)); +} + +TYPED_TEST(InternalSharedPtrUnit, CreateNullptrT) +{ + legate::InternalSharedPtr ptr{nullptr}; + + test_basic_equal(ptr, static_cast(nullptr)); +} + +TYPED_TEST(InternalSharedPtrUnit, CreateWithPtr) +{ + auto sh_ptr = new TypeParam{1}; + legate::InternalSharedPtr ptr{sh_ptr}; + + EXPECT_EQ(ptr.use_count(), 1); + test_basic_equal(ptr, sh_ptr); +} + +TYPED_TEST(InternalSharedPtrUnit, CreateWithCopyEqCtor) +{ + auto bare_ptr = new TypeParam{1}; + legate::InternalSharedPtr ptr1{bare_ptr}; + legate::InternalSharedPtr ptr2 = ptr1; + + test_create_with_copy_n({ptr1, ptr2}, bare_ptr); +} + +// same test as above, but using {} constructor +TYPED_TEST(InternalSharedPtrUnit, CreateWithCopyBraceCtor) +{ + auto bare_ptr = new TypeParam{1}; + legate::InternalSharedPtr ptr1{bare_ptr}; + legate::InternalSharedPtr ptr2{ptr1}; + + test_create_with_copy_n({ptr1, ptr2}, bare_ptr); +} + +TYPED_TEST(InternalSharedPtrUnit, CascadingCopyEqCtor) +{ + auto bare_ptr = new TypeParam{1}; + legate::InternalSharedPtr ptr1{bare_ptr}; + { + legate::InternalSharedPtr ptr2 = ptr1; + + test_create_with_copy_n({ptr1, ptr2}, bare_ptr); + { + legate::InternalSharedPtr ptr3 = ptr2; + + test_create_with_copy_n({ptr1, ptr2, ptr3}, bare_ptr); + } + // ensure that ref counts have decreased again + test_create_with_copy_n({ptr1, ptr2}, bare_ptr); + { + // note initializing with ptr1 now + legate::InternalSharedPtr ptr3 = ptr1; + + test_create_with_copy_n({ptr1, ptr2, ptr3}, bare_ptr); + } + // ensure that ref counts have decreased again + test_create_with_copy_n({ptr1, ptr2}, bare_ptr); + } + // ensure that ref counts have decreased again + test_create_with_copy_n({ptr1}, bare_ptr); +} + +TYPED_TEST(InternalSharedPtrUnit, CascadingCopyBraceCtor) +{ + auto bare_ptr = new TypeParam{1}; + legate::InternalSharedPtr ptr1{bare_ptr}; + { + legate::InternalSharedPtr ptr2{ptr1}; + + test_create_with_copy_n({ptr1, ptr2}, bare_ptr); + { + legate::InternalSharedPtr ptr3{ptr2}; + + test_create_with_copy_n({ptr1, ptr2, ptr3}, bare_ptr); + } + // ensure that ref counts have decreased again + test_create_with_copy_n({ptr1, ptr2}, bare_ptr); + { + // note initializing with ptr1 now + legate::InternalSharedPtr ptr3{ptr1}; + + test_create_with_copy_n({ptr1, ptr2, ptr3}, bare_ptr); + } + // ensure that ref counts have decreased again + test_create_with_copy_n({ptr1, ptr2}, bare_ptr); + } + // ensure that ref counts have decreased again + test_create_with_copy_n({ptr1}, bare_ptr); +} + +TYPED_TEST(InternalSharedPtrUnit, MoveCtor) +{ + auto bare_ptr = new TypeParam{1}; + legate::InternalSharedPtr ptr1{bare_ptr}; + + test_basic_equal(ptr1, bare_ptr); + + legate::InternalSharedPtr ptr2 = std::move(ptr1); + + EXPECT_EQ(ptr2.use_count(), 1); + test_basic_equal(ptr2, bare_ptr); + test_basic_equal(ptr1, static_cast(nullptr)); +} + +TYPED_TEST(InternalSharedPtrUnit, MoveAssign) +{ + auto bare_ptr = new TypeParam{1}; + legate::InternalSharedPtr ptr1{bare_ptr}; + + test_basic_equal(ptr1, bare_ptr); + + legate::InternalSharedPtr ptr2{std::move(ptr1)}; + + EXPECT_EQ(ptr2.use_count(), 1); + test_basic_equal(ptr2, bare_ptr); + test_basic_equal(ptr1, static_cast(nullptr)); +} + +TYPED_TEST(InternalSharedPtrUnit, SelfAssign) +{ + auto bare_ptr = new TypeParam{1}; + legate::InternalSharedPtr ptr1{bare_ptr}; + // Use this silence compiler warnings about self-assignment, as that is indeed the point of + // this test. + auto hide_self_assign = [](auto& lhs, auto& rhs) { lhs = rhs; }; + + hide_self_assign(ptr1, ptr1); + EXPECT_EQ(ptr1.use_count(), 1); + test_basic_equal(ptr1, bare_ptr); +} + +TYPED_TEST(InternalSharedPtrUnit, SelfMoveAssign) +{ + auto bare_ptr = new TypeParam{1}; + legate::InternalSharedPtr ptr1{bare_ptr}; + // Use this silence compiler warnings about self-assignment, as that is indeed the point of + // this test. + auto hide_self_assign = [](auto& lhs, auto& rhs) { lhs = std::move(rhs); }; + + hide_self_assign(ptr1, ptr1); + EXPECT_EQ(ptr1.use_count(), 1); + test_basic_equal(ptr1, bare_ptr); +} + +TYPED_TEST(InternalSharedPtrUnit, Reset) +{ + auto bare_ptr = new TypeParam{1}; + legate::InternalSharedPtr ptr1{bare_ptr}; + + test_basic_equal(ptr1, bare_ptr); + ptr1.reset(); + test_basic_equal(ptr1, static_cast(nullptr)); +} + +TYPED_TEST(InternalSharedPtrUnit, ResetNullPtrT) +{ + auto bare_ptr = new TypeParam{11}; + legate::InternalSharedPtr ptr1{bare_ptr}; + + test_basic_equal(ptr1, bare_ptr); + ptr1.reset(nullptr); + test_basic_equal(ptr1, static_cast(nullptr)); +} + +TYPED_TEST(InternalSharedPtrUnit, ResetOther) +{ + auto bare_ptr1 = new TypeParam{1}; + legate::InternalSharedPtr ptr1{bare_ptr1}; + + test_basic_equal(ptr1, bare_ptr1); + auto bare_ptr2 = new TypeParam{88}; + ptr1.reset(bare_ptr2); + test_basic_equal(ptr1, bare_ptr2); +} + +TEST(InternalSharedPtrUnit, BasicPolymorphism) +{ + auto bare_ptr = new BasicDerived{}; + legate::InternalSharedPtr ptr{bare_ptr}; + + test_basic_equal(ptr, bare_ptr); +} + +TEST(InternalSharedPtrUnit, Polymorphism) +{ + bool toggle = false; + { + auto bare_ptr = new TogglingDerived{&toggle}; + legate::InternalSharedPtr ptr{bare_ptr}; + + ASSERT_FALSE(toggle); // sanity check + test_basic_equal(ptr, bare_ptr); + ASSERT_FALSE(toggle); // still false + ASSERT_EQ(ptr.use_count(), 1); + } + ASSERT_TRUE(toggle); // if properly handled, set to true in most derived dtor +} + +TEST(InternalSharedPtrUnit, PolymorphismReset) +{ + bool toggle = false; + { + auto bare_ptr = new TogglingDerived{&toggle}; + legate::InternalSharedPtr ptr{bare_ptr}; + + ASSERT_FALSE(toggle); // sanity check + test_basic_equal(ptr, bare_ptr); + ASSERT_FALSE(toggle); // still false + ASSERT_EQ(ptr.use_count(), 1); + + auto bare_ptr2 = new BasicDerived{45}; + + ptr.reset(bare_ptr2); + ASSERT_TRUE(toggle); // if properly handled, set to true in most derived dtor + test_basic_equal(ptr, bare_ptr2); + toggle = false; + } + ASSERT_FALSE(toggle); // should not have been touched +} + +TYPED_TEST(InternalSharedPtrUnit, MakeShared) +{ + auto sh_ptr = legate::make_internal_shared(10); + auto bare_ptr = sh_ptr.get(); + + test_basic_equal(sh_ptr, bare_ptr); +} + +TEST(InternalSharedPtrUnit, MakeSharedPolymorphism) +{ + legate::InternalSharedPtr sh_ptr = legate::make_internal_shared(10); + auto bare_ptr = static_cast(sh_ptr.get()); + + test_basic_equal(sh_ptr, bare_ptr); +} + +TYPED_TEST(InternalSharedPtrUnit, UniqueCtor) +{ + auto val = TypeParam{123}; + auto uniq = std::make_unique(val); + + legate::InternalSharedPtr sh_ptr{std::move(uniq)}; + + ASSERT_EQ(sh_ptr.use_count(), 1); + ASSERT_EQ(*sh_ptr, val); + ASSERT_EQ(uniq.get(), nullptr); + ASSERT_FALSE(uniq); +} + +TYPED_TEST(InternalSharedPtrUnit, UniqueAssign) +{ + auto val = TypeParam{123}; + auto uniq = std::make_unique(val); + + auto bare_ptr = new TypeParam{22}; + legate::InternalSharedPtr sh_ptr{bare_ptr}; + + test_basic_equal(sh_ptr, bare_ptr); + + sh_ptr = std::move(uniq); + + ASSERT_EQ(sh_ptr.use_count(), 1); + ASSERT_EQ(*sh_ptr, val); + ASSERT_EQ(uniq.get(), nullptr); + ASSERT_FALSE(uniq); +} diff --git a/tests/cpp/unit/logical_store.cc b/tests/cpp/unit/logical_store.cc index 91a388ac6f..3d308b6f2c 100644 --- a/tests/cpp/unit/logical_store.cc +++ b/tests/cpp/unit/logical_store.cc @@ -60,7 +60,7 @@ TEST_F(Store, Creation) EXPECT_EQ(store.type(), legate::int64()); EXPECT_FALSE(store.transformed()); } - EXPECT_THROW(runtime->create_store(legate::Scalar(int64_t{123}), {1, 2}), + EXPECT_THROW((void)runtime->create_store(legate::Scalar(int64_t{123}), {1, 2}), std::invalid_argument); } } diff --git a/tests/cpp/unit/shared_ptr.cc b/tests/cpp/unit/shared_ptr.cc new file mode 100644 index 0000000000..0aed1f56b9 --- /dev/null +++ b/tests/cpp/unit/shared_ptr.cc @@ -0,0 +1,470 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#include "core/utilities/shared_ptr.h" + +#include "shared_ptr_util.h" + +template +struct SharedPtrUnit : BasicSharedPtrUnit {}; + +TYPED_TEST_SUITE(SharedPtrUnit, BasicSharedPtrTypeList); + +TYPED_TEST(SharedPtrUnit, CreateBasic) +{ + legate::SharedPtr ptr; + + test_basic_equal(ptr, static_cast(nullptr)); +} + +TYPED_TEST(SharedPtrUnit, CreateNullptrT) +{ + legate::SharedPtr ptr{nullptr}; + + test_basic_equal(ptr, static_cast(nullptr)); +} + +TYPED_TEST(SharedPtrUnit, CreateWithPtr) +{ + auto sh_ptr = new TypeParam{1}; + legate::SharedPtr ptr{sh_ptr}; + + EXPECT_EQ(ptr.use_count(), 1); + test_basic_equal(ptr, sh_ptr); +} + +TYPED_TEST(SharedPtrUnit, CreateWithCopyEqCtor) +{ + auto bare_ptr = new TypeParam{1}; + legate::SharedPtr ptr1{bare_ptr}; + legate::SharedPtr ptr2 = ptr1; + + test_create_with_copy_n({ptr1, ptr2}, bare_ptr); +} + +// same test as above, but using {} constructor +TYPED_TEST(SharedPtrUnit, CreateWithCopyBraceCtor) +{ + auto bare_ptr = new TypeParam{1}; + legate::SharedPtr ptr1{bare_ptr}; + legate::SharedPtr ptr2{ptr1}; + + test_create_with_copy_n({ptr1, ptr2}, bare_ptr); +} + +TYPED_TEST(SharedPtrUnit, CascadingCopyEqCtor) +{ + auto bare_ptr = new TypeParam{1}; + legate::SharedPtr ptr1{bare_ptr}; + { + legate::SharedPtr ptr2 = ptr1; + + test_create_with_copy_n({ptr1, ptr2}, bare_ptr); + { + legate::SharedPtr ptr3 = ptr2; + + test_create_with_copy_n({ptr1, ptr2, ptr3}, bare_ptr); + } + // ensure that ref counts have decreased again + test_create_with_copy_n({ptr1, ptr2}, bare_ptr); + { + // note initializing with ptr1 now + legate::SharedPtr ptr3 = ptr1; + + test_create_with_copy_n({ptr1, ptr2, ptr3}, bare_ptr); + } + // ensure that ref counts have decreased again + test_create_with_copy_n({ptr1, ptr2}, bare_ptr); + } + // ensure that ref counts have decreased again + test_create_with_copy_n({ptr1}, bare_ptr); +} + +TYPED_TEST(SharedPtrUnit, CascadingCopyBraceCtor) +{ + auto bare_ptr = new TypeParam{1}; + legate::SharedPtr ptr1{bare_ptr}; + { + legate::SharedPtr ptr2{ptr1}; + + test_create_with_copy_n({ptr1, ptr2}, bare_ptr); + { + legate::SharedPtr ptr3{ptr2}; + + test_create_with_copy_n({ptr1, ptr2, ptr3}, bare_ptr); + } + // ensure that ref counts have decreased again + test_create_with_copy_n({ptr1, ptr2}, bare_ptr); + { + // note initializing with ptr1 now + legate::SharedPtr ptr3{ptr1}; + + test_create_with_copy_n({ptr1, ptr2, ptr3}, bare_ptr); + } + // ensure that ref counts have decreased again + test_create_with_copy_n({ptr1, ptr2}, bare_ptr); + } + // ensure that ref counts have decreased again + test_create_with_copy_n({ptr1}, bare_ptr); +} + +TYPED_TEST(SharedPtrUnit, MoveCtor) +{ + auto bare_ptr = new TypeParam{1}; + legate::SharedPtr ptr1{bare_ptr}; + + test_basic_equal(ptr1, bare_ptr); + + legate::SharedPtr ptr2 = std::move(ptr1); + + EXPECT_EQ(ptr2.use_count(), 1); + test_basic_equal(ptr2, bare_ptr); + test_basic_equal(ptr1, static_cast(nullptr)); +} + +TYPED_TEST(SharedPtrUnit, MoveAssign) +{ + auto bare_ptr = new TypeParam{1}; + legate::SharedPtr ptr1{bare_ptr}; + + test_basic_equal(ptr1, bare_ptr); + + legate::SharedPtr ptr2{std::move(ptr1)}; + + EXPECT_EQ(ptr2.use_count(), 1); + test_basic_equal(ptr2, bare_ptr); + test_basic_equal(ptr1, static_cast(nullptr)); +} + +TYPED_TEST(SharedPtrUnit, SelfAssign) +{ + auto bare_ptr = new TypeParam{1}; + legate::SharedPtr ptr1{bare_ptr}; + // Use this silence compiler warnings about self-assignment, as that is indeed the point of + // this test. + auto hide_self_assign = [](auto& lhs, auto& rhs) { lhs = rhs; }; + + hide_self_assign(ptr1, ptr1); + EXPECT_EQ(ptr1.use_count(), 1); + test_basic_equal(ptr1, bare_ptr); +} + +TYPED_TEST(SharedPtrUnit, SelfMoveAssign) +{ + auto bare_ptr = new TypeParam{1}; + legate::SharedPtr ptr1{bare_ptr}; + // Use this silence compiler warnings about self-assignment, as that is indeed the point of + // this test. + auto hide_self_assign = [](auto& lhs, auto& rhs) { lhs = std::move(rhs); }; + + hide_self_assign(ptr1, ptr1); + EXPECT_EQ(ptr1.use_count(), 1); + test_basic_equal(ptr1, bare_ptr); +} + +TYPED_TEST(SharedPtrUnit, Reset) +{ + auto bare_ptr = new TypeParam{1}; + legate::SharedPtr ptr1{bare_ptr}; + + test_basic_equal(ptr1, bare_ptr); + ptr1.reset(); + test_basic_equal(ptr1, static_cast(nullptr)); +} + +TYPED_TEST(SharedPtrUnit, ResetNullPtrT) +{ + auto bare_ptr = new TypeParam{11}; + legate::SharedPtr ptr1{bare_ptr}; + + test_basic_equal(ptr1, bare_ptr); + ptr1.reset(nullptr); + test_basic_equal(ptr1, static_cast(nullptr)); +} + +TYPED_TEST(SharedPtrUnit, ResetOther) +{ + auto bare_ptr1 = new TypeParam{1}; + legate::SharedPtr ptr1{bare_ptr1}; + + test_basic_equal(ptr1, bare_ptr1); + auto bare_ptr2 = new TypeParam{88}; + ptr1.reset(bare_ptr2); + test_basic_equal(ptr1, bare_ptr2); +} + +TEST(SharedPtrUnit, BasicPolymorphism) +{ + auto bare_ptr = new BasicDerived{}; + legate::SharedPtr ptr{bare_ptr}; + + test_basic_equal(ptr, bare_ptr); +} + +TEST(SharedPtrUnit, Polymorphism) +{ + bool toggle = false; + { + auto bare_ptr = new TogglingDerived{&toggle}; + legate::SharedPtr ptr{bare_ptr}; + + ASSERT_FALSE(toggle); // sanity check + test_basic_equal(ptr, bare_ptr); + ASSERT_FALSE(toggle); // still false + ASSERT_EQ(ptr.use_count(), 1); + } + ASSERT_TRUE(toggle); // if properly handled, set to true in most derived dtor +} + +TYPED_TEST(SharedPtrUnit, CreateFromInternal) +{ + auto bare_ptr = new TypeParam{1}; + legate::InternalSharedPtr itrnl_ptr{bare_ptr}; + legate::SharedPtr sh_ptr{itrnl_ptr}; + + ASSERT_EQ(itrnl_ptr.strong_ref_count(), 2); + ASSERT_EQ(itrnl_ptr.user_ref_count(), 1); + ASSERT_EQ(itrnl_ptr.use_count(), 2); + ASSERT_EQ(sh_ptr.use_count(), itrnl_ptr.use_count()); + ASSERT_EQ(sh_ptr.get(), itrnl_ptr.get()); + ASSERT_EQ(*sh_ptr, *itrnl_ptr); + ASSERT_EQ(static_cast(sh_ptr), static_cast(itrnl_ptr)); + test_basic_equal(itrnl_ptr, bare_ptr); + test_basic_equal(sh_ptr, bare_ptr); +} + +TYPED_TEST(SharedPtrUnit, FromOrphanInternalCopy) +{ + auto bare_ptr = new TypeParam{1}; + legate::SharedPtr sh_ptr; + + // sanity checks + test_basic_equal(sh_ptr, static_cast(nullptr)); + { + legate::InternalSharedPtr itrnl_ptr{bare_ptr}; + + sh_ptr = itrnl_ptr; + ASSERT_EQ(itrnl_ptr.strong_ref_count(), 2); + ASSERT_EQ(itrnl_ptr.user_ref_count(), 1); + ASSERT_EQ(itrnl_ptr.use_count(), 2); + ASSERT_EQ(sh_ptr.use_count(), 2); + ASSERT_EQ(sh_ptr.use_count(), itrnl_ptr.use_count()); + ASSERT_EQ(sh_ptr.get(), itrnl_ptr.get()); + ASSERT_EQ(*sh_ptr, *itrnl_ptr); + ASSERT_EQ(static_cast(sh_ptr), static_cast(itrnl_ptr)); + test_basic_equal(itrnl_ptr, bare_ptr); + test_basic_equal(sh_ptr, bare_ptr); + } + // the pointer should not die here + ASSERT_EQ(sh_ptr.use_count(), 1); + test_basic_equal(sh_ptr, bare_ptr); +} + +TYPED_TEST(SharedPtrUnit, FromOrphanInternalCopyCascading) +{ + auto bare_ptr = new TypeParam{1}; + auto bare_ptr2 = new TypeParam{2}; + legate::SharedPtr sh_ptr; + + // sanity checks + test_basic_equal(sh_ptr, static_cast(nullptr)); + { + legate::InternalSharedPtr itrnl_ptr{bare_ptr}; + + sh_ptr = itrnl_ptr; + ASSERT_EQ(itrnl_ptr.strong_ref_count(), 2); + ASSERT_EQ(itrnl_ptr.user_ref_count(), 1); + ASSERT_EQ(itrnl_ptr.use_count(), 2); + ASSERT_EQ(sh_ptr.use_count(), 2); + ASSERT_EQ(sh_ptr.use_count(), itrnl_ptr.use_count()); + ASSERT_EQ(sh_ptr.get(), itrnl_ptr.get()); + ASSERT_EQ(*sh_ptr, *itrnl_ptr); + ASSERT_EQ(static_cast(sh_ptr), static_cast(itrnl_ptr)); + test_basic_equal(itrnl_ptr, bare_ptr); + test_basic_equal(sh_ptr, bare_ptr); + { + legate::InternalSharedPtr itrnl_ptr2{bare_ptr2}; + + sh_ptr = itrnl_ptr2; + ASSERT_EQ(itrnl_ptr2.strong_ref_count(), 2); + ASSERT_EQ(itrnl_ptr2.user_ref_count(), 1); + ASSERT_EQ(itrnl_ptr2.use_count(), 2); + ASSERT_EQ(sh_ptr.use_count(), 2); + ASSERT_EQ(sh_ptr.use_count(), itrnl_ptr2.use_count()); + ASSERT_EQ(sh_ptr.get(), itrnl_ptr2.get()); + ASSERT_EQ(*sh_ptr, *itrnl_ptr2); + ASSERT_EQ(static_cast(sh_ptr), static_cast(itrnl_ptr2)); + test_basic_equal(itrnl_ptr2, bare_ptr2); + test_basic_equal(sh_ptr, bare_ptr2); + } + } + // the pointer should not die here + ASSERT_EQ(sh_ptr.use_count(), 1); + test_basic_equal(sh_ptr, bare_ptr2); +} + +TYPED_TEST(SharedPtrUnit, FromOrphanInternalMove) +{ + auto bare_ptr = new TypeParam{1}; + legate::SharedPtr sh_ptr; + + // sanity checks + test_basic_equal(sh_ptr, static_cast(nullptr)); + { + legate::InternalSharedPtr itrnl_ptr{bare_ptr}; + + sh_ptr = std::move(itrnl_ptr); + // since itrnl_ptr has moved all of these should be ZERO + ASSERT_EQ(itrnl_ptr.strong_ref_count(), 0); + ASSERT_EQ(itrnl_ptr.user_ref_count(), 0); + ASSERT_EQ(itrnl_ptr.use_count(), 0); + ASSERT_EQ(sh_ptr.use_count(), 1); + test_basic_equal(sh_ptr, bare_ptr); + test_basic_equal(itrnl_ptr, static_cast(nullptr)); + } + // the pointer should not die here + ASSERT_EQ(sh_ptr.use_count(), 1); + test_basic_equal(sh_ptr, bare_ptr); +} + +TYPED_TEST(SharedPtrUnit, FromOrphanInternalMoveCascading) +{ + auto bare_ptr = new TypeParam{1}; + auto bare_ptr2 = new TypeParam{2}; + legate::SharedPtr sh_ptr; + + // sanity checks + test_basic_equal(sh_ptr, static_cast(nullptr)); + { + legate::InternalSharedPtr itrnl_ptr{bare_ptr}; + + sh_ptr = std::move(itrnl_ptr); + // since itrnl_ptr has moved all of these should be ZERO + ASSERT_EQ(itrnl_ptr.strong_ref_count(), 0); + ASSERT_EQ(itrnl_ptr.user_ref_count(), 0); + ASSERT_EQ(itrnl_ptr.use_count(), 0); + ASSERT_EQ(sh_ptr.use_count(), 1); + test_basic_equal(sh_ptr, bare_ptr); + test_basic_equal(itrnl_ptr, static_cast(nullptr)); + { + legate::InternalSharedPtr itrnl_ptr2{bare_ptr2}; + + sh_ptr = std::move(itrnl_ptr2); + // since itrnl_ptr has moved all of these should be ZERO + ASSERT_EQ(itrnl_ptr2.strong_ref_count(), 0); + ASSERT_EQ(itrnl_ptr2.user_ref_count(), 0); + ASSERT_EQ(itrnl_ptr2.use_count(), 0); + ASSERT_EQ(sh_ptr.use_count(), 1); + test_basic_equal(sh_ptr, bare_ptr2); + test_basic_equal(itrnl_ptr2, static_cast(nullptr)); + } + } + // the pointer should not die here + ASSERT_EQ(sh_ptr.use_count(), 1); + test_basic_equal(sh_ptr, bare_ptr2); +} + +TYPED_TEST(SharedPtrUnit, InternalOutlives) +{ + auto bare_ptr = new TypeParam{77}; + legate::InternalSharedPtr intrl_ptr{bare_ptr}; + + test_basic_equal(intrl_ptr, bare_ptr); + { + legate::SharedPtr sh_ptr{intrl_ptr}; + + ASSERT_EQ(intrl_ptr.strong_ref_count(), 2); + ASSERT_EQ(intrl_ptr.use_count(), 2); + ASSERT_EQ(sh_ptr.use_count(), 2); + ASSERT_EQ(intrl_ptr.user_ref_count(), 1); + + ASSERT_TRUE(intrl_ptr.get()); + ASSERT_EQ(intrl_ptr.get(), sh_ptr.get()); + ASSERT_EQ(*intrl_ptr, *sh_ptr); + ASSERT_EQ(static_cast(intrl_ptr), static_cast(sh_ptr)); + test_basic_equal(sh_ptr, bare_ptr); + test_basic_equal(intrl_ptr, bare_ptr); + } + ASSERT_EQ(intrl_ptr.strong_ref_count(), 1); + ASSERT_EQ(intrl_ptr.user_ref_count(), 0); + ASSERT_EQ(intrl_ptr.use_count(), 1); + test_basic_equal(intrl_ptr, bare_ptr); +} + +TYPED_TEST(SharedPtrUnit, MakeShared) +{ + auto sh_ptr = legate::make_shared(10); + auto bare_ptr = sh_ptr.get(); + + test_basic_equal(sh_ptr, bare_ptr); +} + +TEST(SharedPtrUnit, MakeSharedPolymorphism) +{ + legate::SharedPtr sh_ptr = legate::make_shared(10); + auto bare_ptr = static_cast(sh_ptr.get()); + + test_basic_equal(sh_ptr, bare_ptr); +} + +TEST(SharedPtrUnit, PolymorphismReset) +{ + bool toggle = false; + { + auto bare_ptr = new TogglingDerived{&toggle}; + legate::SharedPtr ptr{bare_ptr}; + + ASSERT_FALSE(toggle); // sanity check + test_basic_equal(ptr, bare_ptr); + ASSERT_FALSE(toggle); // still false + ASSERT_EQ(ptr.use_count(), 1); + + auto bare_ptr2 = new BasicDerived{45}; + + ptr.reset(bare_ptr2); + ASSERT_TRUE(toggle); // if properly handled, set to true in most derived dtor + test_basic_equal(ptr, bare_ptr2); + toggle = false; + } + ASSERT_FALSE(toggle); // toggle should not have been touched +} + +TYPED_TEST(SharedPtrUnit, UniqueCtor) +{ + auto val = TypeParam{123}; + auto uniq = std::make_unique(val); + + legate::SharedPtr sh_ptr{std::move(uniq)}; + + ASSERT_EQ(sh_ptr.use_count(), 1); + ASSERT_EQ(*sh_ptr, val); + ASSERT_EQ(uniq.get(), nullptr); + ASSERT_FALSE(uniq); +} + +TYPED_TEST(SharedPtrUnit, UniqueAssign) +{ + auto val = TypeParam{123}; + auto uniq = std::make_unique(val); + + auto bare_ptr = new TypeParam{22}; + legate::SharedPtr sh_ptr{bare_ptr}; + + test_basic_equal(sh_ptr, bare_ptr); + + sh_ptr = std::move(uniq); + + ASSERT_EQ(sh_ptr.use_count(), 1); + ASSERT_EQ(*sh_ptr, val); + ASSERT_EQ(uniq.get(), nullptr); + ASSERT_FALSE(uniq); +} diff --git a/tests/cpp/unit/shared_ptr_util.h b/tests/cpp/unit/shared_ptr_util.h new file mode 100644 index 0000000000..29689412ed --- /dev/null +++ b/tests/cpp/unit/shared_ptr_util.h @@ -0,0 +1,196 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: LicenseRef-NvidiaProprietary + * + * NVIDIA CORPORATION, its affiliates and licensors retain all intellectual + * property and proprietary rights in and to this material, related + * documentation and any modifications thereto. Any use, reproduction, + * disclosure or distribution of this material and related documentation + * without an express license agreement from NVIDIA CORPORATION or + * its affiliates is strictly prohibited. + */ + +#pragma once + +#include + +#include "core/utilities/memory.h" + +#include +#include +#include +#include + +struct UserType { + std::array before_padding{}; + std::int32_t value{}; + std::array after_padding{}; + + explicit UserType(std::int32_t v) : value{v} + { + before_padding.fill(0); + after_padding.fill(0); + } + + UserType& operator++() + { + check_mem_corruption(); + ++value; + return *this; + } + + bool operator==(const UserType& other) const + { + check_mem_corruption(); + other.check_mem_corruption(); + return before_padding == other.before_padding && value == other.value && + after_padding == other.after_padding; + } + + void check_mem_corruption() const + { + // to test for corruption + for (auto&& v : before_padding) { ASSERT_EQ(v, 0); } + for (auto&& v : after_padding) { ASSERT_EQ(v, 0); } + } +}; + +#define SHARED_PTR_UTIL_FN_BEGIN(...) SCOPED_TRACE(__VA_ARGS__) + +template